From 27104f8820b2b3efe66213963a46d1a0477bc546 Mon Sep 17 00:00:00 2001 From: Gauri Lamunion Date: Tue, 29 Oct 2024 13:51:27 -0700 Subject: [PATCH] Install libsecret on arm64 --- release-pipeline.yml | 4325 +++++++++++++++++++++--------------------- 1 file changed, 2166 insertions(+), 2159 deletions(-) diff --git a/release-pipeline.yml b/release-pipeline.yml index 32c7d013f..662f48a87 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -1,2159 +1,2166 @@ -variables: - AZCOPY_GOLANG_VERSION: '1.21.8' - -parameters: - - name: tag - displayName: 'New Release Tag' - type: string - default: 'azcopy-' - - - name: publish_to_container - displayName: 'Publish To Container' - type: boolean - default: false - - - name: publish_to_m1_container - displayName: 'Publish To M1 Container' - type: boolean - default: false - - - name: post_release - displayName: 'Post Release on Github' - type: boolean - default: false - - - name: publish_artifacts - displayName: 'Publish Artifacts to Linux Repos' - type: boolean - default: false - - - name: draft - displayName: 'Post as Draft Release' - type: boolean - default: false - - - name: prerelease - displayName: 'Post as PreRelease' - type: boolean - default: false - - - name: update_version - displayName: 'Update Version' - type: boolean - default: false - - - name: publish_docker_image - displayName: 'Build and Push Docker Image' - type: boolean - default: false - -# Do not trigger this pipeline automatically -trigger: none -pr: none - -stages: - - stage: BuildArtifacts - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - imageName: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(Build.SourcesDirectory)' - - name: archives - value: $(root_dir)/archives - - steps: - - checkout: self - displayName: 'Checkout repository' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" - displayName: 'Generate Linux AMD64' - - - script: | - GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" - displayName: 'Generate Linux AMD64 SE Integration' - - - script: | - sudo ls -lRt $(root_dir)/ - cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - template: setup/azcopy_version_check_linux.yml - parameters: - azcopy_executable: $(root_dir)/azcopy_linux_amd64 - - - script: | - linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" - - linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" - - mkdir -p $linux_amd64_dir - mkdir -p $linux_se_amd64_dir - mkdir -p $(archives) - ls -ltR $(work_dir) - displayName: 'Create required directories' - - - script: | - mkdir -p pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy - cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_amd64_dir)/ - cp NOTICE.txt $(linux_se_amd64_dir)/ - cp NOTICE.txt pkgDir/usr/bin/ - displayName: 'Copy required files for packaging' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 deb Package' - - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make AMD64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Amd Linux Build' - inputs: - rootFolderOrFile: '$(linux_amd64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to signing stage' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'ubuntu-20.04' - container: 'test-cnt-ubn-20' - AgentName: 'blobfuse-ubn20-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(Build.SourcesDirectory)' - - name: archives - value: $(root_dir)/archives - - steps: - - checkout: self - displayName: 'Checkout repository' - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" - displayName: "Installing Go tools" - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Linux' - workingDirectory: $(root_dir) - - # install dependencies required for compiling azcopy - - script: | - sudo apt-get clean - sudo apt-get update --fix-missing - sudo apt-get install pkg-config libsecret-1-dev wget -y - sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y - sudo gem install dotenv -v 2.8.1 - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # get glibc version with which build is done - - script: | - ldd --version - displayName: "GLIBC Version" - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" - displayName: 'Generate Linux ARM64' - - - script: | - GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" - displayName: 'Generate Linux ARM64 SE Integration' - - - script: | - sudo ls -lRt $(root_dir)/ - cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) - displayName: 'List the binaries and add to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish binaries' - condition: succeededOrFailed() - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy* - displayName: "Remove binaries from staging directory" - - - template: setup/azcopy_version_check_linux.yml - parameters: - azcopy_executable: $(root_dir)/azcopy_linux_arm64 - - - script: | - linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" - - linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" - - mkdir -p $linux_arm64_dir - mkdir -p $linux_se_arm64_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mkdir -p pkgDir/usr/bin/ - cp NOTICE.txt pkgDir/usr/bin/ - mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy - mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy - cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ - cp NOTICE.txt $(linux_arm64_dir)/ - cp NOTICE.txt $(linux_se_arm64_dir)/ - displayName: 'Copy required files for packaging' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n azcopy -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb - cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 deb Package' - - - script: | - fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ - -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ - --maintainer "Azcopy v-Team " \ - --url "https://github.com/Azure/azure-storage-azcopy" \ - --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" - mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm - cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make ARM64 rpm Package' - - - task: ArchiveFiles@2 - displayName: 'Archive Standard Linux Build' - inputs: - rootFolderOrFile: '$(linux_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Partner(SE) Linux Build' - inputs: - rootFolderOrFile: '$(linux_se_arm64_dir)' - archiveType: tar - archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' - continueOnError: true - - - script: | - cp -r $(archives)/* $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Add tar.gz to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Windows' - workingDirectory: $(root_dir) - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" - displayName: 'Generate Windows AMD64' - env: - GOARCH: amd64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" - displayName: 'Generate Windows i386' - env: - GOARCH: 386 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm - GOARM: 7 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm64 - GOOS: windows - CGO_ENABLED: 0 - - - script: | - go build -tags "se_integration" -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_se_arm64.exe" - displayName: 'Generate Windows ARM' - env: - GOARCH: arm64 - GOOS: windows - CGO_ENABLED: 0 - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_4 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-latest' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: $(root_dir)/archives - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet - Mac' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" - displayName: 'Generate MacOS Build with AMD64' - - - script: | - GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" - displayName: 'Test Cross-compiled MacOS Build with ARM64' - - - task: Bash@3 - displayName: 'Extract AZCopy version' - inputs: - targetType: 'inline' - script: | - azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) - echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" - - - script: | - darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" - - darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" - - mkdir -p $darwin_amd_dir - mkdir -p $darwin_arm_dir - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - cp NOTICE.txt $(darwin_amd_dir) - cp NOTICE.txt $(darwin_arm_dir) - mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy - mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_amd_dir)' - archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm_dir)' - archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - - job: Set_5 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS-14-ARM64: - imageName: 'ubuntu-latest' - type: 'linux' - pool: - vmImage: $(imageName) - - variables: - - group: AZCOPY_TESTS_VAR - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(Build.SourcesDirectory)' - - name: archives - value: $(root_dir)/archives - - name: Publish_To_M1_Container - value: ${{ parameters.publish_to_m1_container }} - - steps: - - task: GoTool@0 - env: - GO111MODULE: 'on' - inputs: - version: $(AZCOPY_GOLANG_VERSION) - - - script: | - echo 'Running GO Vet' - go vet - displayName: 'Golang Vet' - workingDirectory: $(System.DefaultWorkingDirectory) - - - script: | - CGO_ENABLED=1 go build -o "$(root_dir)/azcopy" - displayName: 'Generate build' - - - template: setup/azcopy_version_check_linux.yml - parameters: - azcopy_executable: $(root_dir)/azcopy - - - template: setup/trigger_m1_build.yml - - - script: | - mkdir -p $(archives) - darwin_arm64_m1_dir="$(work_dir)/azcopy_darwin_m1_arm64_$(azcopy_version)" - - echo "##vso[task.setvariable variable=darwin_arm64_m1_dir]$darwin_arm64_m1_dir" - mkdir -p $darwin_arm64_m1_dir - - cp NOTICE.txt $darwin_arm64_m1_dir - cp $(Build.ArtifactStagingDirectory)/azcopy $darwin_arm64_m1_dir - displayName: 'Copy required files for packaging' - - - task: ArchiveFiles@2 - displayName: 'Archive MacOS M1_ARM64 Build (must happen before signing/notarization)' - inputs: - rootFolderOrFile: '$(darwin_arm64_m1_dir)' - archiveFile: '$(archives)/azcopy_darwin_m1_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azcopy - cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-temp' - displayName: 'Publish Artifacts' - condition: succeededOrFailed() - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - - # BuildArtifacts end here - - - stage: SignArtifacts - dependsOn: BuildArtifacts - condition: succeeded('BuildArtifacts') - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Linux-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_SECRET_VAULT - - steps: - - checkout: none - - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - rm -rf azcopy_linux* - sudo ls -lRt azCopy-linux-temp - md5sum azCopy-linux-temp/* - displayName: 'List Artifacts' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - - script: | - mkdir mariner && chmod 755 mariner - cp azCopy-linux-temp/*arm64.rpm mariner - cp azCopy-linux-temp/*x86_64.rpm mariner - sudo ls -lRt mariner - md5sum mariner/* - displayName: 'Copy artifacts for Mariner' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-3.1 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) - Pattern: '*.rpm, *.deb' - DisplayName: 'ESRP CodeSigning azCopy linux' - - - template: setup/esrp_sign_linux.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/mariner - ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) - Pattern: '*.rpm, *.deb' - DisplayName: 'ESRP CodeSigning azCopy mariner' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* - chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md - rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - md5sum $(Build.ArtifactStagingDirectory)/mariner/* - displayName: 'List Signed Artifacts' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-linux-signed' - displayName: 'Publish Signed Artifacts' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Windows-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_SECRET_VAULT - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - name: archives - value: '$(work_dir)/archives' - - steps: - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download linux binary for extracting version' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(work_dir)/ - itemPattern: 'azCopy-binaries/azcopy_linux_amd64' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md - displayName: 'List Artifacts' - - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - sudo apt-get install -y zip - displayName: "Update dependencies" - - - template: setup/esrp_sign_windows.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) - Pattern: 'azcopy_windows*' - - - script: | - rm -rf azCopy-windows-temp/*.md - cp azCopy-windows-temp/* . - mv azCopy-windows-temp $(work_dir) - displayName: 'Add signed windows binary to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish Signed Artifacts' - - # Validate signed images have md5sum changed - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/* - mv $(work_dir)/azCopy-windows-temp . - rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe - chmod 755 azCopy-windows-temp/* - rm -rf azCopy-windows-temp/*.md - displayName: 'Make Artifacts executable' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - - script: | - chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 - # Run azcopy --version and capture the output - azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) - - if echo "$azcopyOutput" | grep -q "azcopy version"; then - version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') - if [[ -n "$version" ]]; then - echo "Extracted version: $version" - echo "##vso[task.setvariable variable=azcopy_version]$version" - fi - fi - displayName: 'Extract the generated build version' - - - script: | - windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" - - windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" - - windows_se_arm64="$(work_dir)/azcopy_windows_se_arm64_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_se_arm64]$windows_se_arm64" - - windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" - echo "##vso[task.setvariable variable=windows_386]$windows_386" - - mkdir -p $windows_amd64 - mkdir -p $windows_arm64 - mkdir -p $windows_se_arm64 - mkdir -p $windows_386 - mkdir -p $(archives) - displayName: 'Create required directories' - - - script: | - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe - mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_se_arm64.exe $(windows_se_arm64)/azcopy.exe - cp NOTICE.txt $(windows_amd64) - cp NOTICE.txt $(windows_arm64) - cp NOTICE.txt $(windows_386) - cp NOTICE.txt $(windows_se_arm64) - zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . - zip -r $(archives)/azcopy_windows_se_arm64_$(azcopy_version).zip . - displayName: 'Copy required files' - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit AMD Build' - inputs: - rootFolderOrFile: '$(windows_amd64)' - archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 64 bit ARM Build' - inputs: - rootFolderOrFile: '$(windows_arm64)' - archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows 32 bit Build' - inputs: - rootFolderOrFile: '$(windows_386)' - archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' - continueOnError: true - - - task: ArchiveFiles@2 - displayName: 'Archive Windows SE 64 bit ARM Build' - inputs: - rootFolderOrFile: '$(windows_se_arm64)' - archiveFile: '$(archives)/azcopy_windows_se_arm64_$(azcopy_version).zip' - continueOnError: true - - - script: | - cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) - displayName: 'Copy zip to staging directory' - - - script: | - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - rm -rf $(Build.ArtifactStagingDirectory)/*.md - displayName: 'List Signed Artifacts' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-windows-signed' - displayName: 'Publish Signed Artifacts' - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Mac-Signing: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_SECRET_VAULT - - steps: - - checkout: none - - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-temp' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - displayName: 'List Artifacts' - - - script: | - sudo apt-get clean - sudo apt-get update - sudo apt-get upgrade -y - sudo apt-get install unzip -y - sudo apt-get install -f --fix-missing - wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb - sudo dpkg -i packages-microsoft-prod.deb - sudo apt install apt-transport-https -y - sudo apt install dotnet-sdk-6.0 -y - displayName: "Update dependencies" - - - template: setup/esrp_sign_notarize_macos.yml - parameters: - FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp - ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) - Pattern: '*.zip' - - # Validate signed images have md5sum changed - - script: | - chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md - mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ - rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ - displayName: 'Make Artifacts executable' - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - md5sum $(Build.ArtifactStagingDirectory)/* - displayName: 'List Signed Artifacts' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-mac-signed' - displayName: 'Publish Signed Artifacts' - - - script: | - mkdir -p unzipped_amd64 - mkdir -p unzipped_arm64 - mkdir -p unzipped_m1_arm64 - unzip -o azcopy*amd64*.zip -d unzipped_amd64 - unzip -o azcopy_darwin_arm64*.zip -d unzipped_arm64 - unzip -o azcopy_darwin_m1_arm64*.zip -d unzipped_m1_arm64 - - # Use find to retrieve the folder name - amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") - echo "AMD64 Folder name: $amd64FolderName" - - arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") - echo "ARM64 Folder name: $arm64FolderName" - - m1Arm64FolderName=$(find unzipped_m1_arm64 -maxdepth 1 -type d -name "azcopy_darwin_m1_arm64*") - echo "ARM64 M1 Folder name: $m1Arm64FolderName" - - # Check if there is exactly one matching directory for amd64 - if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching AMD64 directory: $amd64FolderName" - cd "$amd64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 - else - echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." - exit 1 - fi - - cd $(Build.ArtifactStagingDirectory) - - # Check if there is exactly one matching directory for arm64 - if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching ARM64 directory: $arm64FolderName" - cd "$arm64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 - else - echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." - exit 1 - fi - - cd $(Build.ArtifactStagingDirectory) - - # Check if there is exactly one matching directory for arm64 - if [ $(echo "$m1Arm64FolderName" | wc -l) -eq 1 ]; then - echo "Found matching ARM64 M1 directory: $m1Arm64FolderName" - cd "$m1Arm64FolderName" - mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_m1_arm64 - else - echo "Error: Expected exactly one matching ARM64 M1 directory, found $(echo "$m1Arm64FolderName" | wc -l)." - exit 1 - fi - - rm -rf $(Build.ArtifactStagingDirectory)/unzipped* - rm -rf $(Build.ArtifactStagingDirectory)/*.zip - sudo ls -ltR $(Build.ArtifactStagingDirectory) - displayName: 'Copy and list signed mac binary to staging directory' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - # Push signed images to artifact directory - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'azCopy-binaries' - displayName: 'Publish Signed Mac Binary' - - # SignArtifacts end here - - - stage: TestArtifacts - dependsOn: SignArtifacts - condition: succeeded('SignArtifacts') - jobs: - - job: Set_1 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-binaries' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-binaries/azcopy_linux*amd64' - - - script: | - ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries/ - chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* - $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version - $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_amd64 --version - rm -rf /usr/local/bin/azcopy - rm -rf /usr/bin/azcopy - displayName: 'Test signed linux binaries' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: '$(System.DefaultWorkingDirectory)' - itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - - script: | - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Install deb Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_2 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_tar_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - steps: - - checkout: none - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy_linux_*amd64*.tar.gz' - - - script: | - TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # Create extraction directory - mkdir -p "$EXTRACT_DIR" - - # Extract the .tar.gz file - tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" - if [ $? -ne 0 ]; then - echo "Error: Failed to extract $TAR_GZ_FILE" - exit 1 - fi - - # Find the directory matching the pattern - matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) - - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - echo "Found matching directory: ${matching_dirs[0]}" - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi - - # Run azcopy commands - ./azcopy --version - ./azcopy --help - displayName: 'Check Version and Help' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - TAR_GZ_FILE=$(ls azcopy_linux_se_amd64*.tar.gz) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" - - # Create extraction directory - mkdir -p "$EXTRACT_DIR" - - # Extract the .tar.gz file - tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" - if [ $? -ne 0 ]; then - echo "Error: Failed to extract $TAR_GZ_FILE" - exit 1 - fi - - # Find the directory matching the pattern - matching_dirs=("$EXTRACT_DIR"/azcopy_linux_se_amd64*) - - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - echo "Found matching directory: ${matching_dirs[0]}" - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi - - # Run azcopy commands - ./azcopy --version - ./azcopy --help - displayName: 'Check Version and Help' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - job: Set_3 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_tar_arm64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) - steps: - - checkout: none - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy_linux_*arm64*.tar.gz' - - - script: | - TAR_GZ_FILE=$(ls azcopy_linux_arm64*.tar.gz) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # Create extraction directory - mkdir -p "$EXTRACT_DIR" - - # Extract the .tar.gz file - tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" - if [ $? -ne 0 ]; then - echo "Error: Failed to extract $TAR_GZ_FILE" - exit 1 - fi - - # Find the directory matching the pattern - matching_dirs=("$EXTRACT_DIR"/azcopy_linux_arm64*) - - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - echo "Found matching directory: ${matching_dirs[0]}" - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi - - # Run azcopy commands - ./azcopy --version - ./azcopy --help - - displayName: 'Check Version and Help for arm64 tar.gz' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - TAR_GZ_FILE=$(ls azcopy_linux_se_arm64*.tar.gz) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" - - # Create extraction directory - mkdir -p "$EXTRACT_DIR" - - # Extract the .tar.gz file - tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" - if [ $? -ne 0 ]; then - echo "Error: Failed to extract $TAR_GZ_FILE" - exit 1 - fi - - # Find the directory matching the pattern - matching_dirs=("$EXTRACT_DIR"/azcopy_linux_se_arm64*) - - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - echo "Found matching directory: ${matching_dirs[0]}" - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi - - # Run azcopy commands - ./azcopy --version - ./azcopy --help - - displayName: 'Check Version and Help for SE arm64 tar.gz' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - job: Set_4 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu_arm64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-binaries' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-binaries/azcopy_linux*arm64' - - - script: | - ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries - chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* - $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_arm64 --version - $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_arm64 --version - displayName: 'Test signed linux binaries' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: '$(System.DefaultWorkingDirectory)' - itemPattern: 'azCopy-linux-signed/azcopy-*arm64.deb' - - - script: | - rm -rf /usr/bin/azcopy - rm -rf /usr/local/bin/azcopy - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_5 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Arm64: - agentName: "blobfuse-rhel9-arm64" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9-arm64' - - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*.arm64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*.arm64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' - - - job: Set_6 - timeoutInMinutes: 120 - strategy: - matrix: - Rhel_Amd64: - agentName: "blobfuse-rhel9" - vmImage: 'RHEL-9.0' - container: 'test-cnt-rhel-9' - - pool: - name: "blobfuse-rhel-pool" - demands: - - ImageOverride -equals $(agentName) - - steps: - - checkout: none - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' - - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo rpm -qip azcopy-*x86_64.rpm - sudo yum groupinstall "Development Tools" -y - sudo rpm -i azcopy-*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' - - - job: Set_7 - timeoutInMinutes: 120 - strategy: - matrix: - Mariner2: - agentName: "blobfuse-mariner2" - DistroVer: "Mariner2" - Description: "CBL-Mariner2 Linux" - container: "test-cnt-mari-2" - - pool: - name: "blobfuse-mariner-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - - template: setup/test_artifacts.yml - parameters: - artifact_name: 'azCopy-linux-signed' - download_path: '$(System.DefaultWorkingDirectory)' - item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' - - - script: | - sudo rpm -qip azcopy*x86_64.rpm - sudo rpm -i azcopy*x86_64.rpm - displayName: 'Install Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help command' - - - job: Set_8 - timeoutInMinutes: 360 - strategy: - matrix: - MacOS: - imageName: 'macos-latest' - type: 'mac-os' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-mac-signed/azcopy*amd64*.zip - - - script: | - # Install Homebrew if not already installed - if ! command -v brew &> /dev/null - then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile - eval "$(/opt/homebrew/bin/brew shellenv)" - fi - - # Define variables - ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) - EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" - - # Create extraction directory - mkdir -p $EXTRACT_DIR - - # Extract the zip file - unzip $ZIP_FILE_PATH -d $EXTRACT_DIR - - # Find the directory matching the pattern - matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) - - # Check if there is exactly one matching directory - if [ ${#matching_dirs[@]} -eq 1 ]; then - cd "${matching_dirs[0]}" - else - echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." - exit 1 - fi - - # Run the azcopy version command - ./azcopy --version - ./azcopy --help - displayName: 'Install Zip, Extract Files, and Run Version and Help Command' - - - job: Set_9 - timeoutInMinutes: 120 - strategy: - matrix: - Windows: - imageName: 'windows-2019' - type: 'windows' - pool: - vmImage: $(imageName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - steps: - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-windows-signed/azcopy*amd64*.zip - - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: | - $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - - # Find the zip file matching the pattern - $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 - - if ($null -eq $zipFile) { - Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" - exit 1 - } - - $zipFilePath = $zipFile.FullName - - # Create extraction directory - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # Extract the zip file - try { - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - } catch { - Write-Error "Failed to extract zip file: $_" - exit 1 - } - - # Change to the directory containing azcopy - Set-Location -Path $extractDir - - # Find directories matching the pattern - $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 - - # Check if there is exactly one matching directory - if ($matchingDirs.Count -eq 0) { - Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" - exit 1 - } elseif ($matchingDirs.Count -gt 1) { - Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" - exit 1 - } - - # Check if the azcopy executable exists in the matching directory - $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" - if (-Not (Test-Path -Path $azcopyPath)) { - Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" - exit 1 - } else { - Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" - # Ensure the azcopy executable has the correct permissions - icacls $azcopyPath - - # Run azcopy --version and capture output - try { - $versionOutput = & $azcopyPath --version 2>&1 - Write-Output "azcopy version output: $versionOutput" - } catch { - Write-Error "Failed to run azcopy --version: $_" - } - - # Run azcopy --help and capture output - try { - $helpOutput = & $azcopyPath --help 2>&1 - Write-Output "azcopy help output: $helpOutput" - } catch { - Write-Error "Failed to run azcopy --help: $_" - } - } - displayName: 'Extract Files and Run Version and Help Command on Windows' - - - job: Set_10 - timeoutInMinutes: 120 - strategy: - matrix: - Debian-12_amd64: - agentName: "blobfuse-debian-12" - pool: - name: "azcopy-debian-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: '$(System.DefaultWorkingDirectory)' - itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' - - - script: | - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Install deb Package' - workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed - - - script: | - azcopy --version - azcopy --help - displayName: 'Check Version and Help' - - - job: Set_11 - timeoutInMinutes: 120 - strategy: - matrix: - Windows-ARM64: - agentName: "windows_arm64" - pool: - name: "azcopy_windows_arm" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - steps: - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - itemPattern: azCopy-windows-signed/azcopy_windows_se_arm64*.zip - - - task: PowerShell@2 - inputs: - targetType: 'inline' - script: | - $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" - $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" - - # Find the zip file matching the pattern - $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*se_arm64*.zip" | Select-Object -First 1 - - if ($null -eq $zipFile) { - Write-Error "No zip file found matching pattern 'azcopy*se_arm64*.zip' in directory: $artifactDir" - exit 1 - } - - $zipFilePath = $zipFile.FullName - - # Create extraction directory - New-Item -ItemType Directory -Path $extractDir -Force | Out-Null - - # Extract the zip file - try { - Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) - } catch { - Write-Error "Failed to extract zip file: $_" - exit 1 - } - - # Change to the directory containing azcopy - Set-Location -Path $extractDir - - # Find directories matching the pattern - $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_se_arm64* | Select-Object -First 1 - - # Check if there is exactly one matching directory - if ($matchingDirs.Count -eq 0) { - Write-Error "No directories found matching the pattern: .\azcopy_windows_se_arm64*" - exit 1 - } elseif ($matchingDirs.Count -gt 1) { - Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_se_arm64*" - exit 1 - } - - # Check if the azcopy executable exists in the matching directory - $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" - if (-Not (Test-Path -Path $azcopyPath)) { - Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" - exit 1 - } else { - Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" - # Ensure the azcopy executable has the correct permissions - icacls $azcopyPath - - # Run azcopy --version and capture output - try { - $versionOutput = & $azcopyPath --version 2>&1 - Write-Output "azcopy version output: $versionOutput" - } catch { - Write-Error "Failed to run azcopy --version: $_" - } - - # Run azcopy --help and capture output - try { - $helpOutput = & $azcopyPath --help 2>&1 - Write-Output "azcopy help output: $helpOutput" - } catch { - Write-Error "Failed to run azcopy --help: $_" - } - } - displayName: 'Extract Files and Run Version and Help Command on Windows Arm64' - - # TestArtifacts ends here - - - stage: ReleaseToContainer - dependsOn: TestArtifacts - condition: succeeded('TestArtifacts') - jobs: - - job: ReleaseToContainer - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - - variables: - - name: Test_Release_Account_Name - value: 'azcopyprivatedrops' - - name: Release_Account_Name - value: 'azcopyvnextrelease' - - name: Publish_To_Container - value: ${{ parameters.publish_to_container }} - - name: Publish_To_M1_Container - value: ${{ parameters.publish_to_m1_container }} - - name: work_dir - value: '$(Build.SourcesDirectory)' - - steps: - - task: DownloadBuildArtifacts@0 - displayName: 'Download Linux Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Windows Build Artifacts' - inputs: - artifactName: 'azCopy-windows-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Mac Build Artifacts' - inputs: - artifactName: 'azCopy-mac-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Binaries Build Artifacts' - inputs: - artifactName: 'azCopy-binaries' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - sudo apt-get install -y zip - cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . - rm -rf /usr/local/bin/azcopy - rm -rf /usr/bin/azcopy - sudo dpkg --info azcopy-*x86_64.deb - sudo dpkg -i azcopy-*x86_64.deb - sudo apt-get install build-essential -y - displayName: 'Create drop.zip file and install azcopy deb package' - - - script: | - cp $(work_dir)/NOTICE.txt $(Build.ArtifactStagingDirectory)/azCopy-binaries - cd $(Build.ArtifactStagingDirectory)/azCopy-binaries - echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries:" - ls -l - - # Create the zip archive with all files inside azCopy-binaries - zip -r $(Build.ArtifactStagingDirectory)/drop.zip . - cd $(Build.ArtifactStagingDirectory) - - # List the contents of the resulting zip file for verification - unzip -l $(Build.ArtifactStagingDirectory)/drop.zip - - mv $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/mariner . - displayName: 'Archive all binaries to drop.zip' - workingDirectory: '$(Build.ArtifactStagingDirectory)' - - - task: Bash@3 - displayName: 'Install Azure CLI' - inputs: - targetType: 'inline' - script: | - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - - task: AzureCLI@2 - displayName: 'Azure CLI' - inputs: - azureSubscription: 'ESRP KeyVault identity' - scriptType: bash - scriptLocation: inlineScript - inlineScript: | - # Find the current version - raw_version_output=$(azcopy --version) - azcopy_version=$(echo $raw_version_output | awk '{print $3}') - - today=$(date +"%Y%m%d") - - if [ $(Publish_To_Container) = "True" ]; then - container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - else - container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" - fi - echo "Artifacts will be uploaded to: $container_url" - - executable_name=azcopy - - # Upload the archived builds - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true - - if [ $(Publish_To_M1_Container) = "True" ]; then - m1_container_url="https://azcopyvnextrelease.blob.core.windows.net/azcopy-m1-drops/azcopy-$azcopy_version-$today" - AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$m1_container_url" --put-md5=true - fi - - - ${{ if eq(parameters.publish_docker_image, true) }}: - - stage: BuildAndPublishDockerImage - dependsOn: TestArtifacts - condition: succeeded('TestArtifacts') - jobs: - - job: Set_1_Ubuntu_Mariner_AMD64 - strategy: - matrix: - Ubuntu_amd64: - agentName: "blobfuse-ubuntu22" - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22' - pool: - vmImage: $(vmImage) - - variables: - - group: AZCOPY_SECRET_VAULT - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout branch' - workingDirectory: $(work_dir) - - - script: | - chmod 777 *.sh - ./dockerinstall.sh - - ./buildcontainer.sh Dockerfile ubuntu-x86_64 - ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ubuntu-x86_64 - - ./buildcontainer.sh DockerfileMariner mariner-x86_64 - ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) mariner-x86_64 - - displayName: "Create docker image and push to the containers registry" - workingDirectory: $(work_dir)/docker - - - job: Set_2_Ubuntu_ARM64 - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22-ARM64: - vmImage: 'Ubuntu-22.04' - container: 'test-cnt-ubn-22-arm64' - AgentName: "blobfuse-ubn22-arm64" - pool: - name: "blobfuse-ubn-arm64-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: AZCOPY_SECRET_VAULT - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' - - steps: - - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-azcopy - displayName: 'Checkout Code' - workingDirectory: $(root_dir) - - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout branch' - workingDirectory: $(work_dir) - - - task: ShellScript@2 - inputs: - scriptPath: "$(work_dir)/go_installer.sh" - args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" - displayName: "Installing Go tools" - - - script: | - sudo apt update - sudo apt --fix-broken install - displayName: "Install dependencies" - - - script: | - chmod 777 *.sh - ./dockerinstall.sh - ./buildcontainer.sh DockerfileArm64 ubuntu-arm64 - ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ubuntu-arm64 - displayName: "Create docker image for arm64 and push to the containers registry" - workingDirectory: $(work_dir)/docker - - - ${{ if eq(parameters.post_release, true) }}: - - stage: ReleaseToGithub - dependsOn: TestArtifacts - condition: succeeded('TestArtifacts') - jobs: - - job: ReleaseToGithub - timeoutInMinutes: 120 - strategy: - matrix: - Ubuntu-22: - vmImage: 'Ubuntu-20.04' - AgentName: "blobfuse-ubuntu20" - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - steps: - - checkout: none - - - script: | - echo ${{ parameters.tag }} - displayName: 'Tag Name' - - - task: GithubRelease@1 - inputs: - githubConnection: 'azcopy-github-connection' - repositoryName: 'Azure/azure-storage-azcopy' - action: 'edit' - - target: '$(Build.SourceVersion)' - tagSource: 'userSpecifiedTag' - - title: ${{ parameters.tag }} - tag: ${{ parameters.tag }} - - changeLogCompareToRelease: 'lastFullRelease' - changeLogType: 'commitBased' - isDraft: ${{ parameters.draft }} - isPreRelease: ${{ parameters.prerelease }} - assetUploadMode: replace - - - ${{ if eq(parameters.publish_artifacts, true) }}: - - stage: PublishArtifacts - dependsOn: ReleaseToContainer - condition: succeeded('ReleaseToContainer') - jobs: - - job: PublishArtifacts - timeoutInMinutes: 120 - pool: - vmImage: 'ubuntu-22.04' - variables: - - group: AZCOPY_SECRET_VAULT - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - steps: - - checkout: none - - - task: PipAuthenticate@1 - inputs: - artifactFeeds: 'DevExGlobalFeed' - displayName: 'Connect to PMC artifact' - - - script: | - pip install pmc-cli - displayName: 'Install pmc-cli' - - - task: DownloadSecureFile@1 - name: pmcCertificate - displayName: 'Download pmc pem file' - inputs: - secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' - - - task: DownloadSecureFile@1 - name: settings - displayName: 'Download settings.toml file' - inputs: - secureFile: 'settings.toml' - - - script: | - pmc --version - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Test PMC installation' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Signed Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - - script: | - cd mariner - arm64file=$(ls azcopy*.arm64.rpm) - amd64file=$(ls azcopy*.x86_64.rpm) - marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" - marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" - mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" - mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" - sudo ls -lRt - mv azcopy*.rpm .. - cd .. - rm -r mariner/ - displayName: 'Rename Mariner binaries' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - - script: | - sudo ls -lRt $(Build.ArtifactStagingDirectory) - displayName: 'List Artifacts' - - - script: | - mv azCopy-linux-signed/*tar.gz . - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed - displayName: 'Upload files' - workingDirectory: $(Build.ArtifactStagingDirectory) - - - script: | - wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv - cat ./packages.csv - displayName: 'Fetch packages.csv' - workingDirectory: $(Build.ArtifactStagingDirectory)/ - - - script: | - azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` - echo "Azcopy AMD DEB ID: $azcopyAmdDeb" - - azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') - azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) - echo "Azcopy AMD RPM ID: $azcopyAmdRpm" - - azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` - echo "Azcopy ARM DEB ID: $azcopyArmDeb" - - azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') - azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) - echo "Azcopy ARM RPM $azcopyArmRpmFile" - echo "Azcopy ARM RPM ID: $azcopyArmRpm" - - # Find the Mariner RPM file and upload - marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') - marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) - echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" - echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" - - marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') - marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) - echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" - echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" - - is_preview="false" - echo "##vso[task.setvariable variable=is_preview]$is_preview" - if [[ $marinerAmdRpmFile == *"preview"* ]]; then - is_preview="true" - echo "##vso[task.setvariable variable=is_preview]$is_preview" - fi - - while IFS=, read -r distro archetype repoName releaseName; do - - # If the package is preview, publish to mariner preview package - if [[ $distro == *"Mariner-"* ]]; then - if [ $is_preview = "true" ]; then - repoName=$(echo $repoName | sed 's/prod/preview/') - fi - fi - - echo "Uploading packages for $distro" - echo "Repo Name: $repoName" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archetype} $repoName $releaseName - done < <(tail -n +3 ../packages.csv) - displayName: 'Add uploaded packages to repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - - script: | - while IFS=, read -r distro archetype repoName releaseName; do - if [[ $archetype == *"Arm"* ]]; then - echo "Skipping for ARM type on $distro" - else - if [[ $distro == *"Mariner-"* ]]; then - if [ "$(is_preview)" = "true" ]; then - repoName=$(echo $repoName | sed 's/prod/preview/') - fi - fi - echo "Repo Name: $repoName" - pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName - fi - done < <(tail -n +3 ../packages.csv) - displayName: 'Publish the repository' - workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ - - - ${{ if eq(parameters.update_version, true) }}: - - stage: UpdateLatestVersion - dependsOn: ReleaseToContainer - condition: succeeded('ReleaseToContainer') - jobs: - - job: UpdateVersion - pool: - vmImage: 'ubuntu-20.04' - variables: - - group: AZCOPY_TESTS_VAR - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - steps: - - checkout: none - - - script: | - sudo apt-get install python3 -y - python3 --version - displayName: 'Installing Python' - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Build Artifacts' - inputs: - artifactName: 'azCopy-linux-signed' - downloadPath: $(Build.ArtifactStagingDirectory) - - # install azcopy - - script: | - rm -rf /usr/local/bin/azcopy - rm -rf /usr/bin/azcopy - cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed - ls azcopy*.arm64.deb - sudo dpkg --info azcopy-*arm64.deb - sudo dpkg -i azcopy-*arm64.deb - azcopy --version - displayName: 'Installing azcopy' - - - script: | - wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py - ls -l - displayName: 'Getting Python script' - workingDirectory: $(root_dir) - - - script: | - pip install azure-storage-blob - python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" - displayName: 'Updating version number' - workingDirectory: $(root_dir) +variables: + AZCOPY_GOLANG_VERSION: '1.21.8' + +parameters: + - name: tag + displayName: 'New Release Tag' + type: string + default: 'azcopy-' + + - name: publish_to_container + displayName: 'Publish To Container' + type: boolean + default: false + + - name: publish_to_m1_container + displayName: 'Publish To M1 Container' + type: boolean + default: false + + - name: post_release + displayName: 'Post Release on Github' + type: boolean + default: false + + - name: publish_artifacts + displayName: 'Publish Artifacts to Linux Repos' + type: boolean + default: false + + - name: draft + displayName: 'Post as Draft Release' + type: boolean + default: false + + - name: prerelease + displayName: 'Post as PreRelease' + type: boolean + default: false + + - name: update_version + displayName: 'Update Version' + type: boolean + default: false + + - name: publish_docker_image + displayName: 'Build and Push Docker Image' + type: boolean + default: false + +# Do not trigger this pipeline automatically +trigger: none +pr: none + +stages: + - stage: BuildArtifacts + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + imageName: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(Build.SourcesDirectory)' + - name: archives + value: $(root_dir)/archives + + steps: + - checkout: self + displayName: 'Checkout repository' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_amd64" + displayName: 'Generate Linux AMD64' + + - script: | + GOARCH=amd64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_amd64" + displayName: 'Generate Linux AMD64 SE Integration' + + - script: | + sudo ls -lRt $(root_dir)/ + cp $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - template: setup/azcopy_version_check_linux.yml + parameters: + azcopy_executable: $(root_dir)/azcopy_linux_amd64 + + - script: | + linux_amd64_dir="$(work_dir)/azcopy_linux_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_amd64_dir]$linux_amd64_dir" + + linux_se_amd64_dir="$(work_dir)/azcopy_linux_se_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_amd64_dir]$linux_se_amd64_dir" + + mkdir -p $linux_amd64_dir + mkdir -p $linux_se_amd64_dir + mkdir -p $(archives) + ls -ltR $(work_dir) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_amd64 $(linux_amd64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_amd64 $(linux_se_amd64_dir)/azcopy + cp $(linux_amd64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_amd64_dir)/ + cp NOTICE.txt $(linux_se_amd64_dir)/ + cp NOTICE.txt pkgDir/usr/bin/ + displayName: 'Copy required files for packaging' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.x86_64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make AMD64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Amd Linux Build' + inputs: + rootFolderOrFile: '$(linux_amd64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_amd64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to signing stage' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'ubuntu-20.04' + container: 'test-cnt-ubn-20' + AgentName: 'blobfuse-ubn20-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(Build.SourcesDirectory)' + - name: archives + value: $(root_dir)/archives + + steps: + - checkout: self + displayName: 'Checkout repository' + + # Custom script to install Go-lang + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + displayName: "Installing Go tools" + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Linux' + workingDirectory: $(root_dir) + + # install dependencies required for compiling azcopy + - script: | + sudo apt-get clean + sudo apt-get update --fix-missing + sudo apt-get install pkg-config libsecret-1-dev wget -y + sudo apt-get install ruby ruby-dev rubygems libglib2.0-dev build-essential rpm -y + sudo gem install dotenv -v 2.8.1 + sudo gem install fpm -V + displayName: "Installing Dependencies" + + # get glibc version with which build is done + - script: | + ldd --version + displayName: "GLIBC Version" + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo" -o "$(root_dir)/azcopy_linux_arm64" + displayName: 'Generate Linux ARM64' + + - script: | + GOARCH=arm64 GOOS=linux go build -tags "netgo,se_integration" -o "$(root_dir)/azcopy_linux_se_arm64" + displayName: 'Generate Linux ARM64 SE Integration' + + - script: | + sudo ls -lRt $(root_dir)/ + cp -r $(root_dir)/azcopy_linux* $(Build.ArtifactStagingDirectory) + displayName: 'List the binaries and add to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish binaries' + condition: succeededOrFailed() + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy* + displayName: "Remove binaries from staging directory" + + - template: setup/azcopy_version_check_linux.yml + parameters: + azcopy_executable: $(root_dir)/azcopy_linux_arm64 + + - script: | + linux_arm64_dir="$(work_dir)/azcopy_linux_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_arm64_dir]$linux_arm64_dir" + + linux_se_arm64_dir="$(work_dir)/azcopy_linux_se_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=linux_se_arm64_dir]$linux_se_arm64_dir" + + mkdir -p $linux_arm64_dir + mkdir -p $linux_se_arm64_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mkdir -p pkgDir/usr/bin/ + cp NOTICE.txt pkgDir/usr/bin/ + mv $(root_dir)/azcopy_linux_arm64 $(linux_arm64_dir)/azcopy + mv $(root_dir)/azcopy_linux_se_arm64 $(linux_se_arm64_dir)/azcopy + cp $(linux_arm64_dir)/azcopy pkgDir/usr/bin/ + cp NOTICE.txt $(linux_arm64_dir)/ + cp NOTICE.txt $(linux_se_arm64_dir)/ + displayName: 'Copy required files for packaging' + + # using fpm tool for packaging of our binary & performing post-install operations + # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ + - script: | + fpm -s dir -t deb -n azcopy -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.deb ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.deb + cp ./azcopy*.deb $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 deb Package' + + - script: | + fpm -s dir -t rpm -n azcopy --rpm-digest sha256 -C pkgDir/ \ + -v `./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3` \ + --maintainer "Azcopy v-Team " \ + --url "https://github.com/Azure/azure-storage-azcopy" \ + --description "A command-line utility that is used to copy data to and from containers and file shares in Azure Storage accounts" + mv ./azcopy*.rpm ./azcopy-`./pkgDir/usr/bin/azcopy --version | cut -d " " -f 3`.arm64.rpm + cp ./azcopy*.rpm $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Make ARM64 rpm Package' + + - task: ArchiveFiles@2 + displayName: 'Archive Standard Linux Build' + inputs: + rootFolderOrFile: '$(linux_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Partner(SE) Linux Build' + inputs: + rootFolderOrFile: '$(linux_se_arm64_dir)' + archiveType: tar + archiveFile: '$(archives)/azcopy_linux_se_arm64_$(azcopy_version).tar.gz' + continueOnError: true + + - script: | + cp -r $(archives)/* $(Build.ArtifactStagingDirectory) + workingDirectory: $(root_dir) + displayName: 'Add tar.gz to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Windows' + workingDirectory: $(root_dir) + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" + displayName: 'Generate Windows AMD64' + env: + GOARCH: amd64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_386.exe" + displayName: 'Generate Windows i386' + env: + GOARCH: 386 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_v7_arm.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm + GOARM: 7 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - script: | + go build -tags "se_integration" -o "$(Build.ArtifactStagingDirectory)\azcopy_windows_se_arm64.exe" + displayName: 'Generate Windows ARM' + env: + GOARCH: arm64 + GOOS: windows + CGO_ENABLED: 0 + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_4 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-latest' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: $(root_dir)/archives + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet - Mac' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_amd64" + displayName: 'Generate MacOS Build with AMD64' + + - script: | + GOARCH=arm64 CGO_ENABLED=1 go build -o "$(root_dir)/azcopy_darwin_arm64" + displayName: 'Test Cross-compiled MacOS Build with ARM64' + + - task: Bash@3 + displayName: 'Extract AZCopy version' + inputs: + targetType: 'inline' + script: | + azcopy_version=$($(root_dir)/azcopy_darwin_amd64 --version | awk '{print $3}' | cut -d'-' -f1) + echo "##vso[task.setvariable variable=azcopy_version]$azcopy_version" + + - script: | + darwin_amd_dir="$(work_dir)/azcopy_darwin_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_amd_dir]$darwin_amd_dir" + + darwin_arm_dir="$(work_dir)/azcopy_darwin_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=darwin_arm_dir]$darwin_arm_dir" + + mkdir -p $darwin_amd_dir + mkdir -p $darwin_arm_dir + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + cp NOTICE.txt $(darwin_amd_dir) + cp NOTICE.txt $(darwin_arm_dir) + mv $(root_dir)/azcopy_darwin_amd64 $(darwin_amd_dir)/azcopy + mv $(root_dir)/azcopy_darwin_arm64 $(darwin_arm_dir)/azcopy + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS AMD64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_amd_dir)' + archiveFile: '$(archives)/azcopy_darwin_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm_dir)' + archiveFile: '$(archives)/azcopy_darwin_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + - job: Set_5 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS-14-ARM64: + imageName: 'ubuntu-latest' + type: 'linux' + pool: + vmImage: $(imageName) + + variables: + - group: AZCOPY_TESTS_VAR + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(Build.SourcesDirectory)' + - name: archives + value: $(root_dir)/archives + - name: Publish_To_M1_Container + value: ${{ parameters.publish_to_m1_container }} + + steps: + - task: GoTool@0 + env: + GO111MODULE: 'on' + inputs: + version: $(AZCOPY_GOLANG_VERSION) + + - script: | + echo 'Running GO Vet' + go vet + displayName: 'Golang Vet' + workingDirectory: $(System.DefaultWorkingDirectory) + + - script: | + CGO_ENABLED=1 go build -o "$(root_dir)/azcopy" + displayName: 'Generate build' + + - template: setup/azcopy_version_check_linux.yml + parameters: + azcopy_executable: $(root_dir)/azcopy + + - template: setup/trigger_m1_build.yml + + - script: | + mkdir -p $(archives) + darwin_arm64_m1_dir="$(work_dir)/azcopy_darwin_m1_arm64_$(azcopy_version)" + + echo "##vso[task.setvariable variable=darwin_arm64_m1_dir]$darwin_arm64_m1_dir" + mkdir -p $darwin_arm64_m1_dir + + cp NOTICE.txt $darwin_arm64_m1_dir + cp $(Build.ArtifactStagingDirectory)/azcopy $darwin_arm64_m1_dir + displayName: 'Copy required files for packaging' + + - task: ArchiveFiles@2 + displayName: 'Archive MacOS M1_ARM64 Build (must happen before signing/notarization)' + inputs: + rootFolderOrFile: '$(darwin_arm64_m1_dir)' + archiveFile: '$(archives)/azcopy_darwin_m1_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azcopy + cp $(archives)/azcopy_darwin* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-temp' + displayName: 'Publish Artifacts' + condition: succeededOrFailed() + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + + # BuildArtifacts end here + + - stage: SignArtifacts + dependsOn: BuildArtifacts + condition: succeeded('BuildArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Linux-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_SECRET_VAULT + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + rm -rf azcopy_linux* + sudo ls -lRt azCopy-linux-temp + md5sum azCopy-linux-temp/* + displayName: 'List Artifacts' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + - script: | + mkdir mariner && chmod 755 mariner + cp azCopy-linux-temp/*arm64.rpm mariner + cp azCopy-linux-temp/*x86_64.rpm mariner + sudo ls -lRt mariner + md5sum mariner/* + displayName: 'Copy artifacts for Mariner' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-3.1 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_KEY_CODE) + Pattern: '*.rpm, *.deb' + DisplayName: 'ESRP CodeSigning azCopy linux' + + - template: setup/esrp_sign_linux.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/mariner + ESRP_AZCOPY_KEY_CODE: $(ESRP_AZCOPY_MARINER_KEY_CODE) + Pattern: '*.rpm, *.deb' + DisplayName: 'ESRP CodeSigning azCopy mariner' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* + chmod 755 $(Build.ArtifactStagingDirectory)/mariner/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/*.md + rm -rf $(Build.ArtifactStagingDirectory)/mariner/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-linux-temp + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + md5sum $(Build.ArtifactStagingDirectory)/mariner/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-linux-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_SECRET_VAULT + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + - name: archives + value: '$(work_dir)/archives' + + steps: + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download linux binary for extracting version' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(work_dir)/ + itemPattern: 'azCopy-binaries/azcopy_linux_amd64' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/*.md + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + sudo apt-get install -y zip + displayName: "Update dependencies" + + - template: setup/esrp_sign_windows.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE) + Pattern: 'azcopy_windows*' + + - script: | + rm -rf azCopy-windows-temp/*.md + cp azCopy-windows-temp/* . + mv azCopy-windows-temp $(work_dir) + displayName: 'Add signed windows binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Artifacts' + + # Validate signed images have md5sum changed + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/* + mv $(work_dir)/azCopy-windows-temp . + rm -rf azCopy-windows-temp/azcopy_windows_v7_arm.exe + chmod 755 azCopy-windows-temp/* + rm -rf azCopy-windows-temp/*.md + displayName: 'Make Artifacts executable' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + - script: | + chmod 755 $(work_dir)/azCopy-binaries/azcopy_linux_amd64 + # Run azcopy --version and capture the output + azcopyOutput=$($(work_dir)/azCopy-binaries/azcopy_linux_amd64 --version) + + if echo "$azcopyOutput" | grep -q "azcopy version"; then + version=$(echo "$azcopyOutput" | sed -n 's/^azcopy version \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/p') + if [[ -n "$version" ]]; then + echo "Extracted version: $version" + echo "##vso[task.setvariable variable=azcopy_version]$version" + fi + fi + displayName: 'Extract the generated build version' + + - script: | + windows_amd64="$(work_dir)/azcopy_windows_amd64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_amd64]$windows_amd64" + + windows_arm64="$(work_dir)/azcopy_windows_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_arm64]$windows_arm64" + + windows_se_arm64="$(work_dir)/azcopy_windows_se_arm64_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_se_arm64]$windows_se_arm64" + + windows_386="$(work_dir)/azcopy_windows_386_$(azcopy_version)" + echo "##vso[task.setvariable variable=windows_386]$windows_386" + + mkdir -p $windows_amd64 + mkdir -p $windows_arm64 + mkdir -p $windows_se_arm64 + mkdir -p $windows_386 + mkdir -p $(archives) + displayName: 'Create required directories' + + - script: | + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_amd64.exe $(windows_amd64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_arm64.exe $(windows_arm64)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_386.exe $(windows_386)/azcopy.exe + mv $(Build.ArtifactStagingDirectory)/azCopy-windows-temp/azcopy_windows_se_arm64.exe $(windows_se_arm64)/azcopy.exe + cp NOTICE.txt $(windows_amd64) + cp NOTICE.txt $(windows_arm64) + cp NOTICE.txt $(windows_386) + cp NOTICE.txt $(windows_se_arm64) + zip -r $(archives)/azcopy_windows_amd64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_arm64_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_386_$(azcopy_version).zip . + zip -r $(archives)/azcopy_windows_se_arm64_$(azcopy_version).zip . + displayName: 'Copy required files' + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit AMD Build' + inputs: + rootFolderOrFile: '$(windows_amd64)' + archiveFile: '$(archives)/azcopy_windows_amd64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_arm64)' + archiveFile: '$(archives)/azcopy_windows_arm64_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows 32 bit Build' + inputs: + rootFolderOrFile: '$(windows_386)' + archiveFile: '$(archives)/azcopy_windows_386_$(azcopy_version).zip' + continueOnError: true + + - task: ArchiveFiles@2 + displayName: 'Archive Windows SE 64 bit ARM Build' + inputs: + rootFolderOrFile: '$(windows_se_arm64)' + archiveFile: '$(archives)/azcopy_windows_se_arm64_$(azcopy_version).zip' + continueOnError: true + + - script: | + cp $(archives)/azcopy_windows* $(Build.ArtifactStagingDirectory) + displayName: 'Copy zip to staging directory' + + - script: | + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-windows-temp + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + rm -rf $(Build.ArtifactStagingDirectory)/*.md + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-windows-signed' + displayName: 'Publish Signed Artifacts' + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Mac-Signing: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_SECRET_VAULT + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-temp' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + displayName: 'List Artifacts' + + - script: | + sudo apt-get clean + sudo apt-get update + sudo apt-get upgrade -y + sudo apt-get install unzip -y + sudo apt-get install -f --fix-missing + wget https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb + sudo dpkg -i packages-microsoft-prod.deb + sudo apt install apt-transport-https -y + sudo apt install dotnet-sdk-6.0 -y + displayName: "Update dependencies" + + - template: setup/esrp_sign_notarize_macos.yml + parameters: + FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-mac-temp + ESRP_AZCOPY_MAC_KEY_CODE: $(ESRP_AZCOPY_MAC_KEY_CODE) + Pattern: '*.zip' + + # Validate signed images have md5sum changed + - script: | + chmod 755 $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/*.md + mv $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/* $(Build.ArtifactStagingDirectory)/ + rm -rf $(Build.ArtifactStagingDirectory)/azCopy-mac-temp/ + displayName: 'Make Artifacts executable' + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + md5sum $(Build.ArtifactStagingDirectory)/* + displayName: 'List Signed Artifacts' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-mac-signed' + displayName: 'Publish Signed Artifacts' + + - script: | + mkdir -p unzipped_amd64 + mkdir -p unzipped_arm64 + mkdir -p unzipped_m1_arm64 + unzip -o azcopy*amd64*.zip -d unzipped_amd64 + unzip -o azcopy_darwin_arm64*.zip -d unzipped_arm64 + unzip -o azcopy_darwin_m1_arm64*.zip -d unzipped_m1_arm64 + + # Use find to retrieve the folder name + amd64FolderName=$(find unzipped_amd64 -maxdepth 1 -type d -name "azcopy_darwin_amd64*") + echo "AMD64 Folder name: $amd64FolderName" + + arm64FolderName=$(find unzipped_arm64 -maxdepth 1 -type d -name "azcopy_darwin_arm64*") + echo "ARM64 Folder name: $arm64FolderName" + + m1Arm64FolderName=$(find unzipped_m1_arm64 -maxdepth 1 -type d -name "azcopy_darwin_m1_arm64*") + echo "ARM64 M1 Folder name: $m1Arm64FolderName" + + # Check if there is exactly one matching directory for amd64 + if [ $(echo "$amd64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching AMD64 directory: $amd64FolderName" + cd "$amd64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_amd64 + else + echo "Error: Expected exactly one matching AMD64 directory, found $(echo "$amd64FolderName" | wc -l)." + exit 1 + fi + + cd $(Build.ArtifactStagingDirectory) + + # Check if there is exactly one matching directory for arm64 + if [ $(echo "$arm64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching ARM64 directory: $arm64FolderName" + cd "$arm64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_arm64 + else + echo "Error: Expected exactly one matching ARM64 directory, found $(echo "$arm64FolderName" | wc -l)." + exit 1 + fi + + cd $(Build.ArtifactStagingDirectory) + + # Check if there is exactly one matching directory for arm64 + if [ $(echo "$m1Arm64FolderName" | wc -l) -eq 1 ]; then + echo "Found matching ARM64 M1 directory: $m1Arm64FolderName" + cd "$m1Arm64FolderName" + mv azcopy $(Build.ArtifactStagingDirectory)/azcopy_darwin_m1_arm64 + else + echo "Error: Expected exactly one matching ARM64 M1 directory, found $(echo "$m1Arm64FolderName" | wc -l)." + exit 1 + fi + + rm -rf $(Build.ArtifactStagingDirectory)/unzipped* + rm -rf $(Build.ArtifactStagingDirectory)/*.zip + sudo ls -ltR $(Build.ArtifactStagingDirectory) + displayName: 'Copy and list signed mac binary to staging directory' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + # Push signed images to artifact directory + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'azCopy-binaries' + displayName: 'Publish Signed Mac Binary' + + # SignArtifacts end here + + - stage: TestArtifacts + dependsOn: SignArtifacts + condition: succeeded('SignArtifacts') + jobs: + - job: Set_1 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-binaries' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-binaries/azcopy_linux*amd64' + + - script: | + ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries/ + chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_amd64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_amd64 --version + rm -rf /usr/local/bin/azcopy + rm -rf /usr/bin/azcopy + displayName: 'Test signed linux binaries' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: '$(System.DefaultWorkingDirectory)' + itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_2 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_tar_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + steps: + - checkout: none + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy_linux_*amd64*.tar.gz' + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_amd64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_se_amd64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_se_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + displayName: 'Check Version and Help' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - job: Set_3 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_tar_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + steps: + - checkout: none + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy_linux_*arm64*.tar.gz' + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_arm64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_arm64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + + displayName: 'Check Version and Help for arm64 tar.gz' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + TAR_GZ_FILE=$(ls azcopy_linux_se_arm64*.tar.gz) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted_se" + + # Create extraction directory + mkdir -p "$EXTRACT_DIR" + + # Extract the .tar.gz file + tar -xzvf "$TAR_GZ_FILE" -C "$EXTRACT_DIR" + if [ $? -ne 0 ]; then + echo "Error: Failed to extract $TAR_GZ_FILE" + exit 1 + fi + + # Find the directory matching the pattern + matching_dirs=("$EXTRACT_DIR"/azcopy_linux_se_arm64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + echo "Found matching directory: ${matching_dirs[0]}" + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run azcopy commands + ./azcopy --version + ./azcopy --help + + displayName: 'Check Version and Help for SE arm64 tar.gz' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - job: Set_4 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu_arm64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + + - script: | + sudo apt update + sudo apt-get install libsecret-1-dev -y + sudo apt --fix-broken install + ldd --version + displayName: "GLIBC Version" + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-binaries' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-binaries/azcopy_linux*arm64' + + - script: | + ls -ltR $(System.DefaultWorkingDirectory)/azCopy-binaries + chmod 755 $(System.DefaultWorkingDirectory)/azCopy-binaries/* + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_arm64 --version + $(System.DefaultWorkingDirectory)/azCopy-binaries/azcopy_linux_se_arm64 --version + displayName: 'Test signed linux binaries' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: '$(System.DefaultWorkingDirectory)' + itemPattern: 'azCopy-linux-signed/azcopy-*arm64.deb' + + - script: | + rm -rf /usr/bin/azcopy + rm -rf /usr/local/bin/azcopy + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_5 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Arm64: + agentName: "blobfuse-rhel9-arm64" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9-arm64' + + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*arm64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*.arm64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*.arm64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_6 + timeoutInMinutes: 120 + strategy: + matrix: + Rhel_Amd64: + agentName: "blobfuse-rhel9" + vmImage: 'RHEL-9.0' + container: 'test-cnt-rhel-9' + + pool: + name: "blobfuse-rhel-pool" + demands: + - ImageOverride -equals $(agentName) + + steps: + - checkout: none + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/azcopy-*x86_64.rpm' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo rpm -qip azcopy-*x86_64.rpm + sudo yum groupinstall "Development Tools" -y + sudo rpm -i azcopy-*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_7 + timeoutInMinutes: 120 + strategy: + matrix: + Mariner2: + agentName: "blobfuse-mariner2" + DistroVer: "Mariner2" + Description: "CBL-Mariner2 Linux" + container: "test-cnt-mari-2" + + pool: + name: "blobfuse-mariner-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + + - template: setup/test_artifacts.yml + parameters: + artifact_name: 'azCopy-linux-signed' + download_path: '$(System.DefaultWorkingDirectory)' + item_pattern: 'azCopy-linux-signed/mariner/azcopy-*x86_64.rpm' + + - script: | + sudo rpm -qip azcopy*x86_64.rpm + sudo rpm -i azcopy*x86_64.rpm + displayName: 'Install Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed/mariner + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help command' + + - job: Set_8 + timeoutInMinutes: 360 + strategy: + matrix: + MacOS: + imageName: 'macos-latest' + type: 'mac-os' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-mac-signed/azcopy*amd64*.zip + + - script: | + # Install Homebrew if not already installed + if ! command -v brew &> /dev/null + then + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo 'eval "$(/opt/homebrew/bin/brew shellenv)"' >> /Users/$(whoami)/.zprofile + eval "$(/opt/homebrew/bin/brew shellenv)" + fi + + # Define variables + ZIP_FILE_PATH=$(ls $(Build.ArtifactStagingDirectory)/azCopy-mac-signed/azcopy*amd64*.zip) + EXTRACT_DIR="$(Build.ArtifactStagingDirectory)/extracted" + + # Create extraction directory + mkdir -p $EXTRACT_DIR + + # Extract the zip file + unzip $ZIP_FILE_PATH -d $EXTRACT_DIR + + # Find the directory matching the pattern + matching_dirs=($EXTRACT_DIR/azcopy_darwin_amd64*) + + # Check if there is exactly one matching directory + if [ ${#matching_dirs[@]} -eq 1 ]; then + cd "${matching_dirs[0]}" + else + echo "Error: Expected exactly one matching directory, found ${#matching_dirs[@]}." + exit 1 + fi + + # Run the azcopy version command + ./azcopy --version + ./azcopy --help + displayName: 'Install Zip, Extract Files, and Run Version and Help Command' + + - job: Set_9 + timeoutInMinutes: 120 + strategy: + matrix: + Windows: + imageName: 'windows-2019' + type: 'windows' + pool: + vmImage: $(imageName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-windows-signed/azcopy*amd64*.zip + + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*amd64*.zip" | Select-Object -First 1 + + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*amd64*.zip' in directory: $artifactDir" + exit 1 + } + + $zipFilePath = $zipFile.FullName + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } + + # Change to the directory containing azcopy + Set-Location -Path $extractDir + + # Find directories matching the pattern + $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_amd64* | Select-Object -First 1 + + # Check if there is exactly one matching directory + if ($matchingDirs.Count -eq 0) { + Write-Error "No directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } elseif ($matchingDirs.Count -gt 1) { + Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_amd64*" + exit 1 + } + + # Check if the azcopy executable exists in the matching directory + $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" + if (-Not (Test-Path -Path $azcopyPath)) { + Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + exit 1 + } else { + Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + # Ensure the azcopy executable has the correct permissions + icacls $azcopyPath + + # Run azcopy --version and capture output + try { + $versionOutput = & $azcopyPath --version 2>&1 + Write-Output "azcopy version output: $versionOutput" + } catch { + Write-Error "Failed to run azcopy --version: $_" + } + + # Run azcopy --help and capture output + try { + $helpOutput = & $azcopyPath --help 2>&1 + Write-Output "azcopy help output: $helpOutput" + } catch { + Write-Error "Failed to run azcopy --help: $_" + } + } + displayName: 'Extract Files and Run Version and Help Command on Windows' + + - job: Set_10 + timeoutInMinutes: 120 + strategy: + matrix: + Debian-12_amd64: + agentName: "blobfuse-debian-12" + pool: + name: "azcopy-debian-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: '$(System.DefaultWorkingDirectory)' + itemPattern: 'azCopy-linux-signed/azcopy-*x86_64.deb' + + - script: | + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Install deb Package' + workingDirectory: $(System.DefaultWorkingDirectory)/azCopy-linux-signed + + - script: | + azcopy --version + azcopy --help + displayName: 'Check Version and Help' + + - job: Set_11 + timeoutInMinutes: 120 + strategy: + matrix: + Windows-ARM64: + agentName: "windows_arm64" + pool: + name: "azcopy_windows_arm" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + itemPattern: azCopy-windows-signed/azcopy_windows_se_arm64*.zip + + - task: PowerShell@2 + inputs: + targetType: 'inline' + script: | + $artifactDir = "$(Build.ArtifactStagingDirectory)\azCopy-windows-signed" + $extractDir = "$(Build.ArtifactStagingDirectory)\extracted" + + # Find the zip file matching the pattern + $zipFile = Get-ChildItem -Path $artifactDir -Filter "azcopy*se_arm64*.zip" | Select-Object -First 1 + + if ($null -eq $zipFile) { + Write-Error "No zip file found matching pattern 'azcopy*se_arm64*.zip' in directory: $artifactDir" + exit 1 + } + + $zipFilePath = $zipFile.FullName + + # Create extraction directory + New-Item -ItemType Directory -Path $extractDir -Force | Out-Null + + # Extract the zip file + try { + Add-Type -AssemblyName System.IO.Compression.FileSystem + [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFilePath, $extractDir) + } catch { + Write-Error "Failed to extract zip file: $_" + exit 1 + } + + # Change to the directory containing azcopy + Set-Location -Path $extractDir + + # Find directories matching the pattern + $matchingDirs = Get-ChildItem -Directory -Path .\azcopy_windows_se_arm64* | Select-Object -First 1 + + # Check if there is exactly one matching directory + if ($matchingDirs.Count -eq 0) { + Write-Error "No directories found matching the pattern: .\azcopy_windows_se_arm64*" + exit 1 + } elseif ($matchingDirs.Count -gt 1) { + Write-Error "Multiple directories found matching the pattern: .\azcopy_windows_se_arm64*" + exit 1 + } + + # Check if the azcopy executable exists in the matching directory + $azcopyPath = Join-Path -Path $matchingDirs.FullName -ChildPath "azcopy.exe" + if (-Not (Test-Path -Path $azcopyPath)) { + Write-Error "azcopy not found in extraction directory: $($matchingDirs.FullName)" + exit 1 + } else { + Write-Output "azcopy found in extraction directory: $($matchingDirs.FullName)" + # Ensure the azcopy executable has the correct permissions + icacls $azcopyPath + + # Run azcopy --version and capture output + try { + $versionOutput = & $azcopyPath --version 2>&1 + Write-Output "azcopy version output: $versionOutput" + } catch { + Write-Error "Failed to run azcopy --version: $_" + } + + # Run azcopy --help and capture output + try { + $helpOutput = & $azcopyPath --help 2>&1 + Write-Output "azcopy help output: $helpOutput" + } catch { + Write-Error "Failed to run azcopy --help: $_" + } + } + displayName: 'Extract Files and Run Version and Help Command on Windows Arm64' + + # TestArtifacts ends here + + - stage: ReleaseToContainer + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: ReleaseToContainer + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + + variables: + - name: Test_Release_Account_Name + value: 'azcopyprivatedrops' + - name: Release_Account_Name + value: 'azcopyvnextrelease' + - name: Publish_To_Container + value: ${{ parameters.publish_to_container }} + - name: Publish_To_M1_Container + value: ${{ parameters.publish_to_m1_container }} + - name: work_dir + value: '$(Build.SourcesDirectory)' + + steps: + - task: DownloadBuildArtifacts@0 + displayName: 'Download Linux Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Windows Build Artifacts' + inputs: + artifactName: 'azCopy-windows-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Mac Build Artifacts' + inputs: + artifactName: 'azCopy-mac-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Binaries Build Artifacts' + inputs: + artifactName: 'azCopy-binaries' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + sudo apt-get install -y zip + cp $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/azcopy-*x86_64.deb . + rm -rf /usr/local/bin/azcopy + rm -rf /usr/bin/azcopy + sudo dpkg --info azcopy-*x86_64.deb + sudo dpkg -i azcopy-*x86_64.deb + sudo apt-get install build-essential -y + displayName: 'Create drop.zip file and install azcopy deb package' + + - script: | + cp $(work_dir)/NOTICE.txt $(Build.ArtifactStagingDirectory)/azCopy-binaries + cd $(Build.ArtifactStagingDirectory)/azCopy-binaries + echo "Contents of $(Build.ArtifactStagingDirectory)/azCopy-binaries:" + ls -l + + # Create the zip archive with all files inside azCopy-binaries + zip -r $(Build.ArtifactStagingDirectory)/drop.zip . + cd $(Build.ArtifactStagingDirectory) + + # List the contents of the resulting zip file for verification + unzip -l $(Build.ArtifactStagingDirectory)/drop.zip + + mv $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/mariner . + displayName: 'Archive all binaries to drop.zip' + workingDirectory: '$(Build.ArtifactStagingDirectory)' + + - task: Bash@3 + displayName: 'Install Azure CLI' + inputs: + targetType: 'inline' + script: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + - task: AzureCLI@2 + displayName: 'Azure CLI' + inputs: + azureSubscription: 'ESRP KeyVault identity' + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + # Find the current version + raw_version_output=$(azcopy --version) + azcopy_version=$(echo $raw_version_output | awk '{print $3}') + + today=$(date +"%Y%m%d") + + if [ $(Publish_To_Container) = "True" ]; then + container_url="https://$(Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + else + container_url="https://$(Test_Release_Account_Name).blob.core.windows.net/releases/release-$azcopy_version-$today" + fi + echo "Artifacts will be uploaded to: $container_url" + + executable_name=azcopy + + # Upload the archived builds + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-windows-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-linux-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-mac-signed/*" "$container_url" --recursive --put-md5=true + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/*.zip" "$container_url" --put-md5=true + + if [ $(Publish_To_M1_Container) = "True" ]; then + m1_container_url="https://azcopyvnextrelease.blob.core.windows.net/azcopy-m1-drops/azcopy-$azcopy_version-$today" + AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$m1_container_url" --put-md5=true + fi + + - ${{ if eq(parameters.publish_docker_image, true) }}: + - stage: BuildAndPublishDockerImage + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: Set_1_Ubuntu_Mariner_AMD64 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + + variables: + - group: AZCOPY_SECRET_VAULT + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - checkout: none + - script: | + git clone https://github.com/Azure/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout branch' + workingDirectory: $(work_dir) + + - script: | + chmod 777 *.sh + ./dockerinstall.sh + + ./buildcontainer.sh Dockerfile ubuntu-x86_64 + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ubuntu-x86_64 + + ./buildcontainer.sh DockerfileMariner mariner-x86_64 + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) mariner-x86_64 + + displayName: "Create docker image and push to the containers registry" + workingDirectory: $(work_dir)/docker + + - job: Set_2_Ubuntu_ARM64 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22-ARM64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_SECRET_VAULT + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - checkout: none + - script: | + git clone https://github.com/Azure/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout branch' + workingDirectory: $(work_dir) + + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + displayName: "Installing Go tools" + + - script: | + sudo apt update + sudo apt --fix-broken install + displayName: "Install dependencies" + + - script: | + chmod 777 *.sh + ./dockerinstall.sh + ./buildcontainer.sh DockerfileArm64 ubuntu-arm64 + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ubuntu-arm64 + displayName: "Create docker image for arm64 and push to the containers registry" + workingDirectory: $(work_dir)/docker + + - ${{ if eq(parameters.post_release, true) }}: + - stage: ReleaseToGithub + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: ReleaseToGithub + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22: + vmImage: 'Ubuntu-20.04' + AgentName: "blobfuse-ubuntu20" + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + steps: + - checkout: none + + - script: | + echo ${{ parameters.tag }} + displayName: 'Tag Name' + + - task: GithubRelease@1 + inputs: + githubConnection: 'azcopy-github-connection' + repositoryName: 'Azure/azure-storage-azcopy' + action: 'edit' + + target: '$(Build.SourceVersion)' + tagSource: 'userSpecifiedTag' + + title: ${{ parameters.tag }} + tag: ${{ parameters.tag }} + + changeLogCompareToRelease: 'lastFullRelease' + changeLogType: 'commitBased' + isDraft: ${{ parameters.draft }} + isPreRelease: ${{ parameters.prerelease }} + assetUploadMode: replace + + - ${{ if eq(parameters.publish_artifacts, true) }}: + - stage: PublishArtifacts + dependsOn: ReleaseToContainer + condition: succeeded('ReleaseToContainer') + jobs: + - job: PublishArtifacts + timeoutInMinutes: 120 + pool: + vmImage: 'ubuntu-22.04' + variables: + - group: AZCOPY_SECRET_VAULT + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - checkout: none + + - task: PipAuthenticate@1 + inputs: + artifactFeeds: 'DevExGlobalFeed' + displayName: 'Connect to PMC artifact' + + - script: | + pip install pmc-cli + displayName: 'Install pmc-cli' + + - task: DownloadSecureFile@1 + name: pmcCertificate + displayName: 'Download pmc pem file' + inputs: + secureFile: 'blobfusebuildvault-blobfuse-release-pmc1-20230911.pem' + + - task: DownloadSecureFile@1 + name: settings + displayName: 'Download settings.toml file' + inputs: + secureFile: 'settings.toml' + + - script: | + pmc --version + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --limit 1 + if [ $? -ne 0 ]; then + exit 1 + fi + displayName: 'Test PMC installation' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Signed Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + - script: | + cd mariner + arm64file=$(ls azcopy*.arm64.rpm) + amd64file=$(ls azcopy*.x86_64.rpm) + marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}" + marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}" + mv -v ./azcopy*.arm64.rpm "$marinerArmFileName" + mv -v ./azcopy*.x86_64.rpm "$marinerAmdFileName" + sudo ls -lRt + mv azcopy*.rpm .. + cd .. + rm -r mariner/ + displayName: 'Rename Mariner binaries' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - script: | + sudo ls -lRt $(Build.ArtifactStagingDirectory) + displayName: 'List Artifacts' + + - script: | + mv azCopy-linux-signed/*tar.gz . + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package upload azCopy-linux-signed + displayName: 'Upload files' + workingDirectory: $(Build.ArtifactStagingDirectory) + + - script: | + wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/packages.csv -O packages.csv + cat ./packages.csv + displayName: 'Fetch packages.csv' + workingDirectory: $(Build.ArtifactStagingDirectory)/ + + - script: | + azcopyAmdDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.x86_64.deb` + echo "Azcopy AMD DEB ID: $azcopyAmdDeb" + + azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm') + azcopyAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyAmdRpmFile) + echo "Azcopy AMD RPM ID: $azcopyAmdRpm" + + azcopyArmDeb=`pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload azcopy*.arm64.deb` + echo "Azcopy ARM DEB ID: $azcopyArmDeb" + + azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm') + azcopyArmRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $azcopyArmRpmFile) + echo "Azcopy ARM RPM $azcopyArmRpmFile" + echo "Azcopy ARM RPM ID: $azcopyArmRpm" + + # Find the Mariner RPM file and upload + marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm') + marinerAmdRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAmdRpmFile) + echo "Azcopy mariner AMD RPM $marinerAmdRpmFile" + echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm" + + marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm') + marinerAarchRpm=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) --id-only package upload $marinerAarchRpmFile) + echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile" + echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm" + + is_preview="false" + echo "##vso[task.setvariable variable=is_preview]$is_preview" + if [[ $marinerAmdRpmFile == *"preview"* ]]; then + is_preview="true" + echo "##vso[task.setvariable variable=is_preview]$is_preview" + fi + + while IFS=, read -r distro archetype repoName releaseName; do + + # If the package is preview, publish to mariner preview package + if [[ $distro == *"Mariner-"* ]]; then + if [ $is_preview = "true" ]; then + repoName=$(echo $repoName | sed 's/prod/preview/') + fi + fi + + echo "Uploading packages for $distro" + echo "Repo Name: $repoName" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --add-packages ${!archetype} $repoName $releaseName + done < <(tail -n +3 ../packages.csv) + displayName: 'Add uploaded packages to repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - script: | + while IFS=, read -r distro archetype repoName releaseName; do + if [[ $archetype == *"Arm"* ]]; then + echo "Skipping for ARM type on $distro" + else + if [[ $distro == *"Mariner-"* ]]; then + if [ "$(is_preview)" = "true" ]; then + repoName=$(echo $repoName | sed 's/prod/preview/') + fi + fi + echo "Repo Name: $repoName" + pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $repoName + fi + done < <(tail -n +3 ../packages.csv) + displayName: 'Publish the repository' + workingDirectory: $(Build.ArtifactStagingDirectory)/azCopy-linux-signed/ + + - ${{ if eq(parameters.update_version, true) }}: + - stage: UpdateLatestVersion + dependsOn: ReleaseToContainer + condition: succeeded('ReleaseToContainer') + jobs: + - job: UpdateVersion + pool: + vmImage: 'ubuntu-20.04' + variables: + - group: AZCOPY_TESTS_VAR + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + + steps: + - checkout: none + + - script: | + sudo apt-get install python3 -y + python3 --version + displayName: 'Installing Python' + + - task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: 'azCopy-linux-signed' + downloadPath: $(Build.ArtifactStagingDirectory) + + # install azcopy + - script: | + rm -rf /usr/local/bin/azcopy + rm -rf /usr/bin/azcopy + cd $(Build.ArtifactStagingDirectory)/azCopy-linux-signed + ls azcopy*.arm64.deb + sudo dpkg --info azcopy-*arm64.deb + sudo dpkg -i azcopy-*arm64.deb + azcopy --version + displayName: 'Installing azcopy' + + - script: | + wget https://raw.githubusercontent.com/Azure/azure-storage-azcopy/`echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement`/setup/releaseVersionUpdate.py + ls -l + displayName: 'Getting Python script' + workingDirectory: $(root_dir) + + - script: | + pip install azure-storage-blob + python3 releaseVersionUpdate.py "$(VERSION_CNT_SAS_URL)" "`azcopy --version`" + displayName: 'Updating version number' + workingDirectory: $(root_dir)