From 3a1c8bbdc928dc46553c6e7f6e20f9cf104c7864 Mon Sep 17 00:00:00 2001 From: dphulkar-msft <166800991+dphulkar-msft@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:04:23 +0530 Subject: [PATCH] Build and push docker image to containers registry (#2817) --- docker/Dockerfile | 16 +++++ docker/DockerfileArm64 | 16 +++++ docker/DockerfileMariner | 15 +++++ docker/README.md | 21 +++++++ docker/build.sh | 5 ++ docker/buildandruncontainer.sh | 29 ++++++++++ docker/buildcontainer.sh | 28 +++++++++ docker/dockerinstall.sh | 47 +++++++++++++++ docker/publishcontainer.sh | 11 ++++ release-pipeline.yml | 103 +++++++++++++++++++++++++++++++++ 10 files changed, 291 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/DockerfileArm64 create mode 100644 docker/DockerfileMariner create mode 100644 docker/README.md create mode 100644 docker/build.sh create mode 100644 docker/buildandruncontainer.sh create mode 100644 docker/buildcontainer.sh create mode 100644 docker/dockerinstall.sh create mode 100644 docker/publishcontainer.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..6e7e16c5c --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,16 @@ +# Create container based on Ubuntu-22.04 Jammy Jellyfish image +FROM mcr.microsoft.com/mirror/docker/library/ubuntu:22.04 + +# Copy azcopy binary to executable path +COPY ./azcopy /usr/local/bin/ + +# Make azcopy executable +RUN chmod +x /usr/local/bin/azcopy + +# Install dependencies +RUN \ + apt update && \ + apt-get install -y ca-certificates + +WORKDIR /azcopy +CMD [ "azcopy" ] \ No newline at end of file diff --git a/docker/DockerfileArm64 b/docker/DockerfileArm64 new file mode 100644 index 000000000..6ff26fab2 --- /dev/null +++ b/docker/DockerfileArm64 @@ -0,0 +1,16 @@ +# Create container based on Ubuntu-22.04 Jammy Jellyfish image +FROM --platform=linux/arm64 mcr.microsoft.com/mirror/docker/library/ubuntu:22.04 + +# Copy azcopy binary to executable path +COPY ./azcopy /usr/local/bin/ + +# Make azcopy executable +RUN chmod +x /usr/local/bin/azcopy + +# Install dependencies +RUN \ + apt update && \ + apt-get install -y ca-certificates + +WORKDIR /azcopy +CMD [ "azcopy" ] \ No newline at end of file diff --git a/docker/DockerfileMariner b/docker/DockerfileMariner new file mode 100644 index 000000000..33480e383 --- /dev/null +++ b/docker/DockerfileMariner @@ -0,0 +1,15 @@ +# Create container based on Ubuntu-22.04 Jammy Jellyfish image +FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 + +# Install dependencies +RUN tdnf update -y +RUN tdnf install -y ca-certificates + +# Copy azcopy binary to executable path +COPY ./azcopy /usr/local/bin/ + +# Make azcopy executable +RUN chmod +x /usr/local/bin/azcopy + +WORKDIR /azcopy +CMD [ "azcopy" ] \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..c1778321a --- /dev/null +++ b/docker/README.md @@ -0,0 +1,21 @@ +# Steps to Pull an Image from Azure Container Registry: + +# 1. Login to Azure + +az login + +# 2. Login to the Azure Container Registry + +az acr login --name azcopycontainers + +# 3. List the Images in Your ACR + +az acr repository list --name azcopycontainers --output table + +# 4. Pull the Image + +docker pull azcopycontainers.azurecr.io/: + +# 5. Run the Image + +docker run --rm -it -v /local/path/to/mount:/azcopy azcopycontainers.azurecr.io/: azcopy copy diff --git a/docker/build.sh b/docker/build.sh new file mode 100644 index 000000000..0b0d2e937 --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo "Using Go - $(go version)" +rm -rf azcopy +rm -rf azure-storage-azcopy +go build -o azcopy diff --git a/docker/buildandruncontainer.sh b/docker/buildandruncontainer.sh new file mode 100644 index 000000000..238031db9 --- /dev/null +++ b/docker/buildandruncontainer.sh @@ -0,0 +1,29 @@ +# Build azcopy binary +./dockerinstall.sh +./buildcontainer.sh Dockerfile ubuntu-x86_64 + +# Fetch the version of azcopy and extract the version number +azcopy_version=$(../azcopy --version | awk '{print $3}') + +# Construct the Docker image tag using the fetched version +docker_image_tag="azure-azcopy-ubuntu-x86_64.$azcopy_version" + +# If build was successful then launch a container instance +status=`docker images | grep $docker_image_tag` + +curr_dir=`pwd` +mkdir -p $curr_dir/azcopy +echo "Hello World" > $curr_dir/azcopy/hello.txt + +if [ $? = 0 ]; then + echo " **** Build successful, running container now ******" + + # Debug: Check the tag being used + echo "Using Docker image: $docker_image_tag" + + docker run -it --rm \ + -v $curr_dir/azcopy:/azcopy \ + $docker_image_tag azcopy --help +else + echo "Failed to build docker image" +fi diff --git a/docker/buildcontainer.sh b/docker/buildcontainer.sh new file mode 100644 index 000000000..a36c1a913 --- /dev/null +++ b/docker/buildcontainer.sh @@ -0,0 +1,28 @@ + +# Build azcopy binary +cd .. +echo "Building azcopy" +./docker/build.sh +ls -l azcopy + +ver=`./azcopy --version | cut -d " " -f 3` +tag="azure-azcopy-$2.$ver" + +# Cleanup older container image from docker +sudo docker image rm $tag -f + +# Build new container image using current code +echo "Build container for azcopy" +cd - +cp ../azcopy ./azcopy +sudo docker build -t $tag -f $1 . + +# List all images to verify if new image is created +sudo docker images + +# Image build is executed so we can clean up temp executable from here +rm -rf ./azcopy + +# If build was successful then launch a container instance +status=`sudo docker images | grep $tag` +echo $status diff --git a/docker/dockerinstall.sh b/docker/dockerinstall.sh new file mode 100644 index 000000000..46a3aad87 --- /dev/null +++ b/docker/dockerinstall.sh @@ -0,0 +1,47 @@ +# Cleanup old installation +sudo apt remove docker-desktop +rm -r $HOME/.docker/desktop +sudo rm /usr/local/bin/com.docker.cli +sudo apt purge docker-desktop +sudo apt-get update + +# Install certificates and pre-requisites +sudo apt-get install ca-certificates curl gnupg lsb-release -y +sudo mkdir -p /etc/apt/keyrings + +# Create keyring for docker +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg -y + +# Create file for installation +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"| sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +# Install docker +sudo apt-get update +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y +sudo apt-get update + +# Resolve permission issues to connect to docker socket +sudo groupadd docker +sudo usermod -aG docker $USER +sudo chown root:docker /var/run/docker.sock + +# Create the .docker directory if it doesn't exist +mkdir -p $HOME/.docker +sudo chown "$USER":"$USER" /home/"$USER"/.docker -R +sudo chmod g+rwx "$HOME/.docker" -R + +# Delete old azcopy image +docker rmi `docker images | grep azcopy | cut -d " " -f1` + +# Remove existing images +docker system prune -f + +# Start docker service +sudo service docker start + +# List docker container images +docker images ls + +# List docker instances running +docker container ls + diff --git a/docker/publishcontainer.sh b/docker/publishcontainer.sh new file mode 100644 index 000000000..a64149d7e --- /dev/null +++ b/docker/publishcontainer.sh @@ -0,0 +1,11 @@ +ver=`../azcopy --version | cut -d " " -f 3` +image="azure-azcopy-$3.$ver" + +sudo docker login azcopycontainers.azurecr.io --username $1 --password $2 + +# Publish Ubn-22 container image +sudo docker tag $image:latest azcopycontainers.azurecr.io/$image +sudo docker push azcopycontainers.azurecr.io/$image + +sudo docker logout azcopycontainers.azurecr.io + diff --git a/release-pipeline.yml b/release-pipeline.yml index 1bd0fafaf..7a30adaf2 100644 --- a/release-pipeline.yml +++ b/release-pipeline.yml @@ -42,6 +42,11 @@ parameters: type: boolean default: false + - name: publish_docker_image + displayName: 'Build and Push Docker Image' + type: boolean + default: false + # Do not trigger this pipeline automatically trigger: none pr: none @@ -1917,6 +1922,104 @@ stages: AZCOPY_AUTO_LOGIN_TYPE=AzCLI $executable_name cp "$(Build.ArtifactStagingDirectory)/azCopy-binaries/azcopy_darwin_m1_arm64" "$m1_container_url" --put-md5=true fi + - ${{ if eq(parameters.publish_docker_image, true) }}: + - stage: BuildAndPublishDockerImage + dependsOn: TestArtifacts + condition: succeeded('TestArtifacts') + jobs: + - job: Set_1_Ubuntu_Mariner_AMD64 + strategy: + matrix: + Ubuntu_amd64: + agentName: "blobfuse-ubuntu22" + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22' + pool: + vmImage: $(vmImage) + + variables: + - group: AZCOPY_SECRET_VAULT + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - checkout: none + - script: | + git clone https://github.com/Azure/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout branch' + workingDirectory: $(work_dir) + + - script: | + chmod 777 *.sh + ./dockerinstall.sh + + ./buildcontainer.sh Dockerfile ubuntu-x86_64 + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ubuntu-x86_64 + + ./buildcontainer.sh DockerfileMariner mariner-x86_64 + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) mariner-x86_64 + + displayName: "Create docker image and push to the containers registry" + workingDirectory: $(work_dir)/docker + + - job: Set_2_Ubuntu_ARM64 + timeoutInMinutes: 120 + strategy: + matrix: + Ubuntu-22-ARM64: + vmImage: 'Ubuntu-22.04' + container: 'test-cnt-ubn-22-arm64' + AgentName: "blobfuse-ubn22-arm64" + pool: + name: "blobfuse-ubn-arm64-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: AZCOPY_SECRET_VAULT + - name: root_dir + value: '$(System.DefaultWorkingDirectory)' + - name: work_dir + value: '$(System.DefaultWorkingDirectory)/azure-storage-azcopy' + + steps: + - checkout: none + - script: | + git clone https://github.com/Azure/azure-storage-azcopy + displayName: 'Checkout Code' + workingDirectory: $(root_dir) + + - script: | + git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` + displayName: 'Checkout branch' + workingDirectory: $(work_dir) + + - task: ShellScript@2 + inputs: + scriptPath: "$(work_dir)/go_installer.sh" + args: "$(root_dir)/ $(AZCOPY_GOLANG_VERSION)" + displayName: "Installing Go tools" + + - script: | + sudo apt update + sudo apt --fix-broken install + displayName: "Install dependencies" + + - script: | + chmod 777 *.sh + ./dockerinstall.sh + ./buildcontainer.sh DockerfileArm64 ubuntu-arm64 + ./publishcontainer.sh $(AZCOPY_DOCKER_REG_USER) $(AZCOPY_DOCKER_REG_PWD) ubuntu-arm64 + displayName: "Create docker image for arm64 and push to the containers registry" + workingDirectory: $(work_dir)/docker + - ${{ if eq(parameters.post_release, true) }}: - stage: ReleaseToGithub dependsOn: TestArtifacts