From eabbee797b54cba1a23c07a70e64b012566d067c Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 25 Jan 2024 10:52:10 +0100 Subject: [PATCH 1/3] ci: test-unit job matrix for win/macos/ubuntu Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- .github/workflows/build.yml | 94 ++++++++++++++++++++++++++++++++----- 1 file changed, 82 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c0f59227ff..ae1218d1812 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,9 +24,13 @@ env: REPO_SLUG: "docker/buildx-bin" DESTDIR: "./bin" TEST_CACHE_SCOPE: "test" + TESTFLAGS: "-v --parallel=6 --timeout=30m" + GOTESTSUM_FORMAT: "standard-verbose" + GO_VERSION: "1.21.6" + GOTESTSUM_VERSION: "v1.9.0" # same as one in Dockerfile jobs: - prepare-test: + prepare-test-integration: runs-on: ubuntu-22.04 steps: - @@ -51,14 +55,12 @@ jobs: *.cache-from=type=gha,scope=${{ env.TEST_CACHE_SCOPE }} *.cache-to=type=gha,scope=${{ env.TEST_CACHE_SCOPE }} - test: + test-integration: runs-on: ubuntu-22.04 needs: - - prepare-test + - prepare-test-integration env: - TESTFLAGS: "-v --parallel=6 --timeout=30m" TESTFLAGS_DOCKER: "-v --parallel=1 --timeout=30m" - GOTESTSUM_FORMAT: "standard-verbose" TEST_IMAGE_BUILD: "0" TEST_IMAGE_ID: "buildx-tests" strategy: @@ -71,9 +73,6 @@ jobs: - remote pkg: - ./tests - include: - - pkg: ./... - skip-integration-tests: 1 steps: - name: Checkout @@ -101,19 +100,19 @@ jobs: - name: Test run: | - export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]') + export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.worker }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]') ./hack/test env: TEST_DOCKERD: "${{ startsWith(matrix.worker, 'docker') && '1' || '0' }}" TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker\\+containerd') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$" TESTPKGS: "${{ matrix.pkg }}" - SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}" - name: Send to Codecov if: always() uses: codecov/codecov-action@v3 with: directory: ./bin/testreports + flags: integration - name: Generate annotations if: always() @@ -128,6 +127,75 @@ jobs: name: test-reports path: ./bin/testreports + test-unit: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-22.04 + - macos-12 + - windows-2022 + env: + SKIP_INTEGRATION_TESTS: 1 + steps: + - + name: Checkout + uses: actions/checkout@v4 + - + name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: "${{ env.GO_VERSION }}" + - + name: Prepare + run: | + testreportsBaseDir=./bin/testreports + testreportsDir=$testreportsBaseDir/unit-${{ matrix.os }} + echo "TESTREPORTS_BASEDIR=$testreportsBaseDir" >> $GITHUB_ENV + echo "TESTREPORTS_DIR=$testreportsDir" >> $GITHUB_ENV + mkdir -p $testreportsDir + shell: bash + - + name: Install gotestsum + run: | + go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VERSION }} + - + name: Test + env: + TMPDIR: ${{ runner.temp }} + run: | + gotestsum \ + --jsonfile="${{ env.TESTREPORTS_DIR }}/go-test-report.json" \ + --junitfile="${{ env.TESTREPORTS_DIR }}/junit-report.xml" \ + --packages="./..." \ + -- \ + "-mod=vendor" \ + "-coverprofile" "${{ env.TESTREPORTS_DIR }}/coverage.txt" \ + "-covermode" "atomic" ${{ env.TESTFLAGS }} + shell: bash + - + name: Send to Codecov + if: always() + uses: codecov/codecov-action@v3 + with: + directory: ${{ env.TESTREPORTS_DIR }} + env_vars: RUNNER_OS + flags: unit + - + name: Generate annotations + if: always() + uses: crazy-max/.github/.github/actions/gotest-annotations@1a64ea6d01db9a48aa61954cb20e265782c167d9 + with: + directory: ${{ env.TESTREPORTS_DIR }} + - + name: Upload test reports + if: always() + uses: actions/upload-artifact@v3 + with: + name: test-reports + path: ${{ env.TESTREPORTS_BASEDIR }} + prepare-binaries: runs-on: ubuntu-22.04 outputs: @@ -192,7 +260,8 @@ jobs: bin-image: runs-on: ubuntu-22.04 needs: - - test + - test-integration + - test-unit if: ${{ github.event_name != 'pull_request' && github.repository == 'docker/buildx' }} steps: - @@ -244,7 +313,8 @@ jobs: release: runs-on: ubuntu-22.04 needs: - - test + - test-integration + - test-unit - binaries steps: - From fb2c62a038d951e1e4737328872b298e6e14fb4c Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 25 Jan 2024 11:58:47 +0100 Subject: [PATCH 2/3] build: resolve 8.3 filename format to long one on Windows Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- build/git.go | 19 +++++++++++++++---- build/git_unix.go | 9 +++++++++ build/git_windows.go | 26 ++++++++++++++++++++++++++ go.mod | 2 +- util/gitutil/gitutil.go | 2 +- util/gitutil/{path_unix.go => path.go} | 2 +- util/gitutil/path_unix_test.go | 4 ++-- util/gitutil/path_windows.go | 2 +- util/gitutil/path_windows_test.go | 2 +- 9 files changed, 57 insertions(+), 11 deletions(-) create mode 100644 build/git_unix.go create mode 100644 build/git_windows.go rename util/gitutil/{path_unix.go => path.go} (97%) diff --git a/build/git.go b/build/git.go index 112ec363b31..8ed0d00e874 100644 --- a/build/git.go +++ b/build/git.go @@ -46,9 +46,9 @@ func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath st if filepath.IsAbs(contextPath) { wd = contextPath } else { - cwd, _ := os.Getwd() - wd, _ = filepath.Abs(filepath.Join(cwd, contextPath)) + wd, _ = filepath.Abs(filepath.Join(getWd(), contextPath)) } + wd = gitutil.SanitizePath(wd) gitc, err := gitutil.New(gitutil.WithContext(ctx), gitutil.WithWorkingDir(wd)) if err != nil { @@ -104,8 +104,7 @@ func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath st dockerfilePath = filepath.Join(wd, "Dockerfile") } if !filepath.IsAbs(dockerfilePath) { - cwd, _ := os.Getwd() - dockerfilePath = filepath.Join(cwd, dockerfilePath) + dockerfilePath = filepath.Join(getWd(), dockerfilePath) } if r, err := filepath.Rel(root, dockerfilePath); err == nil && !strings.HasPrefix(r, "..") { res["label:"+DockerfileLabel] = r @@ -125,9 +124,21 @@ func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath st if err != nil { continue } + if lp, err := getLongPathName(dir); err == nil { + dir = lp + } + dir = gitutil.SanitizePath(dir) if r, err := filepath.Rel(root, dir); err == nil && !strings.HasPrefix(r, "..") { so.FrontendAttrs["vcs:localdir:"+k] = r } } }, nil } + +func getWd() string { + wd, _ := os.Getwd() + if lp, err := getLongPathName(wd); err == nil { + return lp + } + return wd +} diff --git a/build/git_unix.go b/build/git_unix.go new file mode 100644 index 00000000000..5bd8e4d9f28 --- /dev/null +++ b/build/git_unix.go @@ -0,0 +1,9 @@ +//go:build !windows +// +build !windows + +package build + +// getLongPathName is a no-op on non-Windows platforms. +func getLongPathName(path string) (string, error) { + return path, nil +} diff --git a/build/git_windows.go b/build/git_windows.go new file mode 100644 index 00000000000..b3ec71540aa --- /dev/null +++ b/build/git_windows.go @@ -0,0 +1,26 @@ +package build + +import "golang.org/x/sys/windows" + +// getLongPathName converts Windows short pathnames to full pathnames. +// For example C:\Users\ADMIN~1 --> C:\Users\Administrator. +func getLongPathName(path string) (string, error) { + // See https://groups.google.com/forum/#!topic/golang-dev/1tufzkruoTg + p, err := windows.UTF16FromString(path) + if err != nil { + return "", err + } + b := p // GetLongPathName says we can reuse buffer + n, err := windows.GetLongPathName(&p[0], &b[0], uint32(len(b))) + if err != nil { + return "", err + } + if n > uint32(len(b)) { + b = make([]uint16, n) + _, err = windows.GetLongPathName(&p[0], &b[0], uint32(len(b))) + if err != nil { + return "", err + } + } + return windows.UTF16ToString(b), nil +} diff --git a/go.mod b/go.mod index 65f2f1ec777..0cb9c098583 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( go.opentelemetry.io/otel/trace v1.19.0 golang.org/x/mod v0.11.0 golang.org/x/sync v0.3.0 + golang.org/x/sys v0.15.0 golang.org/x/term v0.15.0 google.golang.org/grpc v1.58.3 gopkg.in/yaml.v3 v3.0.1 @@ -152,7 +153,6 @@ require ( golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/sys v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.10.0 // indirect diff --git a/util/gitutil/gitutil.go b/util/gitutil/gitutil.go index 3347415e2b5..a4b1b12bba5 100644 --- a/util/gitutil/gitutil.go +++ b/util/gitutil/gitutil.go @@ -70,7 +70,7 @@ func (c *Git) RootDir() (string, error) { if err != nil { return "", err } - return sanitizePath(root), nil + return SanitizePath(root), nil } func (c *Git) GitDir() (string, error) { diff --git a/util/gitutil/path_unix.go b/util/gitutil/path.go similarity index 97% rename from util/gitutil/path_unix.go rename to util/gitutil/path.go index 41b4bd61d42..9c6c693ef97 100644 --- a/util/gitutil/path_unix.go +++ b/util/gitutil/path.go @@ -45,7 +45,7 @@ func gitPath(wd string) (string, error) { var windowsPathRegex = regexp.MustCompile(`^[A-Za-z]:[\\/].*$`) -func sanitizePath(path string) string { +func SanitizePath(path string) string { // If we're running in WSL, we need to convert Windows paths to Unix paths. // This is because the git binary can be invoked through `git.exe` and // therefore returns Windows paths. diff --git a/util/gitutil/path_unix_test.go b/util/gitutil/path_unix_test.go index 591d5fcd9d5..ef95e6c55c8 100644 --- a/util/gitutil/path_unix_test.go +++ b/util/gitutil/path_unix_test.go @@ -10,10 +10,10 @@ import ( ) func TestSanitizePathUnix(t *testing.T) { - assert.Equal(t, "/home/foobar", sanitizePath("/home/foobar")) + assert.Equal(t, "/home/foobar", SanitizePath("/home/foobar")) } func TestSanitizePathWSL(t *testing.T) { t.Setenv("WSL_DISTRO_NAME", "Ubuntu") - assert.Equal(t, "/mnt/c/Users/foobar", sanitizePath("C:\\Users\\foobar")) + assert.Equal(t, "/mnt/c/Users/foobar", SanitizePath("C:\\Users\\foobar")) } diff --git a/util/gitutil/path_windows.go b/util/gitutil/path_windows.go index 050895f4506..6ec5ea12842 100644 --- a/util/gitutil/path_windows.go +++ b/util/gitutil/path_windows.go @@ -9,6 +9,6 @@ func gitPath(wd string) (string, error) { return exec.LookPath("git.exe") } -func sanitizePath(path string) string { +func SanitizePath(path string) string { return filepath.ToSlash(filepath.Clean(path)) } diff --git a/util/gitutil/path_windows_test.go b/util/gitutil/path_windows_test.go index 5326e305e0f..ee2e452de28 100644 --- a/util/gitutil/path_windows_test.go +++ b/util/gitutil/path_windows_test.go @@ -7,5 +7,5 @@ import ( ) func TestSanitizePathWindows(t *testing.T) { - assert.Equal(t, "C:\\Users\\foobar", sanitizePath("C:/Users/foobar")) + assert.Equal(t, "C:\\Users\\foobar", SanitizePath("C:/Users/foobar")) } From 703c765ec81155aafbd1c333869c0fd6d32a17b6 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Thu, 25 Jan 2024 14:46:14 +0100 Subject: [PATCH 3/3] gitutil: check git bash env when testing Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- util/gitutil/path_windows_test.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/util/gitutil/path_windows_test.go b/util/gitutil/path_windows_test.go index ee2e452de28..0ea32db3b48 100644 --- a/util/gitutil/path_windows_test.go +++ b/util/gitutil/path_windows_test.go @@ -1,11 +1,28 @@ package gitutil import ( + "os" "testing" "github.com/stretchr/testify/assert" ) func TestSanitizePathWindows(t *testing.T) { - assert.Equal(t, "C:\\Users\\foobar", SanitizePath("C:/Users/foobar")) + expected := "C:\\Users\\foobar" + if isGitBash() { + expected = "C:/Users/foobar" + } + assert.Equal(t, expected, SanitizePath("C:/Users/foobar")) +} + +func isGitBash() bool { + // The MSYSTEM environment variable is used in MSYS2 environments, + // including Git Bash, to select the active environment. This variable + // dictates the environment in which the shell operates, influencing + // factors like the path prefixes, default compilers, and system libraries + // used: https://www.msys2.org/docs/environments/ + if _, ok := os.LookupEnv("MSYSTEM"); ok { + return true + } + return false }