From 730da7810cd7e07204188ea44d8ae034f7964134 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:53:17 +0200 Subject: [PATCH 1/3] Fix macOS detection when running standalone --- .github/workflows/pr-check.yml | 41 ++++++++++++++++++++++++++++++++++ alibuild_helpers/utilities.py | 13 ++++++----- tests/test_utilities.py | 13 +++++++++++ tox.ini | 24 +++++++++++--------- 4 files changed, 75 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index cfc15595..a6d7c582 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -63,6 +63,47 @@ jobs: with: files: coverage.json + mac-unittest: + name: mac-python${{ matrix.python-version }} + runs-on: macos-latest + + strategy: + matrix: + python-version: + - '3.11' + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install test dependencies + run: | + brew install modules alisw/system-deps/o2-full-deps + python3 -m pip install --upgrade tox tox-gh-actions coverage + + - name: Run tests + run: tox -e darwin + + - name: Convert coverage information + if: ${{ always() && github.event.repository.owner.login == 'alisw' }} + run: | + coverage combine .tox/coverage.* + # Codecov only understands XML, JSON and LCOV files. + # Apparently, patching os.readlink in unit tests interferes with + # finding some source files, but our source shouldn't be affected, so + # ignore these errors. + coverage json --ignore-errors -o coverage.json + + - name: Upload coverage information + if: ${{ always() && github.event.repository.owner.login == 'alisw' }} + uses: codecov/codecov-action@v3 + with: + files: coverage.json + lint: name: lint runs-on: ubuntu-latest diff --git a/alibuild_helpers/utilities.py b/alibuild_helpers/utilities.py index c7361355..9fec612c 100644 --- a/alibuild_helpers/utilities.py +++ b/alibuild_helpers/utilities.py @@ -133,11 +133,14 @@ def format(s, **kwds): def doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor): if platformSystem == "Darwin": - import platform - if platform.machine() == "x86_64": - return "osx_x86-64" - else: - return "osx_arm64" + processor = platformProcessor + if not processor: + import platform + if platform.machine() == "x86_64": + processor = "x86-64" + else: + processor = "arm64" + return "osx_%s" % processor.replace("_", "-") distribution, version, flavour = platformTuple distribution = distribution.lower() # If platform.dist does not return something sensible, diff --git a/tests/test_utilities.py b/tests/test_utilities.py index b8254053..9ff6ea44 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -115,6 +115,7 @@ architecturePayloads = [ ['osx_x86-64', False, [], ('','',''), 'Darwin', 'x86-64'], + ['osx_arm64', False, [], ('','',''), 'Darwin', 'arm64'], ['slc5_x86-64', False, [], ('redhat', '5.XX', 'Boron'), 'Linux', 'x86-64'], ['slc6_x86-64', False, [], ('centos', '6.X', 'Carbon'), 'Linux', 'x86-64'], ['slc7_x86-64', False, [], ('centos', '7.X', 'Ptor'), 'Linux', 'x86-64'], @@ -131,11 +132,23 @@ ['sabayon2_x86-64', True, SABAYON2_OS_RELEASE.split("\n"), ('gentoo', '2.2', ''), 'Linux', 'x86_64'] ] +macOSArchitecturePayloads = [ + ['osx_x86-64', False, [], ('','',''), 'Darwin', 'x86_64'], + ['osx_arm64', False, [], ('','',''), 'Darwin', 'arm64'], +] + class TestUtilities(unittest.TestCase): def test_osx(self): for payload in architecturePayloads: result, hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor = payload self.assertEqual(result, doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor)) + # Test by mocking platform.processor + def test_osx_mock(self): + for payload in macOSArchitecturePayloads: + result, hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor = payload + with patch('platform.machine', return_value=platformProcessor): + platformProcessor = None + self.assertEqual(result, doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, None)) def test_Hasher(self): h = Hasher() h("foo") diff --git a/tox.ini b/tox.ini index 06a35a42..ed710071 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 3.20 -envlist = lint, py{27, 36, 37, 38, 39, 310, 311} +envlist = lint, py{27, 36, 37, 38, 39, 310, 311}, darwin [gh-actions] # The "lint" job is run separately. @@ -57,6 +57,8 @@ setenv = # Keep coverage info for later upload, if needed. Files in {envtmpdir} are # deleted after each run. COVERAGE_FILE = {toxworkdir}/coverage.{envname} + ARCHITECTURE = slc7_x86-64 + darwin: ARCHITECTURE = osx_x86-64 changedir = {envtmpdir} commands = @@ -78,10 +80,10 @@ commands = git clone -b O2-v1.3.0 --depth 1 https://github.com/alisw/alidist - coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a slc7_x86-64 -z test-init init zlib + coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} -z test-init init zlib # This command is expected to fail, but run it for the coverage anyway. # A leading "-" means tox ignores the exit code. - - coverage run --source={toxinidir} -a {toxinidir}/aliBuild build non-existing -a slc7_x86-64 --no-system --disable GCC-Toolchain + - coverage run --source={toxinidir} -a {toxinidir}/aliBuild build non-existing -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain # TODO: do we need these? This seems to be at least partially covered by # unit tests in tests/tests_parseRecipe.py. @@ -93,19 +95,19 @@ commands = sh -c 'coverage run --source={toxinidir} -a {toxinidir}/aliBuild -c {toxinidir}/tests/testdist build broken6 --force-unknown-architecture --no-system --disable GCC-Toolchain 2>&1 | tee /dev/stderr | grep "while scanning a quoted scalar"' sh -c 'coverage run --source={toxinidir} -a {toxinidir}/aliBuild -c {toxinidir}/tests/testdist build broken7 --force-unknown-architecture --no-system --disable GCC-Toolchain 2>&1 | tee /dev/stderr | grep "Malformed entry prefer_system"' - coverage run --source={toxinidir} -a {toxinidir}/aliBuild build zlib -a slc7_x86-64 --no-system --disable GCC-Toolchain - alienv -a slc7_x86-64 q - alienv -a slc7_x86-64 setenv zlib/latest -c bash -c '[[ $LD_LIBRARY_PATH == */zlib/* ]]' - coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a slc7_x86-64 doctor AliPhysics - coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a slc7_x86-64 build zlib --dry-run - coverage run --source={toxinidir} -a {toxinidir}/aliBuild --aggressive-cleanup --docker -a slc7_x86-64 --always-prefer-system -d build zlib + coverage run --source={toxinidir} -a {toxinidir}/aliBuild build zlib -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain + alienv -a {env:ARCHITECTURE} q + alienv -a {env:ARCHITECTURE} setenv zlib/latest -c bash -c '[[ $LD_LIBRARY_PATH == */zlib/* ]]' + coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} doctor AliPhysics + coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} build zlib --dry-run + py311: coverage run --source={toxinidir} -a {toxinidir}/aliBuild --aggressive-cleanup --docker -a slc7_x86-64 --always-prefer-system -d build zlib # Test for devel packages coverage run --source={toxinidir} -a {toxinidir}/aliBuild init zlib - coverage run --source={toxinidir} -a {toxinidir}/aliBuild --aggressive-cleanup --docker -a slc7_x86-64 --always-prefer-system -d build zlib + py311: coverage run --source={toxinidir} -a {toxinidir}/aliBuild --aggressive-cleanup --docker -a slc7_x86-64 --always-prefer-system -d build zlib # Test that we complain if we have a devel package with an untracked file coverage run --source={toxinidir} -a {toxinidir}/aliBuild init zlib touch zlib/foo - coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a slc7_x86-64 --no-system --disable GCC-Toolchain build zlib + coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain build zlib [coverage:run] branch = True From 3d5e0a41efa3611b662953a61ad28b0e73cf552d Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:53:17 +0200 Subject: [PATCH 2/3] Always import platform Long gone are the days this was not available --- alibuild_helpers/utilities.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alibuild_helpers/utilities.py b/alibuild_helpers/utilities.py index 9fec612c..f0b4b123 100644 --- a/alibuild_helpers/utilities.py +++ b/alibuild_helpers/utilities.py @@ -7,6 +7,8 @@ import sys import os import re +import platform + from datetime import datetime try: from collections import OrderedDict @@ -135,7 +137,6 @@ def doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, pl if platformSystem == "Darwin": processor = platformProcessor if not processor: - import platform if platform.machine() == "x86_64": processor = "x86-64" else: @@ -194,7 +195,6 @@ def detectArch(): osReleaseLines = [] hasOsRelease = False try: - import platform if platform.system() == "Darwin": if platform.machine() == "x86_64": return "osx_x86-64" @@ -203,7 +203,7 @@ def detectArch(): except: pass try: - import platform, distro + import distro platformTuple = distro.linux_distribution() platformSystem = platform.system() platformProcessor = platform.processor() From 083e8292ca797397f8bb491b28ff349cce1c7925 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:55:00 +0200 Subject: [PATCH 3/3] Publish coverage information also from envsitedir This is needed because otherwise we do not publish the coverage information of the unit tests, but only those coming from the aliBuild commands. --- alibuild_helpers/utilities.py | 4 ++++ tests/test_coverage.py | 7 +++++++ tox.ini | 6 ++++-- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 tests/test_coverage.py diff --git a/alibuild_helpers/utilities.py b/alibuild_helpers/utilities.py index f0b4b123..33dfacad 100644 --- a/alibuild_helpers/utilities.py +++ b/alibuild_helpers/utilities.py @@ -29,6 +29,10 @@ class SpecError(Exception): asList = lambda x : x if type(x) == list else [x] +# This function is only needed to check the coverage of the testsuite +# is really happening and we did not made a mistake in tox.ini +def check_coverage(): + return True def resolve_store_path(architecture, spec_hash): """Return the path where a tarball with the given hash is to be stored. diff --git a/tests/test_coverage.py b/tests/test_coverage.py new file mode 100644 index 00000000..4dfcfb37 --- /dev/null +++ b/tests/test_coverage.py @@ -0,0 +1,7 @@ +import unittest +from alibuild_helpers.utilities import check_coverage + +class FooTest(unittest.TestCase): + def test_foo(self): + self.assertTrue(check_coverage()) + diff --git a/tox.ini b/tox.ini index ed710071..da3f14df 100644 --- a/tox.ini +++ b/tox.ini @@ -108,11 +108,13 @@ commands = coverage run --source={toxinidir} -a {toxinidir}/aliBuild init zlib touch zlib/foo coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain build zlib + coverage run --source={envsitepackagesdir} -a -m unittest discover {toxinidir}/tests [coverage:run] branch = True -omit = - */.tox/*/lib/* +include = + */alibuild*/* + */.tox/*/lib/*/alibuild_helpers/* [coverage:report] exclude_lines =