From b87795a846c4a1c41cb852005ad116cec218dfb8 Mon Sep 17 00:00:00 2001 From: Maarten Arnst Date: Sun, 28 Jul 2024 10:36:25 +0200 Subject: [PATCH] Add .devcontainer Add num functions Switch to building and storing image for tests Add nvidia jetson xavier agx and apple m2 --- .devcontainer/devcontainer.json | 12 + .github/workflows/build.image.yml | 56 ++++ .github/workflows/set-vars.yml | 24 ++ .github/workflows/test.yml | 21 +- docker/dockerfile | 18 ++ hwloc_xml_parser/topology.py | 58 +++- .../requirements.python.txt | 0 requirements/requirements.system.txt | 1 + tests/data/dual-intel-xeon-gold-6126.xml | 110 +++++++ tests/data/single-apple-m2.xml | 34 ++ ...4790.xml => single-intel-core-i7-4790.xml} | 0 .../data/single-nvidia-jetson-xavier-agx.xml | 44 +++ tests/test_topology.py | 294 ++++++++++++++++-- 13 files changed, 633 insertions(+), 39 deletions(-) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .github/workflows/build.image.yml create mode 100644 .github/workflows/set-vars.yml create mode 100644 docker/dockerfile rename requirements.txt => requirements/requirements.python.txt (100%) create mode 100644 requirements/requirements.system.txt create mode 100644 tests/data/dual-intel-xeon-gold-6126.xml create mode 100644 tests/data/single-apple-m2.xml rename tests/data/{intel-core-i7-4790.xml => single-intel-core-i7-4790.xml} (100%) create mode 100644 tests/data/single-nvidia-jetson-xavier-agx.xml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..3954c43 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,12 @@ +{ + "image": "ghcr.io/uliegecsm/hwloc-xml-parser", + "extensions" : [ + "eamodio.gitlens", + "mhutchie.git-graph", + "ms-azuretools.vscode-docker" + ], + "runArgs": [ + "--privileged", + ], + "onCreateCommand": "git config --global --add safe.directory $PWD" +} diff --git a/.github/workflows/build.image.yml b/.github/workflows/build.image.yml new file mode 100644 index 0000000..9653e1e --- /dev/null +++ b/.github/workflows/build.image.yml @@ -0,0 +1,56 @@ +name: Build image + +on: + push: + branches: + - main + paths: + - 'requirements/*' + - 'docker/dockerfile' + pull_request: + branches: + - main + paths: + - 'requirements/*' + - 'docker/dockerfile' + +jobs: + + set-vars: + uses: ./.github/workflows/set-vars.yml + + build-image: + needs: [set-vars] + runs-on: [ubuntu-latest] + container: + image: docker:latest + permissions: + packages: write + steps: + - name: Checkout code. + uses: actions/checkout@v4 + + - name: Set up QEMU. + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx. + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry. + uses: docker/login-action@v3 + with: + registry: ${{ needs.set-vars.outputs.CI_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push. + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.ref == 'refs/heads/main' }} + file: docker/dockerfile + tags: ${{ needs.set-vars.outputs.CI_IMAGE }} + cache-from: type=registry,ref=${{ needs.set-vars.outputs.CI_IMAGE }} + cache-to: type=inline + labels: "org.opencontainers.image.source=${{ github.repositoryUrl }}" diff --git a/.github/workflows/set-vars.yml b/.github/workflows/set-vars.yml new file mode 100644 index 0000000..bd930b3 --- /dev/null +++ b/.github/workflows/set-vars.yml @@ -0,0 +1,24 @@ +on: + workflow_call: + outputs: + CI_IMAGE: + value: ${{ jobs.set-vars.outputs.CI_IMAGE }} + CI_REGISTRY: + value: ${{ jobs.set-vars.outputs.CI_REGISTRY }} + +env: + REGISTRY: ghcr.io + +jobs: + + set-vars: + runs-on: [ubuntu-latest] + outputs: + CI_IMAGE : ${{ steps.common.outputs.CI_IMAGE }} + CI_REGISTRY: ${{ steps.common.outputs.CI_REGISTRY }} + steps: + - name: Export common variables. + id : common + run : | + echo "CI_IMAGE=${{ env.REGISTRY }}/${{ github.repository }}" >> $GITHUB_OUTPUT + echo "CI_REGISTRY=${{ env.REGISTRY }}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 412c552..8f2ccea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,33 +9,28 @@ on: - main jobs: + set-vars: + uses: ./.github/workflows/set-vars.yml + test: + needs: [set-vars] runs-on: [ubuntu-latest] container: - image: python:3.9 + image: ${{ needs.set-vars.outputs.CI_IMAGE }} steps: - name: Checkout code. uses: actions/checkout@v4 - - name: Prepare. - run : | - pip install -r requirements.txt - pip install pytest - - name: Run tests. run : | python -m pytest tests/test_topology.py - install-as-package: + install-as-package-and-test: + needs: [set-vars] runs-on: [ubuntu-latest] container: - image: python:3.9 + image: ${{ needs.set-vars.outputs.CI_IMAGE }} steps: - - name: Prepare. - run : | - apt update - apt --yes --no-install-recommends install hwloc - - name: Install as package. run : | pip install git+https://github.com/uliegecsm/hwloc-xml-parser.git@${{ github.sha }} diff --git a/docker/dockerfile b/docker/dockerfile new file mode 100644 index 0000000..64c91eb --- /dev/null +++ b/docker/dockerfile @@ -0,0 +1,18 @@ +FROM python:3.9 + +RUN --mount=type=bind,target=/requirements,type=bind,source=requirements < int: + """ + Returns the number of processing units. + """ + return len(self.pus) + class Package(Object): """ Package. Usually equivalent to a socket. @@ -66,6 +73,27 @@ def __init__(self, element : xml.etree.ElementTree.Element) -> None: self.hierarchical_index = f'Package:{self.os_index}' self.cores = [Core(x, parent = self) for x in element.findall(path = "object[@type='Core']")] + @typeguard.typechecked + def get_num_cores(self) -> int: + """ + Returns the number of cores. + """ + return len(self.cores) + + @typeguard.typechecked + def get_num_pus(self) -> int: + """ + Returns the number of processing units. + """ + return sum([core.get_num_pus() for core in self.cores]) + + @typeguard.typechecked + def all_equal_num_pus_per_core(self) -> bool: + """ + Returns `True` if all cores have the same number of processing units. + """ + return all([core.get_num_pus() == self.cores[0].get_num_pus() for core in self.cores]) + class SystemTopology: """ Read the system topology as reported by `hwloc`'s tool `lstopo`. @@ -121,7 +149,7 @@ def _parse(self, filename : typing.Union[pathlib.Path, str]) -> None: if not self.machine.tag == 'object': raise ValueError(f"Expected 'object' tag, got '{self.machine.tag}'") - + if not self.machine.attrib['type'] == 'Machine': raise ValueError(f"Expected 'Machine' type, got '{self.machine.attrib['type']}'") @@ -173,3 +201,31 @@ def recurse_pus(self) -> typing.Generator: for core in self.recurse_cores(): for pu in core.pus: yield pu + + @typeguard.typechecked + def get_num_packages(self) -> int: + """ + Returns the number of packages. + """ + return len(self.packages) + + @typeguard.typechecked + def get_num_cores(self) -> int: + """ + Returns the number of cores. + """ + return sum([package.get_num_cores() for package in self.packages]) + + @typeguard.typechecked + def get_num_pus(self) -> int: + """ + Returns the number of processing units. + """ + return sum([package.get_num_pus() for package in self.packages]) + + @typeguard.typechecked + def all_equal_num_pus_per_core(self) -> bool: + """ + Returns `True` if all cores have the same number of processing units. + """ + return all([package.all_equal_num_pus_per_core() for package in self.packages]) diff --git a/requirements.txt b/requirements/requirements.python.txt similarity index 100% rename from requirements.txt rename to requirements/requirements.python.txt diff --git a/requirements/requirements.system.txt b/requirements/requirements.system.txt new file mode 100644 index 0000000..c0398db --- /dev/null +++ b/requirements/requirements.system.txt @@ -0,0 +1 @@ +hwloc \ No newline at end of file diff --git a/tests/data/dual-intel-xeon-gold-6126.xml b/tests/data/dual-intel-xeon-gold-6126.xml new file mode 100644 index 0000000..6d31f8d --- /dev/null +++ b/tests/data/dual-intel-xeon-gold-6126.xml @@ -0,0 +1,110 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/data/single-apple-m2.xml b/tests/data/single-apple-m2.xml new file mode 100644 index 0000000..051df53 --- /dev/null +++ b/tests/data/single-apple-m2.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/data/intel-core-i7-4790.xml b/tests/data/single-intel-core-i7-4790.xml similarity index 100% rename from tests/data/intel-core-i7-4790.xml rename to tests/data/single-intel-core-i7-4790.xml diff --git a/tests/data/single-nvidia-jetson-xavier-agx.xml b/tests/data/single-nvidia-jetson-xavier-agx.xml new file mode 100644 index 0000000..253f61e --- /dev/null +++ b/tests/data/single-nvidia-jetson-xavier-agx.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test_topology.py b/tests/test_topology.py index 7737d0e..e656d93 100644 --- a/tests/test_topology.py +++ b/tests/test_topology.py @@ -9,26 +9,26 @@ class TestSystemTopology: Test :py:class:`system.SystemTopology`. """ - def test_parse(self): + def test_parse_single_intel_core_i7_4790(self): """ The test reads an `xml` file with the output of `lstopo-no-graphics` - for a system with the following topology: + for a single `Intel Core i7 4790` machine with the following topology: .. code-block:: python Package(os_index=0, logical_index=0) - Core(os_index=0, logical_index=0) - PU(os_index=0, logical_index=0) - PU(os_index=4, logical_index=1) - Core(os_index=1, logical_index=1) - PU(os_index=1, logical_index=2) - PU(os_index=5, logical_index=3) - Core(os_index=2, logical_index=2) - PU(os_index=2, logical_index=4) - PU(os_index=6, logical_index=5) - Core(os_index=3, logical_index=3) - PU(os_index=3, logical_index=6) - PU(os_index=7, logical_index=7) + Core(os_index=0, logical_index=0) + PU(os_index=0, logical_index=0) + PU(os_index=4, logical_index=1) + Core(os_index=1, logical_index=1) + PU(os_index=1, logical_index=2) + PU(os_index=5, logical_index=3) + Core(os_index=2, logical_index=2) + PU(os_index=2, logical_index=4) + PU(os_index=6, logical_index=5) + Core(os_index=3, logical_index=3) + PU(os_index=3, logical_index=6) + PU(os_index=7, logical_index=7) """ hwloc_calc_values = [ b'0', @@ -41,7 +41,7 @@ def test_parse(self): side_effect = hwloc_calc_values, ): st = SystemTopology(load = False) - st._parse(filename = 'tests/data/intel-core-i7-4790.xml') + st._parse(filename = 'tests/data/single-intel-core-i7-4790.xml') subprocess.check_output.assert_has_calls([ call(args=[ @@ -49,14 +49,14 @@ def test_parse(self): 'Package:0' ]), call(args=[ - 'hwloc-calc', '-I', 'Core', '--physical-input', '--logical-output', + 'hwloc-calc', '-I', 'Core', '--physical-input', '--logical-output', 'Package:0.Core:0', 'Package:0.Core:1', 'Package:0.Core:2', 'Package:0.Core:3' ]), call(args=[ - 'hwloc-calc', '-I', 'PU', '--physical-input', '--logical-output', + 'hwloc-calc', '-I', 'PU', '--physical-input', '--logical-output', 'Package:0.Core:0.PU:0', 'Package:0.Core:0.PU:4', 'Package:0.Core:1.PU:1', 'Package:0.Core:1.PU:5', 'Package:0.Core:2.PU:2', 'Package:0.Core:2.PU:6', @@ -64,19 +64,263 @@ def test_parse(self): ]) ]) - # There is 1 package. + # The machine has 1 package. assert len(st.packages) == 1 + assert st.get_num_packages() == 1 # The package has 4 cores. - assert len(st.packages[0].cores) == 4 + package = st.packages[0] - # Each core has 2 PUs. - assert len(st.packages[0].cores[0].pus) == 2 + assert len(package.cores) == 4 + assert package.get_num_cores() == 4 + + # The first core of the package has 2 PUs. + assert len(package.cores[0].pus) == 2 + assert package.cores[0].get_num_pus() == 2 # The first PU of the second core of the package has OS index 1 and logical index 2. - assert st.packages[0].cores[1].pus[0].os_index == 1 - assert st.packages[0].cores[1].pus[0].logical_index == 2 + assert package.cores[1].pus[0].os_index == 1 + assert package.cores[1].pus[0].logical_index == 2 + + # Repeat the last assertions, but using this time the generator returned by the + # method :py:meth:`hwloc_xml_parser.topology.SystemTopology.recurse_cores`. + gen_cores = st.recurse_cores() + _ = next(gen_cores) + second_core = next(gen_cores) + assert second_core.pus[0].os_index == 1 + assert second_core.pus[0].logical_index == 2 + + # Repeat the last assertions, but using this time the generator returned by the + # method :py:meth:`hwloc_xml_parser.topology.SystemTopology.recurse_pus`. + gen_pus = st.recurse_pus() + _ = next(gen_pus) + _ = next(gen_pus) + third_pu = next(gen_pus) + assert third_pu.os_index == 1 + assert third_pu.logical_index == 2 # The second PU of the second core of the package has OS index 5 and logical index 3. - assert st.packages[0].cores[1].pus[1].os_index == 5 - assert st.packages[0].cores[1].pus[1].logical_index == 3 + assert package.cores[1].pus[1].os_index == 5 + assert package.cores[1].pus[1].logical_index == 3 + + # All cores of the package have 2 PUs. + assert package.all_equal_num_pus_per_core() + + # The machine has 8 PUs in total. + assert st.get_num_cores() == 4 + + # The machine has 8 PUs in total. + assert st.get_num_pus() == 8 + assert package.get_num_pus() == 8 + + # All cores of the machine have the same number of PUs. + assert st.all_equal_num_pus_per_core() + + def test_parse_dual_intel_xeon_gold_6126(self): + """ + The test reads an `xml` file with the output of `lstopo-no-graphics` + for a dual `Intel Xeon Gold 6126` machine with the following topology: + + .. code-block:: python + + Package(os_index=0, logical_index=0) + Core(os_index=0, logical_index=0) + PU(os_index=0, logical_index=0) + PU(os_index=24, logical_index=1) + Core(os_index=1, logical_index=1) + PU(os_index=1, logical_index=2) + PU(os_index=25, logical_index=3) + ... + Core(os_index=6, logical_index=6) + PU(os_index=6, logical_index=12) + PU(os_index=30, logical_index=13) + Core(os_index=8, logical_index=7) + PU(os_index=7, logical_index=14) + PU(os_index=31, logical_index=15) + Core(os_index=10, logical_index=8) + PU(os_index=8, logical_index=16) + PU(os_index=32, logical_index=17) + ... + Core(os_index=13, logical_index=11) + PU(os_index=11, logical_index=22) + PU(os_index=35, logical_index=23) + Package(os_index=1, logical_index=1) + Core(os_index=1, logical_index=12) + PU(os_index=12, logical_index=24) + PU(os_index=36, logical_index=25) + ... + Core(os_index=6, logical_index=17) + PU(os_index=17, logical_index=34) + PU(os_index=41, logical_index=35) + Core(os_index=8, logical_index=18) + PU(os_index=18, logical_index=36) + PU(os_index=42, logical_index=37) + ... + Core(os_index=13, logical_index=23) + PU(os_index=23, logical_index=46) + PU(os_index=47, logical_index=47) + """ + hwloc_calc_values = [ + b'0,1', + ','.join([str(i) for i in range(24)]).encode(), + ','.join([str(i) for i in range(48)]).encode() + ] + + with unittest.mock.patch( + target = 'subprocess.check_output', + side_effect = hwloc_calc_values, + ): + st = SystemTopology(load = False) + st._parse(filename = 'tests/data/dual-intel-xeon-gold-6126.xml') + + subprocess.check_output.assert_has_calls([ + call(args=[ + 'hwloc-calc', '-I', 'Package', '--physical-input', '--logical-output', + 'Package:0', + 'Package:1' + ]), + call(args=[ + 'hwloc-calc', '-I', 'Core', '--physical-input', '--logical-output', + 'Package:0.Core:0', + 'Package:0.Core:1', + 'Package:0.Core:2', + 'Package:0.Core:3', + 'Package:0.Core:4', + 'Package:0.Core:5', + 'Package:0.Core:6', + 'Package:0.Core:8', + 'Package:0.Core:10', + 'Package:0.Core:11', + 'Package:0.Core:12', + 'Package:0.Core:13', + 'Package:1.Core:1', + 'Package:1.Core:2', + 'Package:1.Core:3', + 'Package:1.Core:4', + 'Package:1.Core:5', + 'Package:1.Core:6', + 'Package:1.Core:8', + 'Package:1.Core:9', + 'Package:1.Core:10', + 'Package:1.Core:11', + 'Package:1.Core:12', + 'Package:1.Core:13' + ]), + call(args=[ + 'hwloc-calc', '-I', 'PU', '--physical-input', '--logical-output', + 'Package:0.Core:0.PU:0', 'Package:0.Core:0.PU:24', + 'Package:0.Core:1.PU:1', 'Package:0.Core:1.PU:25', + 'Package:0.Core:2.PU:2', 'Package:0.Core:2.PU:26', + 'Package:0.Core:3.PU:3', 'Package:0.Core:3.PU:27', + 'Package:0.Core:4.PU:4', 'Package:0.Core:4.PU:28', + 'Package:0.Core:5.PU:5', 'Package:0.Core:5.PU:29', + 'Package:0.Core:6.PU:6', 'Package:0.Core:6.PU:30', + 'Package:0.Core:8.PU:7', 'Package:0.Core:8.PU:31', + 'Package:0.Core:10.PU:8', 'Package:0.Core:10.PU:32', + 'Package:0.Core:11.PU:9', 'Package:0.Core:11.PU:33', + 'Package:0.Core:12.PU:10', 'Package:0.Core:12.PU:34', + 'Package:0.Core:13.PU:11', 'Package:0.Core:13.PU:35', + 'Package:1.Core:1.PU:12', 'Package:1.Core:1.PU:36', + 'Package:1.Core:2.PU:13', 'Package:1.Core:2.PU:37', + 'Package:1.Core:3.PU:14', 'Package:1.Core:3.PU:38', + 'Package:1.Core:4.PU:15', 'Package:1.Core:4.PU:39', + 'Package:1.Core:5.PU:16', 'Package:1.Core:5.PU:40', + 'Package:1.Core:6.PU:17', 'Package:1.Core:6.PU:41', + 'Package:1.Core:8.PU:18', 'Package:1.Core:8.PU:42', + 'Package:1.Core:9.PU:19', 'Package:1.Core:9.PU:43', + 'Package:1.Core:10.PU:20', 'Package:1.Core:10.PU:44', + 'Package:1.Core:11.PU:21', 'Package:1.Core:11.PU:45', + 'Package:1.Core:12.PU:22', 'Package:1.Core:12.PU:46', + 'Package:1.Core:13.PU:23', 'Package:1.Core:13.PU:47' + ]) + ]) + + # There are 2 packages. + assert len(st.packages) == 2 + assert st.get_num_packages() == 2 + + # The first package has 12 cores. + first_package = st.packages[0] + + assert len(first_package.cores) == 12 + assert first_package.get_num_cores() == 12 + + # The first core of the first package has 2 PUs. + assert len(first_package.cores[0].pus) == 2 + assert first_package.cores[0].get_num_pus() == 2 + + # All cores of the first package have 2 PUs. + assert first_package.all_equal_num_pus_per_core() + + # The machine has 24 cores in total. + assert st.get_num_cores() == 24 + + # The machine has 48 PUs in total. + assert st.get_num_pus() == 48 + + # All cores of the machine have the same number of PUs. + assert st.all_equal_num_pus_per_core() + + def test_parse_single_nvidia_jetson_xavier_agx(self): + """ + The test reads an `xml` file with the output of `lstopo-no-graphics` + for a single `Nvidia Jetson Xavier AGX` machine. + """ + hwloc_calc_values = [ + b'0,1,2,3', + b'0,1,2,3,4,5,6,7', + b'0,1,2,3,4,5,6,7' + ] + + with unittest.mock.patch( + target = 'subprocess.check_output', + side_effect = hwloc_calc_values, + ): + st = SystemTopology(load = False) + st._parse(filename = 'tests/data/single-nvidia-jetson-xavier-agx.xml') + + # There are 4 packages. + # This is how `hwloc` reports it. Physically, there is a single package, with 4 clusters of 2 cores each. + # + # See also: + # - https://www.anandtech.com/show/13584/nvidia-xavier-agx-hands-on-carmel-and-more + assert st.get_num_packages() == 4 + + # The machine has 8 cores in total. + assert st.get_num_cores() == 8 + + # The machine has 8 PUs in total. + assert st.get_num_pus() == 8 + + # All cores of the machine have the same number of PUs. + assert st.all_equal_num_pus_per_core() + + def test_parse_single_apple_m2(self): + """ + The test reads an `xml` file with the output of `lstopo-no-graphics` + for a single `Apple M2` machine. + """ + hwloc_calc_values = [ + b'0', + b'0,1,2,3,4,5,6,7', + b'0,1,2,3,4,5,6,7' + ] + + with unittest.mock.patch( + target = 'subprocess.check_output', + side_effect = hwloc_calc_values, + ): + st = SystemTopology(load = False) + st._parse(filename = 'tests/data/single-apple-m2.xml') + + # The machine has 1 package. + assert st.get_num_packages() == 1 + + # The machine has 8 cores in total. + assert st.get_num_cores() == 8 + + # The machine has 8 PUs in total. + assert st.get_num_pus() == 8 + + # All cores of the machine have the same number of PUs. + assert st.all_equal_num_pus_per_core()