CI installation refactoring.

Signed-off-by: fruffy <[email protected]>

.github/workflows/ci-lint.yaml

@@ -35,12 +35,16 @@ jobs:
           tools/ci-build.sh
 
       - name: Run cpplint on C/C++ files.
-        run: cmake --build build --target cpplint
+        run: poetry run cmake --build build --target cpplint
 
       - name: Run clang-format on C/C++ files.
-        run: cmake --build build --target clang-format
+        run: |
+          poetry run clang-format --version
+          poetry run cmake --build build --target clang-format
 
       - name: Run black and isort on Python files.
         run: |
-          cmake --build build --target black
-          cmake --build build --target isort
+          poetry run black --version
+          poetry run cmake --build build --target black
+          poetry run isort --version
+          poetry run cmake --build build --target isort

.github/workflows/ci-p4tools.yml

@@ -40,5 +40,5 @@ jobs:
 
       - name: Run tests (Ubuntu 22.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest -R "testgen|smith" --output-on-failure --schedule-random
+        run: sudo -E env PATH="$PATH" poetry run ctest -R "testgen|smith" --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-ptf-kernels-weekly.yml

@@ -119,4 +119,4 @@ jobs:
 
       - name: Install VM and run PTF tests for eBPF backend
         run: |
-          sudo -E ./tools/ci-ptf/run_test.sh sudo docker run --privileged -v /sys/fs/bpf:/sys/fs/bpf -w /p4c/backends/ebpf/tests p4c ./test.sh
+          sudo -E ./tools/ci-ptf/run_test.sh sudo docker run --privileged -v /sys/fs/bpf:/sys/fs/bpf -w /p4c/backends/ebpf/tests p4c poetry run ./test.sh

.github/workflows/ci-ptf.yml

@@ -53,5 +53,5 @@ jobs:
           tools/ci-build.sh
 
       - name: Run PTF tests for eBPF backend (Ubuntu 20.04)
-        run: sudo -E ./test.sh
+        run: sudo -E env PATH="$PATH" poetry run ./test.sh
         working-directory: ./backends/ebpf/tests

.github/workflows/ci-static-build-test.yml

@@ -48,6 +48,6 @@ jobs:
       - shell: bash
         name: Build (Ubuntu 20.04)
         run: |
-          sudo -E tools/ci-build.sh
+          tools/ci-build.sh
           ./tools/ci-check-static.sh ./build/p4c-bm2-ss ./build/p4c-dpdk ./build/p4c-ebpf \
               ./build/p4c-pna-p4tc ./build/p4c-ubpf ./build/p4test ./build/p4testgen

.github/workflows/ci-test-debian.yml

@@ -14,6 +14,36 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
+
+  # Build with GCC and test P4C on Ubuntu 24.04.
+  test-ubuntu24:
+    runs-on: ubuntu-24.04
+    env:
+      CTEST_PARALLEL_LEVEL: 4
+      IMAGE_TYPE: test
+      BUILD_GENERATOR: Ninja
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1
+        with:
+          key: test-${{ runner.os }}-gcc
+          max-size: 1000M
+
+      - name: Build (Ubuntu 24.04, GCC)
+        run: |
+          tools/ci-build.sh
+
+      - name: Run tests (Ubuntu 24.04)
+        # Need to use sudo for the eBPF kernel tests.
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
+        working-directory: ./build
+
   # Build with GCC and test P4C on Ubuntu 22.04.
   test-ubuntu22:
     runs-on: ubuntu-22.04
@@ -39,7 +69,8 @@ jobs:
 
       - name: Run tests (Ubuntu 22.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build
 
   # Build with GCC and test Tofino backend on Ubuntu 22.04.
@@ -112,6 +143,7 @@ jobs:
 
       - name: Run tests (Ubuntu 20.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build
         if: matrix.unity == 'ON' && matrix.gtest == 'ON'

.github/workflows/ci-test-fedora.yml

@@ -35,7 +35,9 @@ jobs:
           submodules: recursive
 
       - name: Install dependencies (Fedora Linux)
-        run: tools/install_fedora_deps.sh
+        run: |
+          export PATH="$HOME/.local/bin:$PATH"
+          tools/install_fedora_deps.sh
 
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1
@@ -45,10 +47,12 @@ jobs:
 
       - name: Build p4c (Fedora Linux)
         run: |
+          export PATH="$HOME/.local/bin:$PATH"
           ./bootstrap.sh -DCMAKE_BUILD_TYPE=Release -DCMAKE_UNITY_BUILD=ON --build-generator "Ninja"
           cmake --build build -- -j $(nproc)
 
       - name: Run p4c tests (Fedora Linux)
         run: |
-          export PATH="$HOME/.local/bin:$PATH"; ctest --output-on-failure --schedule-random
+          export PATH="$HOME/.local/bin:$PATH"
+          poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-test-mac.yml

@@ -46,6 +46,7 @@ jobs:
 
       - name: Install dependencies (MacOS)
         run: |
+          source ~/.bash_profile
           tools/install_mac_deps.sh
 
       - name: Build (MacOS)
@@ -58,7 +59,7 @@ jobs:
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          poetry run ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
         working-directory: ./build
 
   # Build and test p4c on MacOS 13 on x86.
@@ -93,6 +94,7 @@ jobs:
 
       - name: Install dependencies (MacOS)
         run: |
+          source ~/.bash_profile
           tools/install_mac_deps.sh
 
       - name: Build (MacOS)
@@ -105,5 +107,5 @@ jobs:
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          poetry run ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
         working-directory: ./build

.github/workflows/ci-ubuntu-18-nightly.yml

@@ -39,4 +39,4 @@ jobs:
         # this is needed to create network namespaces for the ebpf tests.
       - name: Run tests (Ubuntu 18.04)
         run: |
-          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c ctest --output-on-failure --schedule-random
+          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c  poetry run --output-on-failure --schedule-random

.github/workflows/ci-ubuntu-20-sanitizer-nightly.yml

@@ -40,5 +40,5 @@ jobs:
 
       - name: Run tests (Ubuntu 20.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-validation-nightly.yml

@@ -38,5 +38,5 @@ jobs:
 
       - name: Validate
         run: |
-          ctest  -R toz3-validate-p4c --output-on-failure --schedule-random
+          poetry run ctest  -R toz3-validate-p4c --output-on-failure --schedule-random
         working-directory: ./build

.gitignore

@@ -52,3 +52,6 @@ docs/doxygen/awesome_css/
 
 # thirdparty
 backends/tofino/third_party/
+
+# Poetry
+poetry.lock

Dockerfile

@@ -46,6 +46,7 @@ ARG BUILD_AUTO_VAR_INIT_PATTERN=OFF
 # - Disable leaks detector as p4c uses GC.
 ENV UBSAN_OPTIONS=print_stacktrace=1
 ENV ASAN_OPTIONS=print_stacktrace=1:detect_leaks=0
+ENV PATH="/root/.local/bin:$PATH"
 
 # Delegate the build to tools/ci-build.
 COPY . /p4c/

backends/bmv2/run-bmv2-ptf-test.py

@@ -42,7 +42,7 @@
 PARSER.add_argument(
     "-n",
     "--num-ifaces",
-    default=8,
+    default=10,
     dest="num_ifaces",
     help="How many virtual interfaces to create.",
 )
@@ -91,7 +91,7 @@ class Options:
     # The base directory where tests are executed.
     rootdir: Path = Path(".")
     # The number of interfaces to create for this particular test.
-    num_ifaces: int = 8
+    num_ifaces: int = 10
     # Whether to use nanomsg for packet delivery as opposed to Linux veth interfaces.
     use_nn: bool = False
 
@@ -201,7 +201,7 @@ def run_ptf(self, grpc_port: int, json_name: Path, info_name: Path) -> int:
         )
         # TODO: There is currently a bug where we can not support more than 344 ports at once.
         # The nanomsg test back end simply hangs, the reason is unclear.
-        port_range = "0-8"
+        port_range = f"0-{self.options.num_ifaces - 1}"
         run_ptf_cmd = (
             f"ptf --platform nn --device-socket 0-{{{port_range}}}@ipc://{self.options.testdir}/"
             f"bmv2_packets_1.ipc --pypath {pypath} "
@@ -244,7 +244,7 @@ def run_simple_switch_grpc(self, switchlog: Path, grpc_port: int) -> Optional[su
         thrift_port = testutils.pick_tcp_port(GRPC_ADDRESS, THRIFT_PORT)
         simple_switch_grpc = (
             f"simple_switch_grpc --thrift-port {thrift_port} --device-id 0 --log-file {switchlog} "
-            f"{ifaces} --log-flush -i 0@0 --no-p4 "
+            f"{ifaces} --log-flush --no-p4 "
             f"-- --grpc-server-addr {GRPC_ADDRESS}:{grpc_port}"
         )
         bridge_cmd = self.bridge.get_ns_prefix() + " " + simple_switch_grpc
@@ -271,7 +271,7 @@ def run_ptf(self, grpc_port: int, json_name: Path, info_name: Path) -> int:
         ifaces = self.get_iface_str(num_ifaces=self.options.num_ifaces, prefix="br_")
         test_params = (
             f"grpcaddr='{GRPC_ADDRESS}:{grpc_port}';p4info='{info_name}';config='{json_name}';"
-            f"packet_wait_time='0.1';"
+            f"packet_wait_time='0.5';"
         )
         run_ptf_cmd = (
             f"ptf --pypath {pypath} {ifaces} --log-file {self.options.testdir.joinpath('ptf.log')} "

backends/ebpf/CMakeLists.txt

@@ -223,7 +223,7 @@ else()
 endif()
 
 # check for the libbpf library
-find_library(LIBBPF NAMES bpf HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/usr/lib64/")
+find_library(LIBBPF NAMES bpf HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/install/libbpf/")
 if (LIBBPF)
   message(STATUS "Found libbpf library at ${LIBBPF}")
 else()

pyproject.toml

@@ -51,3 +51,52 @@ disable = [
   "missing-function-docstring",
   "too-few-public-methods",
 ]
+
+[tool.poetry]
+name = "p4c"
+version = "1.2.5.1"
+description = ""
+authors = ["Your Name <[email protected]>"]
+license = "Apache"
+readme = "README.md"
+package-mode = false
+
+[tool.poetry.dependencies]
+# Fixme: This restriction is because of pyinstaller
+python = "<3.14,>=3.8"
+# General testing utilties.
+pyroute2 = "0.7.3"
+ply = "3.11"
+scapy = "2.5.0"
+# Utilities for BMv2 and PTF tests.
+ptf = "0.10.0"
+googleapis-common-protos = "1.53.0"
+grpcio = [
+  {version = "1.67.0", python = ">3.6"},
+  {version = "1.48.2", python = "<=3.6"}
+]
+thrift = "0.21.0 "
+protobuf = [
+  {version = "3.20.2", python = ">3.6"},
+  {version = "3.19.2", python = "<=3.6"}
+]
+# Linters and formatters.
+clang-format = "18.1.0"
+black = [
+  {version = "24.3.0", python = ">3.6"},
+  {version = "22.8.0", python = "<=3.6"}
+]
+isort = [
+  {version = "5.13.2", python = ">3.6"},
+  {version = "5.10.1", python = "<=3.6"}
+]
+# FIXME: We should figure out a way to synchronize P4Runtime versions across CMake and Poetry. This is the same commit hash as defined in the top-level CMakelists.txt
+p4runtime = { git = "https://github.com/p4lang/p4runtime.git", rev = "ec4eb5ef70dbcbcbf2f8357a4b2b8c2f218845a5", subdirectory = "py"}
+# FIXME: The Tofino back end should vendor these tools or use them locally.
+jsl= "0.2.4"
+pyinstaller= "6.11.0"
+jsonschema= "4.23.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

tools/ci-build.sh

@@ -72,9 +72,12 @@ function build_cmake_enabled_backend_string() {
 . /etc/lsb-release
 
 # In Docker builds, sudo is not available. So make it a noop.
-if [ "$IN_DOCKER" == "TRUE" ]; then
+if [ "$IN_DOCKER" = "TRUE" ]; then
   echo "Executing within docker container."
-  function sudo() { command "$@"; }
+  sudo() { "$@"; }  # No-op function; just execute the command
+else
+  # Preserve PATH and environment variables when using sudo
+  sudo() { command sudo -E env PATH="$PATH" "$@"; }
 fi
 
 
@@ -104,8 +107,10 @@ fi
 
 sudo apt-get update
 sudo apt-get install -y --no-install-recommends ${P4C_DEPS}
-sudo pip3 install --upgrade pip
-sudo pip3 install -r ${P4C_DIR}/requirements.txt
+# Set up poetry.
+sudo apt-get install -y python3-venv curl
+curl -sSL https://install.python-poetry.org | python3 -
+poetry install -C ${P4C_DIR}
 
 if [ "${BUILD_GENERATOR,,}" == "ninja" ] && [ ! $(command -v ninja) ]
 then
@@ -128,8 +133,6 @@ function build_bmv2() {
   # TODO: Remove this check once 18.04 is deprecated.
   if [[ "${DISTRIB_RELEASE}" == "18.04" ]] ; then
     P4C_RUNTIME_DEPS_BOOST="libboost-graph1.65.1 libboost-iostreams1.65.1"
-  else
-    P4C_RUNTIME_DEPS_BOOST="libboost-graph1.7* libboost-iostreams1.7*"
   fi
 
   P4C_RUNTIME_DEPS="cpp \
@@ -146,7 +149,7 @@ function build_bmv2() {
     P4C_RUNTIME_DEPS+=" gcc-9 g++-9"
     export CC=gcc-9
     export CXX=g++-9
-  else
+  elif [[ "${DISTRIB_RELEASE}" != "24.04" ]] ; then
    sudo apt-get install -y wget ca-certificates
     # Add the p4lang opensuse repository.
     echo "deb http://download.opensuse.org/repositories/home:/p4lang/xUbuntu_${DISTRIB_RELEASE}/ /" | sudo tee /etc/apt/sources.list.d/home:p4lang.list
@@ -155,11 +158,6 @@ function build_bmv2() {
   fi
 
   sudo apt-get update && sudo apt-get install -y --no-install-recommends ${P4C_RUNTIME_DEPS}
-
-  if [[ "${DISTRIB_RELEASE}" != "18.04" ]] ; then
-    # To run PTF nanomsg tests. Not available on 18.04.
-    sudo pip3 install nnpy
-  fi
 }
 
 if [[ "${ENABLE_BMV2}" == "ON" ]] ; then
@@ -292,12 +290,12 @@ fi
 if [ -e build ]; then /bin/rm -rf build; fi
 mkdir -p ${P4C_DIR}/build
 cd ${P4C_DIR}/build
-cmake ${CMAKE_FLAGS} -G "${BUILD_GENERATOR}" ..
+poetry run cmake ${CMAKE_FLAGS} -G "${BUILD_GENERATOR}" ..
 
 # If CMAKE_ONLY is active, only run CMake. Do not build.
 if [ "$CMAKE_ONLY" == "OFF" ]; then
-  cmake --build . -- -j $(nproc)
-  sudo cmake --install .
+  poetry run cmake --build . -- -j $(nproc)
+  sudo poetry run cmake --install .
   # Print ccache statistics after building
   ccache -p -s
 fi