From 89ccc30e9f5adb90332a4cf1fe83eec05bb22f74 Mon Sep 17 00:00:00 2001 From: Ivan Nardi <12729895+IvanNardi@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:23:23 +0100 Subject: [PATCH] Update some CI jobs (#2660) * Move ThreadSanitizer job to the scheduled jobs (once a day): all our tests are intrinsically mono-thread and this job takes quite some time * Two explicit jobs to test LTO and Gold linker, used by oss-fuzz * Two explicit jobs for Windows (with msys2) * Run address sanitizer only on the 4 main jobs: newest/oldest gcc/clang * Reduce the time used by fuzzing jobs. Note that oss-fuzz is continuosly fuzzing our code! * Move the no x86_64 jobs to a dedicated file This way, the main matrix is a little bit simpler and the CI jobs last a little shorter --- .github/workflows/build.yml | 287 +++++++++----------------- .github/workflows/build_no_x86_64.yml | 102 +++++++++ .github/workflows/build_scheduled.yml | 25 +++ .github/workflows/cifuzz.yml | 2 +- 4 files changed, 226 insertions(+), 190 deletions(-) create mode 100644 .github/workflows/build_no_x86_64.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2e3fd6f08f2..321deb6b065 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,7 @@ jobs: run: | cd python python tests.py + test-scripts: name: Test Utils (ubuntu-latest) runs-on: ubuntu-latest @@ -67,7 +68,7 @@ jobs: git diff-index --quiet HEAD -- || true test: - name: ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.gcrypt }} ${{ matrix.compiler }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} ${{ matrix.msan }} ${{ matrix.nBPF }} ${{matrix.lto_gold_linker}} ${{matrix.global_context}} + name: ${{ matrix.os }} ${{ matrix.gcrypt }} ${{ matrix.compiler }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} ${{ matrix.msan }} ${{ matrix.nBPF }} ${{matrix.global_context}} runs-on: ${{ matrix.os }} env: CC: ${{ matrix.compiler }} @@ -77,22 +78,17 @@ jobs: matrix: # macOS-latest == macos-14 on **ARM64**. Also macos-15 is on arm64 # There are some issues with external dependencies on macOS-14/15. Disable it for the time being - os: ["ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04", "macOS-13", "windows-latest"] - arch: ["x86_64"] + os: ["ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04", "macOS-13"] gcrypt: ["--with-local-libgcrypt", ""] compiler: ["cc"] - ar: ["ar"] - ranlib: ["ranlib"] pcre: [""] maxminddb: [""] msan: [""] nBPF: [""] - lto_gold_linker: [""] - global_context: [""] #Enable by default + global_context: [""] # Enable by default include: - compiler: "gcc-4.9" # "Oldest" gcc easily available. To simulate RHEL7 os: ubuntu-20.04 - arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" @@ -100,60 +96,42 @@ jobs: nBPF: "" - compiler: "gcc-14" # "Newest" gcc easily available os: ubuntu-24.04 - arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" msan: "--with-sanitizer" nBPF: "" - lto_gold_linker: "--with-lto-and-gold-linker" - compiler: "clang-9" # "Oldest" clang easily available os: ubuntu-20.04 - arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" msan: "--with-sanitizer" nBPF: "" - compiler: "clang-18" # "Newest" clang easily available. See also below... - ar: "llvm-ar-18" - ranlib: "llvm-ranlib-18" os: ubuntu-24.04 - arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" msan: "--with-sanitizer" nBPF: "" - lto_gold_linker: "--with-lto-and-gold-linker" - - compiler: "cc" - os: ubuntu-latest - arch: "x86_64" - gcrypt: "" - pcre: "--with-pcre2" - maxminddb: "--with-maxminddb" - msan: "--with-thread-sanitizer" - nBPF: "" - compiler: "cc" os: ubuntu-latest - arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" - msan: "--with-sanitizer" + msan: "" nBPF: "nBPF" - compiler: "cc" os: ubuntu-latest - arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" - msan: "--with-sanitizer" + msan: "" nBPF: "" global_context: "--disable-global-context-support" - - compiler: "clang" #TODO: some issues with masan/clang/ubuntu-24.04 + - compiler: "clang" # TODO: some issues with masan/clang/ubuntu-24.04 os: ubuntu-22.04 - arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" @@ -161,7 +139,6 @@ jobs: nBPF: "" - compiler: "cc" os: macOS-13 - arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" @@ -169,7 +146,6 @@ jobs: nBPF: "" - compiler: "cc" os: macos-14 - arch: "x86_64" gcrypt: "" pcre: "" maxminddb: "" @@ -177,68 +153,39 @@ jobs: nBPF: "" - compiler: "cc" os: macos-15 - arch: "x86_64" gcrypt: "" pcre: "" maxminddb: "" msan: "" # Disable sanitizer on macos nBPF: "" - - compiler: "cc" - os: ubuntu-latest - arch: "arm64" - gcrypt: "" - pcre: "--with-pcre2" - maxminddb: "--with-maxminddb" - msan: "" # Disable sanitizer on arm64 - nBPF: "" - - compiler: "cc" - os: ubuntu-latest - arch: "armhf" - gcrypt: "" - pcre: "--with-pcre2" - maxminddb: "--with-maxminddb" - msan: "" - nBPF: "" - - compiler: "cc" - os: ubuntu-latest - arch: "s390x" - gcrypt: "" - pcre: "--with-pcre2" - maxminddb: "--with-maxminddb" - msan: "" - nBPF: "" steps: - - name: Setup multiarch/qemu-user-static - if: startsWith(matrix.os, 'ubuntu') && !startsWith(matrix.arch, 'x86_64') - run: | - docker run --rm --privileged multiarch/qemu-user-static:register --reset - uses: actions/checkout@v4 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: Install Ubuntu Prerequisites - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') + if: startsWith(matrix.os, 'ubuntu') run: | sudo apt-get update sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev sudo apt-get install rrdtool librrd-dev parallel - name: Install Ubuntu Prerequisites [Mingw-w64] (runs only on ubuntu jobs) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers + if: startsWith(matrix.os, 'ubuntu') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') && !startsWith(matrix.global_context, '--without') # Only on a few "standard" builds run: | sudo apt-get install gcc-mingw-w64 libc6-dev - name: Install Ubuntu Prerequisites (libgcrypt) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt') + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.gcrypt, '--with-local-libgcrypt') run: | sudo apt-get install libgcrypt20-dev - name: Install Ubuntu Prerequisites (libpcre2) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2') + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.pcre, '--with-pcre2') run: | sudo apt-get install libpcre3-dev - name: Install Ubuntu Prerequisites (maxminddb) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.maxminddb, '--with-maxminddb') + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.maxminddb, '--with-maxminddb') run: | sudo apt-get install libmaxminddb-dev - name: Install Ubuntu Prerequisites (nBPF) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.nBPF, 'nBPF') + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.nBPF, 'nBPF') run: | git clone https://github.com/ntop/PF_RING.git ../PF_RING cd ../PF_RING/userland/nbpf @@ -246,57 +193,36 @@ jobs: make cd - - name: Setup Ubuntu specified compiler - if: startsWith(matrix.os, 'ubuntu-20.04') && startsWith(matrix.arch, 'x86_64') && ! startsWith(matrix.compiler, 'cc') + if: startsWith(matrix.os, 'ubuntu-20.04') && ! startsWith(matrix.compiler, 'cc') run: | - #For gcc-4.9 (on ubuntu-20.04) + # For gcc-4.9 (on ubuntu-20.04) echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial main" | sudo tee -a /etc/apt/sources.list echo "deb http://dk.archive.ubuntu.com/ubuntu/ xenial universe" | sudo tee -a /etc/apt/sources.list sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5 sudo apt-get update sudo apt-get install ${{ matrix.compiler }} - - name: Install Windows msys2 prerequisites - if: startsWith(matrix.os, 'windows') - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW64 - update: true - install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap mingw-w64-x86_64-libgcrypt parallel - name: Installing MacOS prerequisites - if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') + if: startsWith(matrix.os, 'macOS') run: | # Avoid (re)installing pkg-config. See: https://github.com/actions/runner-images/issues/10984 brew install coreutils wdiff colordiff autoconf automake libtool gettext json-c rrdtool parallel - name: Install MacOS Prerequisites (libgcrypt) - if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt') + if: startsWith(matrix.os, 'macOS') && startsWith(matrix.gcrypt, '--with-local-libgcrypt') run: | brew install libgcrypt - name: Install MacOS Prerequisites (libpcre2) - if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2') + if: startsWith(matrix.os, 'macOS') && startsWith(matrix.pcre, '--with-pcre2') run: | brew install pcre - name: Install MacOS Prerequisites (maxminddb) - if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.maxminddb, '--with-maxminddb') + if: startsWith(matrix.os, 'macOS') && startsWith(matrix.maxminddb, '--with-maxminddb') run: | brew install libmaxminddb - - name: Configure nDPI on Ubuntu - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') - run: | - AR=${{ matrix.ar }} RANLIB=${{ matrix.ranlib }} ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs ${{matrix.lto_gold_linker}} ${{matrix.global_context}} - - name: Configure nDPI on MacOS - if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc') + - name: Configure nDPI run: | - ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs - - name: Configure nDPI on Windows msys2 - if: startsWith(matrix.os, 'windows') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc') - run: | - msys2 -c './autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap ${{ matrix.gcrypt }}' - - name: Build nDPI on Windows msys2 - if: startsWith(matrix.os, 'windows') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc') - run: | - msys2 -c 'make -j all' - msys2 -c 'ldd ./example/ndpiReader.exe' + ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs ${{ matrix.global_context}} - name: Build nDPI - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') + if: ${{ !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') }} run: | make -j all make -C example ndpiSimpleIntegration @@ -306,137 +232,120 @@ jobs: run: | make -j all make -C example ndpiSimpleIntegration - #There are somes issues with librrd - #make -C rrdtool + # There are somes issues with librrd + # make -C rrdtool - name: Print nDPI long help - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') run: | cd ./example && ./ndpiReader -H - name: Install nDPI - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') run: | DESTDIR=/tmp/ndpi make install ls -alhHR /tmp/ndpi - name: Test nDPI [SYMBOLS] - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers + if: startsWith(matrix.os, 'ubuntu') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') && !startsWith(matrix.global_context, '--without') # Only on a few "standard" builds run: | ./utils/check_symbols.sh || { FAILED=$?; echo "::error file=${NDPI_LIB}::Unwanted libc symbols found: ${FAILED}. Please make sure to use only ndpi_malloc/ndpi_calloc/ndpi_realloc/ndpi_free wrapper instead of malloc/calloc/realloc/free."; false; } env: NDPI_LIB: src/lib/libndpi.a - name: Test nDPI [DIFF] - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') run: | NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh - name: Test nDPI [UNIT] - #Some issues with masan + json-c. Disable the test as workaround - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') + # Some issues with masan + json-c. Disable the test as workaround + if: ${{ !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') }} run: | ./tests/do-unit.sh - name: Test nDPI [DGA] - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') run: | ./tests/do-dga.sh - - name: Test nDPI [DIFF] (runs only on windows jobs) - if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows') - run: | - msys2 -c 'NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh' - - name: Test nDPI [UNIT] (runs only on windows jobs) - if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows') - run: | - msys2 -c './tests/do-unit.sh' - - name: Test nDPI [DGA] (runs only on windows jobs) - if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows') - run: | - msys2 -c './tests/do-dga.sh' - name: Generate/Verify tarball - if: startsWith(matrix.os, 'ubuntu-latest') && startsWith(matrix.arch, 'x86_64') + if: startsWith(matrix.os, 'ubuntu-latest') run: | make dist ./utils/verify_dist_tarball.sh - name: Build Debian/Ubuntu package - if: startsWith(matrix.os, 'ubuntu-24.04') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc') + if: startsWith(matrix.os, 'ubuntu-24.04') && startsWith(matrix.compiler, 'cc') run: | cd packages/ubuntu ./configure --enable-no-sign make cd ../.. - name: Build nDPI [Mingw-w64] (runs only on ubuntu jobs) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers + if: startsWith(matrix.os, 'ubuntu') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') && !startsWith(matrix.global_context, '--without') # Only on a few "standard" builds run: | make distclean ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --host=x86_64-w64-mingw32 make -j $(nproc) all env: CC: - - name: Display qemu specified architecture (arm64 - little endian) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'arm64') - uses: docker://multiarch/ubuntu-core:arm64-bionic - with: - args: > - bash -c - "uname -a && - lscpu | grep Endian - " - - name: Configure, compile and test using qemu for the specified architecture (arm64 - little endian) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'arm64') - uses: docker://multiarch/ubuntu-core:arm64-bionic - with: - args: > - bash -c - "apt-get -y update && - apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev && - git config --global --add safe.directory $(realpath .) && - env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs && - make -j all && - make -C example ndpiSimpleIntegration && - make -C rrdtool && - make check VERBOSE=1 - " - - name: Display qemu specified architecture (armhf - little endian) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'armhf') - uses: docker://multiarch/ubuntu-core:armhf-bionic - with: - args: > - bash -c - "uname -a && - lscpu | grep Endian - " - - name: Configure, compile and test using qemu for the specified architecture (armhf - little endian) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'armhf') - uses: docker://multiarch/ubuntu-core:armhf-bionic + + test-lto-gold: + # Options used by oss-fuzz: we only want to check that everything compile fine; no need to run the tests + name: LTO and Gold Linker ${{ matrix.compiler }} + runs-on: ubuntu-24.04 + env: + CC: ${{ matrix.compiler }} + CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS + strategy: + fail-fast: true + matrix: + compiler: ["gcc-14", "clang-18"] # "Newest" gcc/clang easily available + include: + - compiler: "gcc-14" + ar: "ar" + ranlib: "ranlib" + - compiler: "clang-18" + ar: "llvm-ar-18" + ranlib: "llvm-ranlib-18" + steps: + - uses: actions/checkout@v4 with: - args: > - bash -c - "apt-get -y update && - apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev && - git config --global --add safe.directory $(realpath .) && - env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs && - make -j all && - make -C example ndpiSimpleIntegration && - make -C rrdtool && - make check VERBOSE=1 - " - - name: Display qemu specified architecture (s390x - big endian) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 's390x') - uses: docker://multiarch/ubuntu-core:s390x-bionic + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Install Ubuntu Prerequisites + run: | + sudo apt-get update + sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev parallel + - name: Configure nDPI + run: | + AR=${{ matrix.ar }} RANLIB=${{ matrix.ranlib }} ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --with-sanitizer --with-lto-and-gold-linker + - name: Build nDPI + run: | + make -j $(nproc) all + make -j $(nproc) -C example ndpiSimpleIntegration + make -j $(nproc) -C rrdtool + - name: Print nDPI long help + run: | + cd ./example && ./ndpiReader -H + + test-windows: + name: ${{ matrix.os }} (msys2) ${{ matrix.gcrypt }} + runs-on: ${{ matrix.os }} + env: + CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS + strategy: + fail-fast: true + matrix: + os: ["windows-latest"] + gcrypt: ["--with-local-libgcrypt", ""] + steps: + - uses: actions/checkout@v4 with: - args: > - bash -c - "uname -a && - lscpu | grep Endian - " - - name: Configure and compile using qemu for the specified architecture (s390x - big endian) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 's390x') - uses: docker://multiarch/ubuntu-core:s390x-bionic + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Install Windows msys2 prerequisites + uses: msys2/setup-msys2@v2 with: - args: > - bash -c - "apt-get -y update && - apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev libgcrypt20-dev libpcre3-dev libmaxminddb-dev rrdtool librrd-dev && - git config --global --add safe.directory $(realpath .) && - env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs && - make -j all && - make -C example ndpiSimpleIntegration && - make -C rrdtool && - make check VERBOSE=1 - " + msystem: MINGW64 + update: true + install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap mingw-w64-x86_64-libgcrypt parallel + - name: Configure nDPI on Windows msys2 + run: | + msys2 -c './autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap ${{ matrix.gcrypt }}' + - name: Build nDPI on Windows msys2 + run: | + msys2 -c 'make -j all' + msys2 -c 'ldd ./example/ndpiReader.exe' + - name: Tests + run: | + msys2 -c 'NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh' + msys2 -c './tests/do-unit.sh' + msys2 -c './tests/do-dga.sh' diff --git a/.github/workflows/build_no_x86_64.yml b/.github/workflows/build_no_x86_64.yml new file mode 100644 index 00000000000..af28c95a6eb --- /dev/null +++ b/.github/workflows/build_no_x86_64.yml @@ -0,0 +1,102 @@ +name: Build No x86_64 archs +on: + push: + branches: + - dev + pull_request: + branches: + - dev + types: [opened, synchronize, reopened] + release: + types: [created] +jobs: + + test: + name: Ubuntu ${{ matrix.arch }} + runs-on: ubuntu-latest + env: + CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS + strategy: + fail-fast: true + matrix: + arch: ["arm64", "armhf", "s390x"] + steps: + - name: Setup multiarch/qemu-user-static + run: | + docker run --rm --privileged multiarch/qemu-user-static:register --reset + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Display qemu specified architecture (arm64 - little endian) + if: startsWith(matrix.arch, 'arm64') + uses: docker://multiarch/ubuntu-core:arm64-bionic + with: + args: > + bash -c + "uname -a && + lscpu | grep Endian + " + - name: Configure, compile and test using qemu for the specified architecture (arm64 - little endian) + if: startsWith(matrix.arch, 'arm64') + uses: docker://multiarch/ubuntu-core:arm64-bionic + with: + args: > + bash -c + "apt-get -y update && + apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev && + git config --global --add safe.directory $(realpath .) && + env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages && + make -j $(nproc) all && + make -C example ndpiSimpleIntegration && + make -C rrdtool && + make check VERBOSE=1 + " + - name: Display qemu specified architecture (armhf - little endian) + if: startsWith(matrix.arch, 'armhf') + uses: docker://multiarch/ubuntu-core:armhf-bionic + with: + args: > + bash -c + "uname -a && + lscpu | grep Endian + " + - name: Configure, compile and test using qemu for the specified architecture (armhf - little endian) + if: startsWith(matrix.arch, 'armhf') + uses: docker://multiarch/ubuntu-core:armhf-bionic + with: + args: > + bash -c + "apt-get -y update && + apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev && + git config --global --add safe.directory $(realpath .) && + env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages && + make -j $(nproc) all && + make -C example ndpiSimpleIntegration && + make -C rrdtool && + make check VERBOSE=1 + " + - name: Display qemu specified architecture (s390x - big endian) + if: startsWith(matrix.arch, 's390x') + uses: docker://multiarch/ubuntu-core:s390x-bionic + with: + args: > + bash -c + "uname -a && + lscpu | grep Endian + " + - name: Configure and compile using qemu for the specified architecture (s390x - big endian) + if: startsWith(matrix.arch, 's390x') + uses: docker://multiarch/ubuntu-core:s390x-bionic + with: + args: > + bash -c + "apt-get -y update && + apt-get -y install git wdiff colordiff autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev && + git config --global --add safe.directory $(realpath .) && + env CC=gcc CFLAGS='-Werror' ./autogen.sh --enable-option-checking=fatal --enable-debug-messages && + make -j $(nproc) all && + make -C example ndpiSimpleIntegration && + make -C rrdtool && + make check VERBOSE=1 + " + diff --git a/.github/workflows/build_scheduled.yml b/.github/workflows/build_scheduled.yml index 962da97152b..a295cdfedf2 100644 --- a/.github/workflows/build_scheduled.yml +++ b/.github/workflows/build_scheduled.yml @@ -109,3 +109,28 @@ jobs: name: ndpi-performance path: ndpi-performance-upload retention-days: 7 + + threadsanitizer: + name: Thread Sanitizer (ubuntu-latest) + runs-on: ubuntu-latest + env: + CFLAGS: -Wextra -Werror + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install Ubuntu Prerequisites + run: | + sudo apt-get update + sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libjson-c libpcap-dev rrdtool librrd-dev parallel + - name: Configure nDPI + run: | + ./autogen.sh --enable-option-checking=fatal --with-thread-sanitizer + - name: Build nDPI + run: | + make -j $(nproc) all + make -j $(nproc) -C example ndpiSimpleIntegration + make -j $(nproc) -C rrdtool + - name: Tests + run: | + NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index de65fccb6c0..28cfc9d0a92 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -18,7 +18,7 @@ jobs: uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master with: oss-fuzz-project-name: 'ndpi' - fuzz-seconds: 1200 + fuzz-seconds: 600 dry-run: false sanitizer: ${{ matrix.sanitizer }} - name: Check Crash (fails when a crash is detected)