diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2e3fd6f08f2..4d6cd2372c7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,7 @@ jobs: run: | cd python python tests.py + test-scripts: name: Test Utils (ubuntu-latest) runs-on: ubuntu-latest @@ -67,7 +68,7 @@ jobs: git diff-index --quiet HEAD -- || true test: - name: ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.gcrypt }} ${{ matrix.compiler }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} ${{ matrix.msan }} ${{ matrix.nBPF }} ${{matrix.lto_gold_linker}} ${{matrix.global_context}} + name: ${{ matrix.os }} ${{ matrix.arch }} ${{ matrix.gcrypt }} ${{ matrix.compiler }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} ${{ matrix.msan }} ${{ matrix.nBPF }} ${{matrix.global_context}} runs-on: ${{ matrix.os }} env: CC: ${{ matrix.compiler }} @@ -77,17 +78,14 @@ jobs: matrix: # macOS-latest == macos-14 on **ARM64**. Also macos-15 is on arm64 # There are some issues with external dependencies on macOS-14/15. Disable it for the time being - os: ["ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04", "macOS-13", "windows-latest"] + os: ["ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04", "macOS-13", "macOS-14"] arch: ["x86_64"] gcrypt: ["--with-local-libgcrypt", ""] compiler: ["cc"] - ar: ["ar"] - ranlib: ["ranlib"] pcre: [""] maxminddb: [""] msan: [""] nBPF: [""] - lto_gold_linker: [""] global_context: [""] #Enable by default include: - compiler: "gcc-4.9" # "Oldest" gcc easily available. To simulate RHEL7 @@ -106,7 +104,6 @@ jobs: maxminddb: "--with-maxminddb" msan: "--with-sanitizer" nBPF: "" - lto_gold_linker: "--with-lto-and-gold-linker" - compiler: "clang-9" # "Oldest" clang easily available os: ubuntu-20.04 arch: "x86_64" @@ -116,8 +113,6 @@ jobs: msan: "--with-sanitizer" nBPF: "" - compiler: "clang-18" # "Newest" clang easily available. See also below... - ar: "llvm-ar-18" - ranlib: "llvm-ranlib-18" os: ubuntu-24.04 arch: "x86_64" gcrypt: "" @@ -125,22 +120,13 @@ jobs: maxminddb: "--with-maxminddb" msan: "--with-sanitizer" nBPF: "" - lto_gold_linker: "--with-lto-and-gold-linker" - compiler: "cc" os: ubuntu-latest arch: "x86_64" gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" - msan: "--with-thread-sanitizer" - nBPF: "" - - compiler: "cc" - os: ubuntu-latest - arch: "x86_64" - gcrypt: "" - pcre: "--with-pcre2" - maxminddb: "--with-maxminddb" - msan: "--with-sanitizer" + msan: "" nBPF: "nBPF" - compiler: "cc" os: ubuntu-latest @@ -148,7 +134,7 @@ jobs: gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" - msan: "--with-sanitizer" + msan: "" nBPF: "" global_context: "--disable-global-context-support" - compiler: "clang" #TODO: some issues with masan/clang/ubuntu-24.04 @@ -254,13 +240,6 @@ jobs: sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5 sudo apt-get update sudo apt-get install ${{ matrix.compiler }} - - name: Install Windows msys2 prerequisites - if: startsWith(matrix.os, 'windows') - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW64 - update: true - install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap mingw-w64-x86_64-libgcrypt parallel - name: Installing MacOS prerequisites if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') run: | @@ -269,6 +248,7 @@ jobs: - name: Install MacOS Prerequisites (libgcrypt) if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt') run: | + echo ${HOMEBREW_DIR} brew install libgcrypt - name: Install MacOS Prerequisites (libpcre2) if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.pcre, '--with-pcre2') @@ -281,22 +261,13 @@ jobs: - name: Configure nDPI on Ubuntu if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') run: | - AR=${{ matrix.ar }} RANLIB=${{ matrix.ranlib }} ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs ${{matrix.lto_gold_linker}} ${{matrix.global_context}} + ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs ${{matrix.global_context}} - name: Configure nDPI on MacOS if: startsWith(matrix.os, 'macOS') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc') run: | ./autogen.sh --enable-option-checking=fatal --enable-debug-messages ${{ matrix.gcrypt }} ${{ matrix.msan }} ${{ matrix.pcre }} ${{ matrix.maxminddb }} --enable-tls-sigs - - name: Configure nDPI on Windows msys2 - if: startsWith(matrix.os, 'windows') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc') - run: | - msys2 -c './autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap ${{ matrix.gcrypt }}' - - name: Build nDPI on Windows msys2 - if: startsWith(matrix.os, 'windows') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.compiler, 'cc') - run: | - msys2 -c 'make -j all' - msys2 -c 'ldd ./example/ndpiReader.exe' - name: Build nDPI - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') + if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') run: | make -j all make -C example ndpiSimpleIntegration @@ -309,45 +280,33 @@ jobs: #There are somes issues with librrd #make -C rrdtool - name: Print nDPI long help - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') + if: startsWith(matrix.arch, 'x86_64') run: | cd ./example && ./ndpiReader -H - name: Install nDPI - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') + if: startsWith(matrix.arch, 'x86_64') run: | DESTDIR=/tmp/ndpi make install ls -alhHR /tmp/ndpi - name: Test nDPI [SYMBOLS] - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') #Only on a few "standard" builds run: | ./utils/check_symbols.sh || { FAILED=$?; echo "::error file=${NDPI_LIB}::Unwanted libc symbols found: ${FAILED}. Please make sure to use only ndpi_malloc/ndpi_calloc/ndpi_realloc/ndpi_free wrapper instead of malloc/calloc/realloc/free."; false; } env: NDPI_LIB: src/lib/libndpi.a - name: Test nDPI [DIFF] - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') + if: startsWith(matrix.arch, 'x86_64') run: | NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh - name: Test nDPI [UNIT] #Some issues with masan + json-c. Disable the test as workaround - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') && !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') + if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-memory-sanitizer') && !startsWith(matrix.os, 'macos-14') && !startsWith(matrix.os, 'macos-15') run: | ./tests/do-unit.sh - name: Test nDPI [DGA] - if: startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.os, 'windows') + if: startsWith(matrix.arch, 'x86_64') run: | ./tests/do-dga.sh - - name: Test nDPI [DIFF] (runs only on windows jobs) - if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows') - run: | - msys2 -c 'NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh' - - name: Test nDPI [UNIT] (runs only on windows jobs) - if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows') - run: | - msys2 -c './tests/do-unit.sh' - - name: Test nDPI [DGA] (runs only on windows jobs) - if: startsWith(matrix.arch, 'x86_64') && startsWith(matrix.os, 'windows') - run: | - msys2 -c './tests/do-dga.sh' - name: Generate/Verify tarball if: startsWith(matrix.os, 'ubuntu-latest') && startsWith(matrix.arch, 'x86_64') run: | @@ -361,7 +320,7 @@ jobs: make cd ../.. - name: Build nDPI [Mingw-w64] (runs only on ubuntu jobs) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') && !startsWith(matrix.nBPF, 'nBPF') #Only on a few "standard" builds run: | make distclean ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --host=x86_64-w64-mingw32 @@ -440,3 +399,74 @@ jobs: make -C rrdtool && make check VERBOSE=1 " + + test-lto-gold: + # Options used by oss-fuzz: we only want to check that everything compile fine; no need to run the tests + name: LTO and Gold Linker ${{ matrix.compiler }} + runs-on: ubuntu-24.04 + env: + CC: ${{ matrix.compiler }} + CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS + strategy: + fail-fast: true + matrix: + compiler: ["gcc-14", "clang-18"] # "Newest" gcc/clang easily available + include: + - compiler: "gcc-14" + ar: "ar" + ranlib: "ranlib" + - compiler: "clang-18" + ar: "llvm-ar-18" + ranlib: "llvm-ranlib-18" + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Install Ubuntu Prerequisites + run: | + sudo apt-get update + sudo apt-get install autoconf automake debhelper libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev rrdtool librrd-dev parallel + - name: Configure nDPI + run: | + AR=${{ matrix.ar }} RANLIB=${{ matrix.ranlib }} ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --with-sanitizer --with-lto-and-gold-linker + - name: Build nDPI + run: | + make -j $(nproc) all + make -j $(nproc) -C example ndpiSimpleIntegration + make -j $(nproc) -C rrdtool + - name: Print nDPI long help + run: | + cd ./example && ./ndpiReader -H + + test-windows: + name: ${{ matrix.os }} (msys2) ${{ matrix.gcrypt }} + runs-on: ${{ matrix.os }} + env: + CFLAGS: -Wextra -Werror -DNDPI_EXTENDED_SANITY_CHECKS + strategy: + fail-fast: true + matrix: + os: ["windows-latest"] + gcrypt: ["--with-local-libgcrypt", ""] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: Install Windows msys2 prerequisites + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: git mingw-w64-x86_64-toolchain automake1.16 automake-wrapper autoconf libtool make mingw-w64-x86_64-json-c mingw-w64-x86_64-crt-git mingw-w64-x86_64-pcre mingw-w64-x86_64-libpcap mingw-w64-x86_64-libgcrypt parallel + - name: Configure nDPI on Windows msys2 + run: | + msys2 -c './autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --disable-npcap ${{ matrix.gcrypt }}' + - name: Build nDPI on Windows msys2 + run: | + msys2 -c 'make -j all' + msys2 -c 'ldd ./example/ndpiReader.exe' + - name: Tests + run: | + msys2 -c 'NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh' + msys2 -c './tests/do-unit.sh' + msys2 -c './tests/do-dga.sh' diff --git a/.github/workflows/build_scheduled.yml b/.github/workflows/build_scheduled.yml index 962da97152b..a295cdfedf2 100644 --- a/.github/workflows/build_scheduled.yml +++ b/.github/workflows/build_scheduled.yml @@ -109,3 +109,28 @@ jobs: name: ndpi-performance path: ndpi-performance-upload retention-days: 7 + + threadsanitizer: + name: Thread Sanitizer (ubuntu-latest) + runs-on: ubuntu-latest + env: + CFLAGS: -Wextra -Werror + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install Ubuntu Prerequisites + run: | + sudo apt-get update + sudo apt-get install autoconf automake libtool pkg-config gettext flex bison libjson-c libpcap-dev rrdtool librrd-dev parallel + - name: Configure nDPI + run: | + ./autogen.sh --enable-option-checking=fatal --with-thread-sanitizer + - name: Build nDPI + run: | + make -j $(nproc) all + make -j $(nproc) -C example ndpiSimpleIntegration + make -j $(nproc) -C rrdtool + - name: Tests + run: | + NDPI_FORCE_PARALLEL_UTESTS=1 NDPI_SKIP_PARALLEL_BAR=1 ./tests/do.sh diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index de65fccb6c0..28cfc9d0a92 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -18,7 +18,7 @@ jobs: uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master with: oss-fuzz-project-name: 'ndpi' - fuzz-seconds: 1200 + fuzz-seconds: 600 dry-run: false sanitizer: ${{ matrix.sanitizer }} - name: Check Crash (fails when a crash is detected) diff --git a/configure.ac b/configure.ac index 43c8a04fd48..b355f3b10ad 100644 --- a/configure.ac +++ b/configure.ac @@ -228,6 +228,13 @@ AS_IF([test "x${enable_tls_sigs}" = "xyes"],[ NDPI_CFLAGS="-W -Wall -Wno-address-of-packed-member ${NDPI_CFLAGS}" +dnl> MacOS brew.sh +#HOMEBREW_DIR=/opt/homebrew +if test -d ${HOMEBREW_DIR}; then + NDPI_CFLAGS="${NDPI_CFLAGS} -I ${HOMEBREW_DIR}/include" + NDPI_LDFLAGS="${NDPI_LDFLAGS} -L ${HOMEBREW_DIR}/lib" +fi + AS_IF([test "${with_lto_and_gold_linker+set}" = set], [ NDPI_CFLAGS="${NDPI_CFLAGS} -flto -fuse-ld=gold -Wno-unused-command-line-argument" NDPI_LDFLAGS="${NDPI_LDFLAGS} ${NDPI_CFLAGS}"