Merge remote-tracking branch 'origin/master' into netlist-modifier-wip

.github/workflows/macOS.yml

@@ -89,12 +89,18 @@ jobs:
         run: |
           mkdir -p build
           cd build
-          export PATH="/usr/local/opt/qt@5/bin:$PATH"
-          ls ..
-          cmake -G Ninja .. -DQt5_DIR=/usr/local/opt/qt@5/lib/cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_ALL_PLUGINS=ON -DBUILD_TESTS=ON -DPL_GUI=ON -DCMAKE_C_COMPILER=/usr/local/opt/llvm@14/bin/clang -DCMAKE_CXX_COMPILER=/usr/local/opt/llvm@14/bin/clang++
+          export PATH="$(brew --prefix qt@5)/bin:$PATH"
+          brew ls llvm@14
+          brew --prefix llvm@14
+          ls -l /usr/local
+          ls -l /opt/homebrew/opt
+          ls -l /opt/homebrew/opt/llvm@14
+          ls -l /opt/homebrew/opt/llvm@14/bin
+          ls -l /opt/homebrew/opt/llvm@14/bin/clang
+          cmake -G Ninja .. -DQt5_DIR="$(brew --prefix qt@5)/lib/cmake" -DCMAKE_BUILD_TYPE=Debug -DBUILD_ALL_PLUGINS=ON -DBUILD_TESTS=ON -DPL_GUI=ON -DCMAKE_C_COMPILER=$(brew --prefix llvm@14)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm@14)/bin/clang++
         env:
-          LDFLAGS: "-L/usr/local/opt/qt@5/lib -L/usr/local/opt/llvm@14/lib -Wl,-rpath,/usr/local/opt/llvm@14/lib"
-          CPPFLAGS: "-I/usr/local/opt/llvm@14/include"
+          LDFLAGS: "-L$(brew --prefix qt@5)/lib -L$(brew --prefix llvm@14)/lib -Wl,-rpath,$(brew --prefix llvm@14)/lib"
+          CPPFLAGS: "-I$(brew --prefix llvm@14)/include"
           HAL_BASE_PATH: ${{runner.workspace}}/hal/build
           CCACHE_DIR: ${{runner.workspace}}/.ccache
           CCACHE_COMPRESS: true
@@ -107,12 +113,12 @@ jobs:
         # Execute the build.  You can specify a specific target with "--target <NAME>"
         run: |
           cd build
-          export PATH="/usr/local/opt/qt/bin:$PATH"
+          export PATH="$(brew --prefix qt@5)/bin:$PATH"
           export BUILD_TYPE=Debug
           cmake --build . --target all --clean-first --config $BUILD_TYPE
         env:
-          LDFLAGS: "-L/usr/local/opt/qt@5/lib -L/usr/local/opt/llvm@14/lib -Wl,-rpath,/usr/local/opt/llvm@14/lib"
-          CPPFLAGS: "-I/usr/local/opt/llvm@14/include"
+          LDFLAGS: "-L$(brew --prefix qt@5)/lib -L$(brew --prefix llvm@14)/lib -Wl,-rpath,$(brew --prefix llvm@14)/lib"
+          CPPFLAGS: "-I$(brew --prefix llvm@14)/include"
           HAL_BASE_PATH: ${{runner.workspace}}/hal/build
           CCACHE_DIR: ${{runner.workspace}}/.ccache
           CCACHE_COMPRESS: true
@@ -133,8 +139,8 @@ jobs:
         #  ninja -v hal_coverage
         # bash <(curl -s https://codecov.io/bash) -f hal_coverage.info.cleaned || echo "Codecov did not collect coverage reports"
         env:
-          LDFLAGS: "-L/usr/local/opt/qt@5/lib -L/usr/local/opt/llvm@14/lib -Wl,-rpath,/usr/local/opt/llvm@14/lib"
-          CPPFLAGS: "-I/usr/local/opt/llvm@14/include"
+          LDFLAGS: "-L$(brew --prefix qt@5)/lib -L$(brew --prefix llvm@14)/lib -Wl,-rpath,$(brew --prefix llvm@14)/lib"
+          CPPFLAGS: "-I$(brew --prefix llvm@14)/include"
           HAL_BASE_PATH: ${{runner.workspace}}/hal/build
           CCACHE_DIR: ${{runner.workspace}}/.ccache
           CCACHE_COMPRESS: true

.github/workflows/ubuntu20.04.yml → .github/workflows/ubuntu24.04.yml

@@ -1,4 +1,4 @@
-name: Ubuntu 20.04
+name: Ubuntu 24.04
 
 on:
   push:
@@ -7,13 +7,13 @@ on:
 #      - v*
 
 jobs:
-  build_ubuntu_20_04:
-    name:  Build and Test on Ubuntu 20.04
+  build_ubuntu_24_04:
+    name: Build and Test on Ubuntu 24.04
 
     strategy:
       matrix:
-        #        runs-on: [ ubuntu-20.04, macOS-latest]
-        runs-on: [ ubuntu-20.04 ]
+        #        runs-on: [ ubuntu-22.04, macOS-latest]
+        runs-on: [ ubuntu-24.04 ]
       fail-fast: false
 
     runs-on: ${{ matrix.runs-on }}

CHANGELOG.md

@@ -2,32 +2,66 @@
 All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
-* refactored module widget
-  * added option to show gate content for each module
-  * added option to show interior nets for each module
-  * added `Isolate in new view` feature for nets
-  * added button to expand or collapse all tree items
-  * added delete module action and shortcut
-  * added entries for context menu
-* refactored search bar
-  * changed appearance of search bar to be more intuitive
-  * added menu for extended options - e.g. option to search in selected columns only
-  * added search history
-  * added filter proxy class for trees and tables increasing the search performance
-* refactored layouter module
-  * switched to multithreaded algorithm
-  * boosted performance by using classes with faster memory access
-  * removed layouter code used prior to version 3.1.0 - thus removing the setting option to use that code
-  * added setting option to dump junction layout input data for experts to debug in case of layout errors
-* module pins
-  * added qualifier for `pin_changed` core events telling receiver details about the recent modification
-  * added event scope and stacking classes so that `pin_changed` events can be collected and prioritized
-  * added specific GUI handler for every `pin_changed` event thus replacing the reload-entire-pingroup-tree policy
-  * added class `ActionPingroup` so that UNDO function works for all pin / pin group actions issued from GUI
+* **WARNING:** this release breaks compatibility with Ubuntu 20.04 LTS
+* **WARNING:** this release breaks the API of the `graph_algorithm`, `dataflow`, and `xilinx_toolbox` plugins
+* GUI
+  * refactored module widget
+    * added option to show gate content for each module
+    * added option to show interior nets for each module
+    * added `Isolate in new view` feature for nets
+    * added button to expand or collapse all tree items
+    * added delete module action and shortcut
+    * added entries for context menu
+    * adapted appearance for menu content tree, selection details tree, grouping content tree (same model for all)
+  * refactored search bar
+    * changed appearance of search bar to be more intuitive
+    * added menu for extended options - e.g. option to search in selected columns only
+    * added search history
+    * added filter proxy class for trees and tables increasing the search performance
+  * refactored layouter module
+    * switched to multithreaded algorithm
+    * boosted performance by using classes with faster memory access
+    * removed layouter code used prior to version 3.1.0 - thus removing the setting option to use that code
+    * added setting option to dump junction layout input data for experts to debug in case of layout errors
+* plugins
+  * added `hawkeye` plugin for the detection of symmetric cryptographic implementations in gate-level netlists
+    * see publication `HAWKEYE - Recovering Symmetric Cryptography From Hardware Circuits` at CRYPTO'24 for details
+  * changed `graph_algorithm` plugin
+    * updated the igraph dependency shipped with HAL
+    * graph corresponding to a netlist is now encapsulated within an `NetlistGraph` object that allows easy interaction with the graph
+    * added new functions for computing neighborhoods, shortest paths, subgraphs, and (strongly) connected components
+    * changed the API to facilitate for the aforementioned changes and made everything accessible via Python
+  * changed `dataflow_analysis` plugin
+    * can now operate on arbitrary, user-defined gate types, not only FFs 
+    * user can now specify the pin types to be considered as control pins
+    * can now take known registers and other known word-level structures into account during analysis
+    * changed the API to facilitate for the aforementioned changes
+  * changed `simulator` plugin
+    * added feature to VCD parser: removal of leading backslash and trailing whitespace from waveform name
+    * extended maximum line with the CSV parser can handle
+    * changed warning messages for waveform parsing and made them more specific
+    * changed policy toward 'dangling' wires, they are no longer ignored but considered as global inputs or outputs
+  * changed `xilinx_toolbox` plugin
+    * added `split_shift_registers` function to split `SRL16E` gates into multiple flip-flops
+    * changed Python bindings for better usability
+* netlist
+  * module pins
+    * added qualifier for `pin_changed` core events telling receiver details about the recent modification
+    * added event scope and stacking classes so that `pin_changed` events can be collected and prioritized
+    * added specific GUI handler for every `pin_changed` event thus replacing the reload-entire-pingroup-tree policy
+    * added class `ActionPingroup` so that UNDO function works for all pin / pin group actions issued from GUI
+* decorators
+  * added `NetlistTraversalDecorator` to ease exploration of a netlist
+    * added `get_next_matching_gates` to get successor/predecessor gates matching a certain condition
+    * added `get_next_matching_gates_until` to get successor/predecessor gates until a certain condition is fulfilled
+    * added `get_next_matching_gates_until_depth` to get successor/predecessor gates up to a certain depth
+    * added `get_next_sequential_gates` and `get_next_sequential_gates_map` to get the next layer of sequential successors/predecessors
+    * added `get_next_combinational_gates` to get all combinational gates until the next non-combinational gates are reached
 * miscellaneous
+  * added support for Ubuntu 24.04 LTS
   * added INIT field declaration to FF-gate-types in example library
   * added drag'n drop feature allowing to move several nodes in graph view at same time
-  * added functions to Python GUI API to create, modifiy and delete views
+  * added functions to Python GUI API to create, modify and delete views
   * added GUI PluginParameter type `ComboBox` for parameters that can be requested from plugin
   * added GUI PluginParameter types `Module` and `Gated` for parameters that can be requested from plugin
   * added `Show content` button to `Groupings` widget to show content of grouping as a list
@@ -36,11 +70,20 @@ All notable changes to this project will be documented in this file.
   * added extended gate library picker when importing a netlist
   * added keyboard shortcut for delete-item action from toolbar
   * added parameter `force_name` to enforce pin (group) renaming to `Module::set_pin_name`, `Module::set_pin_group_name`, `Module::create_pin`, and `Module::create_pin_group`
+  * added gate type properties `fifo` and `shift_register`
+  * added pin types `status`, `error`, `error_detection`, `done`, and `control`
+  * added optional filter to `Net::get_num_of_sources` and `Net::get_num_of_destinations`
+  * added function `unify_ff_outputs` to netlist preprocessing plugin
   * changed supported input file formats for import from hard coded list to list provided by loadable parser plugins
   * changed behavior of import netlist dialog, suggest only non-existing directory names and loop until an acceptable name was entered
   * changed appearance and behavior of import project dialog, make sure existing hal projects don't get overwritten
   * changed installation script policy to install Python packages (omit 'pip install' which would need virtual environment)
+  * removed hard coded path names from CI MacOS workflow script
+  * deprecated many functions in `netlist_utils` as they have been moved somewhere else
 * bugfixes
+  * fixed saleae input data reader which gets linked into external Verilator simulation code
+  * fixed waveform viewer: opening old results will no longer generate the same view twice
+  * fixed waveform viewer: opening old results will by now also update waveform time axis
   * fixed colors in Python Console when switching between color schemes
   * fixed pybind of `Module::get_gates`
   * fixed Python script execution abort button disappearing when switching tabs
@@ -54,8 +97,12 @@ All notable changes to this project will be documented in this file.
   * fixed format string handling of enums in log outputs
   * fixed restoring user assigned module colors from project file
   * fixed no scrollbar shown in `Data` tab of `Selection Details` widget
+  * fixed declaration of FF-gate type in example gate library
+  * fixed error which could cause crashes in do-not-render-layout-until-complex-operation-finished algorithm
+  * fixed wrong placements of nodes in view by XML-macro (might even crash)
   * fixed problems in GUI plugin management caused by addressing plugins by absolute path
   * fixed several bugs related to moving node boxes in GUI by drag'n'drop
+  * fixed several bugs in automated tests, eliminate cases which produce non-deterministic results
 
 ## [4.2.0](v4.2.0) - 2023-05-24 10:02:04-07:00 (urgency: medium)
 * GUI plugin manager

CMakeLists.txt

@@ -62,6 +62,10 @@ option(HAL_VERSION_MAJOR "Pass major version via cmake options" "")
 option(HAL_VERSION_MINOR "Pass minor version via cmake options" "")
 option(HAL_VERSION_PATCH "Pass patch version via cmake options" "")
 option(USE_LIBCXX "Force the use of LIBCXX for e.g. gcc" FALSE)
+option(USE_VENDORED_PYBIND11 "Use vendored 'pybind11' Python library" ON)
+option(USE_VENDORED_SPDLOG "Use vendored 'spdlog' library" ON)
+option(USE_VENDORED_QUAZIP "Use vendored 'quazip' library" ON)
+option(USE_VENDORED_IGRAPH "Use vendored 'igraph' library" ON)
 option(BUILD_ALL_PLUGINS "Build all available plugins" OFF)
 option(BUILD_TESTS "Enable test builds" OFF)
 option(BUILD_COVERAGE "Enable code coverage build" OFF)
@@ -325,8 +329,12 @@ install(FILES tools/genversion.py
     GROUP_EXECUTE
     WORLD_READ
     WORLD_EXECUTE)
-install(DIRECTORY ${CMAKE_SOURCE_DIR}/deps/pybind11 DESTINATION ${HAL_CMAKECONFIG_INSTALL_DIR})
-install(DIRECTORY ${CMAKE_SOURCE_DIR}/deps/spdlog-${spdlog_VERSION} DESTINATION ${HAL_CMAKECONFIG_INSTALL_DIR})
+if(USE_VENDORED_PYBIND11)
+    install(DIRECTORY ${CMAKE_SOURCE_DIR}/deps/pybind11 DESTINATION ${HAL_CMAKECONFIG_INSTALL_DIR})
+endif()
+if(USE_VENDORED_SPDLOG)
+    install(DIRECTORY ${CMAKE_SOURCE_DIR}/deps/spdlog-${spdlog_VERSION} DESTINATION ${HAL_CMAKECONFIG_INSTALL_DIR})
+endif()
 install(DIRECTORY ${CMAKE_SOURCE_DIR}/deps/subprocess DESTINATION ${HAL_CMAKECONFIG_INSTALL_DIR})
 
 

Dockerfile

@@ -14,7 +14,7 @@ RUN apt-get update -y && \
     apt-get install -y lsb-release
 
 COPY . .
-RUN ./install_dependencies.sh
+RUN HAL_DOCKER=1 ./install_dependencies.sh
 
 RUN mkdir build
 WORKDIR ${hal_path}/build/

README.md

@@ -1,5 +1,5 @@
 # Welcome to HAL! 
-[![Ubuntu 20.04](https://github.com/emsec/hal/actions/workflows/ubuntu20.04.yml/badge.svg)](https://github.com/emsec/hal/actions/workflows/ubuntu20.04.yml)  [![Ubuntu 22.04](https://github.com/emsec/hal/actions/workflows/ubuntu22.04.yml/badge.svg)](https://github.com/emsec/hal/actions/workflows/ubuntu22.04.yml)  [![macOS](https://github.com/emsec/hal/actions/workflows/macOS.yml/badge.svg)](https://github.com/emsec/hal/actions/workflows/macOS.yml) [![Deploy Documentation](https://github.com/emsec/hal/actions/workflows/releaseDoc.yml/badge.svg)](https://github.com/emsec/hal/actions/workflows/releaseDoc.yml) [![Doc: C++](https://img.shields.io/badge/doc-c%2B%2B-orange)](https://emsec.github.io/hal/doc/) [![Doc: Python](https://img.shields.io/badge/doc-python-red)](https://emsec.github.io/hal/pydoc/)
+[![Ubuntu 22.04](https://github.com/emsec/hal/actions/workflows/ubuntu22.04.yml/badge.svg)](https://github.com/emsec/hal/actions/workflows/ubuntu22.04.yml)  [![Ubuntu 24.04](https://github.com/emsec/hal/actions/workflows/ubuntu24.04.yml/badge.svg)](https://github.com/emsec/hal/actions/workflows/ubuntu24.04.yml)  [![macOS](https://github.com/emsec/hal/actions/workflows/macOS.yml/badge.svg)](https://github.com/emsec/hal/actions/workflows/macOS.yml) [![Deploy Documentation](https://github.com/emsec/hal/actions/workflows/releaseDoc.yml/badge.svg)](https://github.com/emsec/hal/actions/workflows/releaseDoc.yml) [![Doc: C++](https://img.shields.io/badge/doc-c%2B%2B-orange)](https://emsec.github.io/hal/doc/) [![Doc: Python](https://img.shields.io/badge/doc-python-red)](https://emsec.github.io/hal/pydoc/)
 
 
 HAL \[/hel/\] is a comprehensive netlist reverse engineering and manipulation framework.
@@ -30,7 +30,7 @@ We want HAL to enable a common baseline for researchers and analysts to improve
 - **Stability** is ensured via a rich test suite
 
 HAL is actively developed by the Embedded Security group of the [Max Planck Institute for Security and Privacy](https://www.mpi-sp.org).
-Apart from multiple research projects, it is also used in our university lecture [Introduction to Hardware Reverse Engineering](https://www.ei.ruhr-uni-bochum.de/studium/lehrveranstaltungen/832/).
+Apart from multiple research projects, it is also used in our university lecture "Einführung ins Hardware Reverse Engineering" (Introduction to Hardware Reverse Engineering) at Ruhr University Bochum (RUB).
 
 Note that we also have a set of **modern** state-of-the-art benchmark circuits for the evaluation of netlist reverse engineering techniques available in a seperate [repository](https://github.com/emsec/hal-benchmarks).
 
@@ -54,9 +54,6 @@ This repository contains a selection of curated plugins:
 ## Documentation
 A comprehensive documentation of HAL's features from a user perspective is available in our [Wiki](https://github.com/emsec/hal/wiki). In addition, we provide a full [C++ API](https://emsec.github.io/hal/doc/) and [Python API](https://emsec.github.io/hal/pydoc/) documentation.
 
-## Slack, Contact and Support
-For all kinds of inquiries, please contact us using our dedicated e-mail address: [[email protected]](mailto:[email protected]).
-
 <a name="build-instructions"></a>
 # Build Instructions 
 

cmake/detect_dependencies.cmake

@@ -153,20 +153,33 @@ find_package(pybind11 2.7 CONFIG)
 if(${pybind11_FOUND})
     message(VERBOSE "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}")
     message(VERBOSE "Found pybind11 >= 2.7")
-else()
+elseif(USE_VENDORED_PYBIND11)
     message(STATUS "pybind11 >= 2.7 not found, will build our provided version")
     add_subdirectory(deps/pybind11)
+else()
+    message(FATAL_ERROR "pybind11 >= 2.7 not found and USE_VENDORED_PYBIND11 is OFF")
 endif()
 
 # ###############################
 # ####   spdlog
 # ###############################
-message(STATUS "using spdlog from deps")
-set(spdlog_VERSION 1.9.2)
-add_library(spdlog::spdlog INTERFACE IMPORTED)
-set_target_properties(spdlog::spdlog PROPERTIES
-    INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/deps/spdlog-${spdlog_VERSION}/include"
-)
+
+if(USE_VENDORED_SPDLOG)
+    message(STATUS "using spdlog from deps")
+    set(spdlog_VERSION 1.9.2)
+    add_library(spdlog::spdlog INTERFACE IMPORTED)
+    set_target_properties(spdlog::spdlog PROPERTIES
+        INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/deps/spdlog-${spdlog_VERSION}/include"
+    )
+else()
+    find_package(spdlog REQUIRED)
+    if(spdlog_FOUND)
+        get_target_property(SPDLOG_HEADERS_DIR spdlog::spdlog INTERFACE_INCLUDE_DIRECTORIES)
+        message(STATUS "Using system's spdlog headers at ${SPDLOG_HEADERS_DIR}")
+    else()
+        message(FATAL_ERROR "spdlog was not found and USE_VENDORED_SPDLOG is OFF")
+    endif()
+endif()
 
 # ###############################
 # ####   subprocess
@@ -259,7 +272,15 @@ endif(Z3_FOUND)
 # ###############################
 # ####   igraph
 # ###############################
-set(IGRAPH_SUBDIR "${CMAKE_SOURCE_DIR}/deps/igraph-0.9.10")
-add_subdirectory(${IGRAPH_SUBDIR})
-get_directory_property(IGRAPH_INCLUDES DIRECTORY ${IGRAPH_SUBDIR} DEFINITION IGRAPH_INCLUDES)
-get_directory_property(IGRAPH_LIB DIRECTORY ${IGRAPH_SUBDIR} DEFINITION IGRAPH_LIB)
+if(USE_VENDORED_IGRAPH)
+    set(IGRAPH_SUBDIR "${CMAKE_SOURCE_DIR}/deps/igraph-0.10.x")
+    add_subdirectory(${IGRAPH_SUBDIR})
+    get_directory_property(igraph_INCLUDES DIRECTORY ${IGRAPH_SUBDIR} DEFINITION IGRAPH_INCLUDES)
+    get_directory_property(igraph_LIBRARIES      DIRECTORY ${IGRAPH_SUBDIR} DEFINITION IGRAPH_LIB)
+    message(STATUS "Using igraph from ${IGRAPH_SUBDIR}")
+else()
+    find_package(igraph REQUIRED)
+    get_target_property(igraph_LIBRARIES igraph::igraph IMPORTED_LOCATION_RELEASE)
+    get_target_property(igraph_INCLUDES igraph::igraph INTERFACE_INCLUDE_DIRECTORIES)
+    message(STATUS "Using system's igraph from ${igraph_LIBRARIES}")
+endif()