Knutel/peter d cmake include fixes

.circleci/fast_finish_ci_pr_build.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
-curl https://raw.githubusercontent.com/conda-forge/conda-forge-ci-setup-feedstock/master/recipe/conda_forge_ci_setup/ff_ci_pr_build.py | \
+curl https://raw.githubusercontent.com/conda-forge/conda-forge-ci-setup-feedstock/main/recipe/conda_forge_ci_setup/ff_ci_pr_build.py | \
      python - -v --ci "circle" "${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" "${CIRCLE_BUILD_NUM}" "${CIRCLE_PR_NUMBER}"

.github/workflows/argos_lint.yml

@@ -0,0 +1,45 @@
+name: Argos Linters
+
+on:
+  # Trigger the workflow on push or pull request,
+  # but only for the main branch
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+defaults:
+  run:
+    working-directory: helios/pipeViewer
+
+jobs:
+  build:
+    name: pylint
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install mypy numpy flake8
+
+    - name: Running mypy
+      run: |
+        mypy --install-types --non-interactive pipe_view/argos.py
+
+    - name: Running flake8
+      run: |
+        flake8 $(git ls-files 'pipe_view/*.py')

.scripts/build_steps.sh

@@ -33,9 +33,9 @@ CONDARC
 
 
 mamba install --update-specs --yes --quiet --channel conda-forge \
-    conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12"
+    conda-build pip boa conda-forge-ci-setup=3
 mamba update --update-specs --yes --quiet --channel conda-forge \
-    conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12"
+    conda-build pip boa conda-forge-ci-setup=3
 
 # set up the condarc
 setup_conda_rc "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}"

.scripts/create_conda_build_artifacts.sh

@@ -79,8 +79,8 @@ if [[ ! -z "$BLD_ARTIFACT_PREFIX" ]]; then
         echo "##vso[task.setVariable variable=BLD_ARTIFACT_NAME]$BLD_ARTIFACT_NAME"
         echo "##vso[task.setVariable variable=BLD_ARTIFACT_PATH]$BLD_ARTIFACT_PATH"
     elif [[ "$CI" == "github_actions" ]]; then
-        echo "::set-output name=BLD_ARTIFACT_NAME::$BLD_ARTIFACT_NAME"
-        echo "::set-output name=BLD_ARTIFACT_PATH::$BLD_ARTIFACT_PATH"
+        echo "BLD_ARTIFACT_NAME=$BLD_ARTIFACT_NAME" >> $GITHUB_OUTPUT
+        echo "BLD_ARTIFACT_PATH=$BLD_ARTIFACT_PATH" >> $GITHUB_OUTPUT
     fi
 fi
 
@@ -107,7 +107,7 @@ if [[ ! -z "$ENV_ARTIFACT_PREFIX" ]]; then
         echo "##vso[task.setVariable variable=ENV_ARTIFACT_NAME]$ENV_ARTIFACT_NAME"
         echo "##vso[task.setVariable variable=ENV_ARTIFACT_PATH]$ENV_ARTIFACT_PATH"
     elif [[ "$CI" == "github_actions" ]]; then
-        echo "::set-output name=ENV_ARTIFACT_NAME::$ENV_ARTIFACT_NAME"
-        echo "::set-output name=ENV_ARTIFACT_PATH::$ENV_ARTIFACT_PATH"
+        echo "ENV_ARTIFACT_NAME=$ENV_ARTIFACT_NAME" >> $GITHUB_OUTPUT
+        echo "ENV_ARTIFACT_PATH=$ENV_ARTIFACT_PATH" >> $GITHUB_OUTPUT
     fi
 fi

.scripts/run_osx_build.sh

@@ -24,9 +24,9 @@ source ${MINIFORGE_HOME}/etc/profile.d/conda.sh
 conda activate base
 
 mamba install --update-specs --quiet --yes --channel conda-forge \
-    conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12"
+    conda-build pip boa conda-forge-ci-setup=3
 mamba update --update-specs --yes --quiet --channel conda-forge \
-    conda-build pip boa conda-forge-ci-setup=3 "py-lief<0.12"
+    conda-build pip boa conda-forge-ci-setup=3
 
 
 

README.md

@@ -15,13 +15,69 @@ MAP is broken into two parts:
 1. **Sparta** -- A set of C++ classes (C++17) used to construct, bind, and run full simulation designs and produce performance analysis data in text form, database form, or HDF5. It's a modeling framework.
 1. **Helios** -- A set of python tools used to visualize, analyze, and deep dive data generated for a Sparta-built simulator.  It's a visualization toolset.
 
-## Current build status
+## Current Regression Status
 
 [![CircleCI](https://circleci.com/gh/sparcians/map.svg?style=svg)](https://circleci.com/gh/sparcians/map)
 [![MacOS Build Status](https://dev.azure.com/sparcians/map/_apis/build/status/sparcians.map?branchName=master&label=MacOS)](https://dev.azure.com/sparcians/map/_build/latest?definitionId=1&branchName=master)
 [![Documentation](https://github.com/sparcians/map/workflows/Documentation/badge.svg)](https://sparcians.github.io/map/)
 
-## Updating Regression/Build Environments
+## Building MAP
+
+Building MAP is done in two parts
+
+1. Sparta, the modeling framework: build sparta only in the sparta folder
+2. Argos, the transaction viewer in Helios in the helios folder. Note that to build and use helios, you will need sparta built and installed somwehere on your system.
+
+The MAP repository has numerous dependencies, which are listed in a
+[conda recipe](https://github.com/sparcians/map/blob/master/conda.recipe/meta.yaml),
+and the versions of these libraries continuously change.
+
+However, with the use of the conda recipe, users can set up a conda
+environment that will build and run the tools found in this repository.
+
+This guide assumes the user is not familiar with conda nor has it
+installed and would like to build everything (not just sparta).
+
+1. If conda is not installed, install it
+   * Get miniconda and install: https://docs.conda.io/en/latest/miniconda.html
+   * You can install miniconda anywhere
+1. Activate conda `conda activate`
+1. Go to the root of MAP
+   * `cd map`
+1. Install JSON and YAML parsers
+   * `conda install -c conda-forge jq`
+   * `conda install -c conda-forge yq`
+1. Create a sparta conda development environment
+   * `./scripts/create_conda_env.sh sparta dev`
+1. Activate the environment
+   * `conda activate sparta`
+1. To build MAP and it's components
+   * `conda activate sparta`
+   * `cd map/sparta; mkdir release; cd release`
+   * `cmake -DCMAKE_BUILD_TYPE=Release ..`
+   * `make`
+   * `cmake --install . --prefix $CONDA_PREFIX`
+1. To build Helios/Argos transaction viewer:
+   * `conda activate sparta`
+   * `cd map/helios; mkdir release; cd release`
+   * `cmake -DCMAKE_BUILD_TYPE=Release -DSPARTA_SEARCH_DIR=<SPARTA_INSTALLED_LOCATION> ..`
+   * `make`
+   * `cmake --install . --prefix $CONDA_PREFIX`
+
+A few interesting cmake options to help resolve dependencies are:
+
+For both Sparta and Helios:
+
+* `-DBOOST_ROOT=<BOOST_LOCATION>`: Custom Boost location
+* `-DCMAKE_INSTALL_PREFIX=`: Install prefix, defaults to a system wide location normally so you can use this for a local install in a home folder for example.
+
+Helios only:
+
+* `-DSPARTA_SEARCH_DIR=<SPARTA_INSTALLED_LOCATION>`: Use this to ensure helios finds Sparta, when you installed it in a non-default location  Not providing this will try and find sparta in the map source tree, but this might fail, if you did not build in a folder named `release`
+* `-DPython3_ROOT_DIR=<PYTHON_LOCATION>`: Not often needed but useful to point to the right python if you are not in a conda env)
+
+
+## Updating Regression/Build Environments for CI
 
 CI files are generated when the command `conda smithy rerender` is run
 inside a MAP clone.  That command uses the following files to control
@@ -33,5 +89,10 @@ the generation of the CI-specific control files:
 
 To update versions of OSes, edit the following file:
 https://github.com/sparcians/map/blob/master/conda.recipe/conda_build_config.yaml
-and then run `conda smithy rerender`.  (Ensure `conda install
-conda-smithy` into your conda installation for that command to exist).
+and then run `conda smithy rerender`.
+
+Install `conda smithy` instructions:
+```
+conda install -n root -c conda-forge conda-smithy
+conda install -n root -c conda-forge conda-package-handling
+```

conda.recipe/build.sh

@@ -41,12 +41,13 @@ env | sort
 
 ################################################################################
 #
-#  BUILD & TEST MAP
+#  BUILD & TEST MAP/SPARTA
 #
 ################################################################################
 
 df -h /
 
+pushd sparta
 mkdir -p release
 pushd release
 cmake -DCMAKE_BUILD_TYPE=Release \
@@ -67,16 +68,31 @@ df -h /
 # by cd'ing into the example subdir and running ctest
 # because not all of the subdirs of example create their
 # own <subdir>_regress target like the core example.
-(cd sparta/example && CTEST_OUTPUT_ON_FAILURE=1 ctest -j "$CPU_COUNT" --test-action test)
+(cd example && CTEST_OUTPUT_ON_FAILURE=1 ctest -j "$CPU_COUNT" --test-action test)
 df -h /
 
 # if we want to create individual packages this should move into a separate install script for only SPARTA
 # and we might want to create separate install targets for the headers and the libs and the doc
 cmake --build . --target install
 df -h /
-
+popd
 popd
 
+################################################################################
+#
+#  BUILD MAP/HELIOS
+#
+################################################################################
+pushd helios
+mkdir -p release
+pushd release
+cmake -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_INSTALL_PREFIX:PATH="$PREFIX" \
+      "${CMAKE_PLATFORM_FLAGS[@]}" \
+      ..
+cmake --build . -j "$CPU_COUNT" || cmake --build . -v
+popd
+popd
 
 ################################################################################
 #

helios/CMakeLists.txt

@@ -1,3 +1,5 @@
 cmake_minimum_required (VERSION 3.15)
 
+project(helios)
+
 add_subdirectory (pipeViewer)

helios/pipeViewer/.gitignore

@@ -0,0 +1,4 @@
+build/
+dist/
+pipe_view.egg-info/
+setup.cfg

helios/pipeViewer/CMakeLists.txt

@@ -1,31 +1,108 @@
 cmake_minimum_required(VERSION 3.15)
 project (PipeViewer)
 
-find_package(PythonInterp 3.7 REQUIRED)
-find_package(PythonLibs 3.7 REQUIRED)
-
-set (CMAKE_CXX_STANDARD 17)
-set (CMAKE_CXX_STANDARD_REQUIRED ON)
+# Set up Sparta
+if(IS_DIRECTORY ${SPARTA_SEARCH_DIR})
+  set(CMAKE_MODULE_PATH "${SPARTA_SEARCH_DIR}/lib/cmake/sparta" ${CMAKE_MODULE_PATH})
+  find_package(Sparta REQUIRED)
+else()
+  set(SPARTA_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../sparta)
+  message (STATUS "Looking for a built version of sparta in the source tree at ${SPARTA_DIR}")
+  set(CMAKE_MODULE_PATH "${SPARTA_DIR}/cmake" ${CMAKE_MODULE_PATH})
+  set(SPARTA_INCLUDE_DIR ${SPARTA_DIR} ${SPARTA_DIR}/simdb)
+  set(SPARTA_LIBRARY ${SPARTA_DIR}/release ${SPARTA_DIR}/release/simdb)
+  include(${SPARTA_DIR}/cmake/FindSparta.cmake)
+endif()
 
-if(NOT ${PYTHON})
-    find_program(PYTHON 'python3')
+if(NOT SPARTA_FOUND)
+  message (FATAL_ERROR "Could not find Sparta. (${SPARTA_FOUND}) If needed, please provide the location where sparta is installed: -DSPARTA_SEARCH_DIR=<directory>")
 endif()
 
-set(SETUP_PY "${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/core/setup.py")
+# If we are in virtualenv or conda, that takes prio over any system python there might be
+set(Python3_FIND_VIRTUALENV FIRST)
+find_package(Python3 3.7 REQUIRED COMPONENTS Interpreter)
 
-set(PY_OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/build/core.so.built")
+# Look for wxWidgets
+find_package(wxWidgets REQUIRED core base)
+include(${wxWidgets_USE_FILE})
+
+# Populate list of include dirs:
+# Little helper script to find wxPython include path
+execute_process(
+  COMMAND ${Python3_EXECUTABLE} wxPythonInclude.py
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  OUTPUT_VARIABLE WX_PYTHON_INC
+  ERROR_VARIABLE WXCHECK_LOG
+  RESULT_VARIABLE WXCHECK_RESULT
+  OUTPUT_STRIP_TRAILING_WHITESPACE
+)
+if(NOT WXCHECK_RESULT EQUAL "0")
+  message(FATAL_ERROR "Could not find wxPython: ${WXCHECK_LOG}")
+endif()
+list(APPEND _INC_DIRS "$<TARGET_PROPERTY:SPARTA::sparta,INTERFACE_INCLUDE_DIRECTORIES>" ${WX_PYTHON_INC} ${wxWidgets_INCLUDE_DIRS})
 
-add_custom_command(OUTPUT "${PY_OUTPUT}"
-		   COMMAND ${CMAKE_COMMAND} -E env CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} LD=${CMAKE_LINKER} TARGETDIR=${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/core ${PYTHON} ${SETUP_PY} build_ext --inplace
-                   COMMAND ${CMAKE_COMMAND} -E touch ${PY_OUTPUT}
-                   DEPENDS ${SETUP_PY} "${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/core/src/core.pyx"
-                   )
-add_custom_target(pipe_view_py ALL DEPENDS "${PY_OUTPUT}")
+# Populate list of library dirs:
+# Set up the include and link dirs for python's setuptools using setup.cfg file
+# HDF5: need to extract dirs from paths to libs
+foreach(lib IN LISTS HDF5_LIBRARIES)
+  get_filename_component(d ${lib} DIRECTORY)
+  list(APPEND _LIB_DIRS ${d})
+endforeach()
+list(REMOVE_DUPLICATES _LIB_DIRS)
+# And we add sparta and simdb location
+# they are in the same folder so we look at libsparta only to find the right location
+get_property(_SPARTA_LIB TARGET SPARTA::libsparta PROPERTY IMPORTED_LOCATION)
+get_filename_component(SPARTA_LIBDIR ${_SPARTA_LIB} DIRECTORY)
+list(APPEND _LIB_DIRS ${SPARTA_LIBDIR}  ${wxWidgets_LIBRARY_DIRS})
 
-include_directories(${SPARTA_BASE} ${SPARTA_BASE}/simdb/include ./)
+# Those get pushed to setup.cfg file
+file(GENERATE
+  OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/setup.cfg
+  CONTENT
+"
+[build_ext]
+include_dirs=$<JOIN:${_INC_DIRS},:>
+library_dirs=$<JOIN:${_LIB_DIRS},:>
+")
 
-include(${SPARTA_BASE}/cmake/sparta-config.cmake)
+# Wx specific arguments get passed via LDFLAGS and CXXFLAGS in the command
+foreach(def IN LISTS wxWidgets_DEFINITIONS)
+  string(APPEND PYTHON_CFLAGS "-D${def} ")
+endforeach()
+string(JOIN " " PYTHON_LDFLAGS ${wxWidgets_LIBRARIES})
+
+add_custom_command(
+  OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/cython.stamp
+  COMMAND ${CMAKE_COMMAND} -E env CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} LD=${CMAKE_LINKER}  CFLAGS=${PYTHON_CFLAGS} LDFLAGS=${PYTHON_LDFLAGS} ${Python3_EXECUTABLE} setup.py build_ext --inplace VERBATIM
+  COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/cython.stamp
+  WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+  DEPENDS
+      ${CMAKE_CURRENT_SOURCE_DIR}/setup.py
+      ${CMAKE_CURRENT_SOURCE_DIR}/setup.cfg
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/core/src/common.pxd
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/core/src/core.pyx
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/core/src/helpers.h
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/logsearch/src/log_search.cpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/logsearch/src/logsearch.pyx
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/PipelineDataCallback.hpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/Reader.hpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/SimpleOutputInterface.hpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/TransactionDatabaseInterface.hpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/TransactionInterval.hpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/helpers.hpp
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/common.pxd
+      ${CMAKE_CURRENT_SOURCE_DIR}/pipe_view/transactiondb/src/transactiondb.pyx
+    ${CMAKE_CURRENT_SOURCE_DIR}/setup.cfg
+  COMMENT "Building pipeViwer and its dependencies"
+)
+add_custom_target(pipeView ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/cython.stamp)
 
-add_subdirectory(transactiondb)
 add_subdirectory(transactionsearch)
+
+install(CODE
+  "execute_process(
+    COMMAND ${CMAKE_COMMAND} -E env PYTHONUSERBASE=${CMAKE_INSTALL_PREFIX} ${Python3_EXECUTABLE} -m pip install .
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})"
+)
+
 add_subdirectory(argos_dumper)