From 961ae859c978fe6b7e85079cd196c1d3ae7b212f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Thu, 18 Jun 2020 13:34:01 +0100 Subject: [PATCH 01/65] Added makfile for docker. --- Makefile.docker | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 Makefile.docker diff --git a/Makefile.docker b/Makefile.docker new file mode 100644 index 0000000..07d0f7f --- /dev/null +++ b/Makefile.docker @@ -0,0 +1,21 @@ +# Docker repository +repository=iontichy +image=meanie3d +version=0.1.0 +work_dir=. +build_args= + +all: image push +.PHONY : all + +image: + docker build $(build_args) --tag ${repository}/${image}:${version} -f Dockerfile ${work_dir} + docker tag ${repository}/${image}:${version} ${repository}/${image}:latest + +image-nocache: + docker build --no-cache $(build_args) --tag ${repository}/${image}:${version} -f Dockerfile ${work_dir} + docker tag ${repository}/${image}:${version} ${repository}/${image}:latest + +push: + docker push ${repository}/${image}:${version} + docker push ${repository}/${image}:latest \ No newline at end of file From 240031d379a92d3811ed890ea5004f7da6379902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Thu, 18 Jun 2020 13:48:55 +0100 Subject: [PATCH 02/65] Adding preset for docker. --- CMakeLists.txt | 94 ++++++++++++++++++++++++++++---------------------- Dockerfile | 22 ++++++------ 2 files changed, 63 insertions(+), 53 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f4a1d3b..34ff8bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ SET(PACKAGE_VERSION "1.6.0") SET(PROJECT_URL "http://git.meteo.uni-bonn.de/projects/meanie3d") SET(PROJECT_LICENSE "MIT License") SET(PROJECT_AUTHOR "Juergen Simon") -SET(PROJECT_AUTHOR_EMAIL "juergen.simon@uni-bonn.de") +SET(PROJECT_AUTHOR_EMAIL "tachyonimpulse@gmail.com") # ------------------------------------- # Paths and build configuration @@ -54,7 +54,7 @@ INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/include) # ------------------------------------- # Options # ------------------------------------- - +OPTION(FOR_DOCKER "Build for dockerized version" OFF) OPTION(WITH_OPENMP "Enable/Disable OpenMP" ON) OPTION(WITH_TESTS "Enable/Disable building of tests" OFF) OPTION(WITH_VTK "Enable/Disable visualisation code (requires VTK)" OFF) @@ -69,53 +69,63 @@ ADD_SUBDIRECTORY(radolan) INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/radolan/include) # ------------------------------------- -# Multithreading / Parallelism Choices +# PRESETS # ------------------------------------- -SET(OPENMP_ENABLED ${WITH_OPENMP}) -IF (OPENMP_ENABLED) - MESSAGE(STATUS "OpenMP is enabled") - ADD_DEFINITIONS(-DWITH_OPENMP=1) -ELSE () - MESSAGE(STATUS "OpenMP is disabled") -ENDIF () - -# ------------------------------------- -# Visualization on/off -# ------------------------------------- +if (FOR_DOCKER) + MESSAGE("Setting preset options for building docker container") + SET(CMAKE_BUILD_TYPE "Release") + ADD_DEFINITIONS(-DWITH_OPENMP=1) +ELSE() + # ------------------------------------- + # Multithreading / Parallelism Choices + # ------------------------------------- + + SET(OPENMP_ENABLED ${WITH_OPENMP}) + IF (OPENMP_ENABLED) + MESSAGE(STATUS "OpenMP is enabled") + ADD_DEFINITIONS(-DWITH_OPENMP=1) + ELSE () + MESSAGE(STATUS "OpenMP is disabled") + ENDIF () -SET(VTK_ENABLED ${WITH_VTK}) -IF (VTK_ENABLED) - MESSAGE(STATUS "VTK is enabled") - ADD_DEFINITIONS(-DWITH_VTK=1) -ELSE () - MESSAGE(STATUS "VTK is disabled") -ENDIF () + # ------------------------------------- + # Visualization on/off + # ------------------------------------- -# ------------------------------------- -# Automated tests yes/no -# ------------------------------------- + SET(VTK_ENABLED ${WITH_VTK}) + IF (VTK_ENABLED) + MESSAGE(STATUS "VTK is enabled") + ADD_DEFINITIONS(-DWITH_VTK=1) + ELSE () + MESSAGE(STATUS "VTK is disabled") + ENDIF () -SET(TESTS_ENABLED ${WITH_TESTS}) -IF (TESTS_ENABLED) - MESSAGE(STATUS "Tests are enabled.") - ENABLE_TESTING() - ADD_SUBDIRECTORY(gtest-1.7.0) - INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0) - INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0/include) -ELSE () - MESSAGE(STATUS "Tests are disabled. Use -DWITH_TESTS=ON/OFF to switch tests on or off.") -ENDIF () + # ------------------------------------- + # Automated tests yes/no + # ------------------------------------- + + SET(TESTS_ENABLED ${WITH_TESTS}) + IF (TESTS_ENABLED) + MESSAGE(STATUS "Tests are enabled.") + ENABLE_TESTING() + ADD_SUBDIRECTORY(gtest-1.7.0) + INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0) + INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0/include) + ELSE () + MESSAGE(STATUS "Tests are disabled. Use -DWITH_TESTS=ON/OFF to switch tests on or off.") + ENDIF () -# ------------------------------------- -# OpenCV -# ------------------------------------- + # ------------------------------------- + # OpenCV + # ------------------------------------- -SET(OPENCV_ENABLED ${WITH_OPENCV}) -IF (OPENCV_ENABLED) - MESSAGE(STATUS "OpenCV is enabled.") - ADD_DEFINITIONS(-DWITH_OPENMP=1) -ENDIF () + SET(OPENCV_ENABLED ${WITH_OPENCV}) + IF (OPENCV_ENABLED) + MESSAGE(STATUS "OpenCV is enabled.") + ADD_DEFINITIONS(-DWITH_OPENCV=1) + ENDIF () +ENDIF() # ------------------------------------- # Compiler Flags diff --git a/Dockerfile b/Dockerfile index 37f824d..31f9db1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ python3 python3-pip \ cmake \ doxygen \ libflann1.9 libflann-dev \ -libboost-all-dev\ +libboost-all-dev \ blitz++ \ shapelib \ libhdf5-dev \ @@ -36,19 +36,19 @@ RUN rm -rf netcdf-cxx4-4.2.1 && rm v4.2.1.tar.gz #RUN rm -rf netCDF4* # Visualisation -RUN pip3 install Cython h5py netcdf4 -RUN apt-get -y --fix-missing install gnuplot vtk6 libvtk6-dev -RUN wget --quiet http://portal.nersc.gov/project/visit/releases/2.10.0/visit2_10_0.linux-x86_64-rhel6-wmesa.tar.gz -RUN wget --quiet http://portal.nersc.gov/project/visit/releases/2.10.0/visit-install2_10_0 -RUN chmod a+x visit-install2_10_0 -RUN echo "1" | ./visit-install2_10_0 2.10.0 linux-x86_64-rhel6-wmesa /usr/local/visit -ENV VISIT_EXECUTABLE=/usr/local/visit/bin/visit -RUN rm -rf visit* +# RUN pip3 install Cython h5py netcdf4 +# RUN apt-get -y --fix-missing install gnuplot vtk6 libvtk6-dev +# RUN wget --quiet http://portal.nersc.gov/project/visit/releases/2.10.0/visit2_10_0.linux-x86_64-rhel6-wmesa.tar.gz +# RUN wget --quiet http://portal.nersc.gov/project/visit/releases/2.10.0/visit-install2_10_0 +# RUN chmod a+x visit-install2_10_0 +# RUN echo "1" | ./visit-install2_10_0 2.10.0 linux-x86_64-rhel6-wmesa /usr/local/visit +# ENV VISIT_EXECUTABLE=/usr/local/visit/bin/visit +# RUN rm -rf visit* # Meanie3D -RUN git clone --recurse-submodules --depth=1 https://github.com/meteo-ubonn/meanie3D.git +RUN git clone --recurse-submodules --depth=1 https://github.com/meteo-ubonn/meanie3D.git#dockerize WORKDIR /meanie3D -RUN cmake -DWITH_OPENMP=1 -DWITH_VTK=1 -DCMAKE_BUILD_TYPE=Release . +RUN cmake -DFOR_DOCKER=1 . RUN make install # Cleanup From 2a807f1bbd8e1cdf6754381341b6c76facb500e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Thu, 18 Jun 2020 13:50:09 +0100 Subject: [PATCH 03/65] Repo URL fixed. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 31f9db1..284af0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ RUN rm -rf netcdf-cxx4-4.2.1 && rm v4.2.1.tar.gz # RUN rm -rf visit* # Meanie3D -RUN git clone --recurse-submodules --depth=1 https://github.com/meteo-ubonn/meanie3D.git#dockerize +RUN git clone --recurse-submodules --depth=1 https://github.com/JuergenSimon/meanie3D.git#dockerize WORKDIR /meanie3D RUN cmake -DFOR_DOCKER=1 . RUN make install From 11c97d977e4caf90c01ec7afda1d608cd20ff761 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Thu, 18 Jun 2020 15:20:01 +0100 Subject: [PATCH 04/65] Fix for HDF5 on Docker. --- Dockerfile | 8 ++++---- cmake_modules/FindHDF5.cmake | 19 ++++++++++++------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 284af0d..00702b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,6 @@ shapelib \ libhdf5-dev \ netcdf-bin libnetcdf-dev libnetcdf-c++4 python3-netcdf4 \ zlib1g zlib1g-dev - RUN pip3 install setuptools # Build NetCDF-CXX (always an extra bloody sausage with this package...) @@ -46,10 +45,11 @@ RUN rm -rf netcdf-cxx4-4.2.1 && rm v4.2.1.tar.gz # RUN rm -rf visit* # Meanie3D -RUN git clone --recurse-submodules --depth=1 https://github.com/JuergenSimon/meanie3D.git#dockerize +RUN git clone --recurse-submodules --depth=1 https://github.com/JuergenSimon/meanie3D WORKDIR /meanie3D -RUN cmake -DFOR_DOCKER=1 . -RUN make install +RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch --all +RUN git checkout --track origin/dockerize && git pull +RUN cmake -DFOR_DOCKER=1 . && make install # Cleanup WORKDIR / diff --git a/cmake_modules/FindHDF5.cmake b/cmake_modules/FindHDF5.cmake index f877c22..03d4c10 100644 --- a/cmake_modules/FindHDF5.cmake +++ b/cmake_modules/FindHDF5.cmake @@ -5,26 +5,31 @@ MESSAGE(STATUS "WITH_HDF5=${WITH_HDF5}") SET(HDF5_ROOT ${WITH_HDF5}) -IF (HDF5_ROOT) +IF (FOR_DOCKER) + MESSAGE(STATUS "Looking for HDF5 in Debian locations") + FIND_PATH(HDF5_INCLUDE_DIR hdf5.h PATHS /usr/include/hdf5/serial NO_DEFAULT_PATH) + FIND_LIBRARY(HDF5 NAMES hdf5 PATHS /usr/lib/x86_64-linux-gnu/hdf5/serial NO_DEFAULT_PATH) + FIND_LIBRARY(HDF5_HL NAMES hdf5_hl PATHS /usr/lib/x86_64-linux-gnu/hdf5/serial NO_DEFAULT_PATH) +ELSEIF (HDF5_ROOT) MESSAGE(STATUS "Looking for HDF5 in specified location: ${HDF5_ROOT}") SET(HDF5_INC_ROOT "${HDF5_ROOT}/include") SET(HDF5_LIB_ROOT "${HDF5_ROOT}/lib") FIND_PATH(HDF5_INCLUDE_DIR H5FDcore.h PATHS ${HDF5_INC_ROOT} NO_DEFAULT_PATH) FIND_LIBRARY(HDF5 NAMES hdf5 PATHS ${HDF5_LIB_ROOT} NO_DEFAULT_PATH) - FIND_LIBRARY(HDF5_H1 NAMES hdf5_hl PATHS ${HDF5_LIB_ROOT} NO_DEFAULT_PATH) + FIND_LIBRARY(HDF5_HL NAMES hdf5_hl PATHS ${HDF5_LIB_ROOT} NO_DEFAULT_PATH) ELSE() SET(HDF5_INC_ROOT "/usr/local/include /opt/local/include /usr/include") SET(HDF5_LIB_ROOT "/usr/local/lib /opt/local/lib /usr/lib") FIND_PATH(HDF5_INCLUDE_DIR H5FDcore.h PATHS ${HDF5_INC_ROOT}) FIND_LIBRARY(HDF5 NAMES hdf5 PATHS ${HDF5_LIB_ROOT}) - FIND_LIBRARY(HDF5_H1 NAMES hdf5_hl PATHS ${HDF5_LIB_ROOT}) + FIND_LIBRARY(HDF5_HL NAMES hdf5_hl PATHS ${HDF5_LIB_ROOT}) ENDIF() -IF (HDF5 AND HDF5_H1) - SET(HDF5_LIBRARIES ${HDF5} ${HDF5_H1}) -ELSE (HDF5 AND HDF5_H1) +IF (HDF5 AND HDF5_HL) + SET(HDF5_LIBRARIES ${HDF5} ${HDF5_HL}) +ELSE (HDF5 AND HDF5_HL) SET(HDF5_LIBRARIES "NOTFOUND") -ENDIF(HDF5 AND HDF5_H1) +ENDIF(HDF5 AND HDF5_HL) IF (HDF5_INCLUDE_DIR AND HDF5_LIBRARIES) SET(HDF5_FOUND TRUE) From 4efce10f230e6529802c3b12b1ccafb570c83e73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Thu, 18 Jun 2020 15:29:52 +0100 Subject: [PATCH 05/65] Trying to fix checkout out a branch in Dockerfile. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 00702b7..b8856b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ RUN rm -rf netcdf-cxx4-4.2.1 && rm v4.2.1.tar.gz RUN git clone --recurse-submodules --depth=1 https://github.com/JuergenSimon/meanie3D WORKDIR /meanie3D RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch --all -RUN git checkout --track origin/dockerize && git pull +RUN git checkout --track remotes/origin/dockerize && git pull RUN cmake -DFOR_DOCKER=1 . && make install # Cleanup From 80a43afb238a43bd0f0bb96fc9e6db8280a665db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Thu, 18 Jun 2020 16:03:03 +0100 Subject: [PATCH 06/65] Misc fixes. --- CMakeLists.txt | 42 ++++++++++++++++++------------------ cmake_modules/FindHDF5.cmake | 33 ++++++++++++---------------- python/README.txt | 2 +- python/setup.py | 2 +- 4 files changed, 37 insertions(+), 42 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 34ff8bd..f2f5021 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -201,27 +201,27 @@ ELSE (FLANN_FOUND) ENDIF (FLANN_FOUND) # OpenMP -IF (OPENMP_ENABLED) - FIND_PACKAGE(OpenMP REQUIRED) - IF (OPENMP_FOUND) - MESSAGE(STATUS "Found OpenMP (C_FLAGS=${OpenMP_C_FLAGS}, CXX_FLAGS=${OpenMP_CXX_FLAGS})") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") - ELSE (OPENMP_FOUND) - MESSAGE(ERROR "-- OpenMP is not enabled in your compiler") - ENDIF (OPENMP_FOUND) - - IF (NOT CMAKE_COMPILER_IS_GNUCXX) - FIND_PACKAGE(OpenMP_RT REQUIRED) - IF (OpenMP_RT_FOUND) - INCLUDE_DIRECTORIES(${OpenMP_RT_INCLUDE_DIRS}) - ELSE () - MESSAGE(FATAL_ERROR "OpenMP Runtime not found (https://www.openmprtl.org)") - ENDIF () - ENDIF () - -ENDIF (OPENMP_ENABLED) +# IF (OPENMP_ENABLED) +# FIND_PACKAGE(OpenMP REQUIRED) +# IF (OPENMP_FOUND) +# MESSAGE(STATUS "Found OpenMP (C_FLAGS=${OpenMP_C_FLAGS}, CXX_FLAGS=${OpenMP_CXX_FLAGS})") +# SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") +# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") +# SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") +# ELSE (OPENMP_FOUND) +# MESSAGE(ERROR "-- OpenMP is not enabled in your compiler") +# ENDIF (OPENMP_FOUND) +# +# IF (NOT CMAKE_COMPILER_IS_GNUCXX) +# FIND_PACKAGE(OpenMP_RT REQUIRED) +# IF (OpenMP_RT_FOUND) +# INCLUDE_DIRECTORIES(${OpenMP_RT_INCLUDE_DIRS}) +# ELSE () +# MESSAGE(FATAL_ERROR "OpenMP Runtime not found (https://www.openmprtl.org)") +# ENDIF () +# ENDIF () +# +# ENDIF (OPENMP_ENABLED) # OpenCV IF (OPENCV_ENABLED) diff --git a/cmake_modules/FindHDF5.cmake b/cmake_modules/FindHDF5.cmake index 03d4c10..07c47cf 100644 --- a/cmake_modules/FindHDF5.cmake +++ b/cmake_modules/FindHDF5.cmake @@ -1,28 +1,23 @@ -# CMAKE module for locating HDF5. -# Options: -DWITH_HDF5= -# Example: -DWITH_HDF5=/usr/local - -MESSAGE(STATUS "WITH_HDF5=${WITH_HDF5}") +# CMAKE module for locating HDF5. +# Options: -DWITH_HDF5= +# Example: -DWITH_HDF5=/usr/local SET(HDF5_ROOT ${WITH_HDF5}) IF (FOR_DOCKER) MESSAGE(STATUS "Looking for HDF5 in Debian locations") - FIND_PATH(HDF5_INCLUDE_DIR hdf5.h PATHS /usr/include/hdf5/serial NO_DEFAULT_PATH) - FIND_LIBRARY(HDF5 NAMES hdf5 PATHS /usr/lib/x86_64-linux-gnu/hdf5/serial NO_DEFAULT_PATH) - FIND_LIBRARY(HDF5_HL NAMES hdf5_hl PATHS /usr/lib/x86_64-linux-gnu/hdf5/serial NO_DEFAULT_PATH) + FIND_PATH(HDF5_INCLUDE_DIR hdf5.h PATHS /usr/include/hdf5/serial) + FIND_LIBRARY(HDF5 NAMES hdf5 PATHS /usr/lib/x86_64-linux-gnu/hdf5/serial) + FIND_LIBRARY(HDF5_HL NAMES hdf5_hl PATHS /usr/lib/x86_64-linux-gnu/hdf5/serial) ELSEIF (HDF5_ROOT) MESSAGE(STATUS "Looking for HDF5 in specified location: ${HDF5_ROOT}") - SET(HDF5_INC_ROOT "${HDF5_ROOT}/include") - SET(HDF5_LIB_ROOT "${HDF5_ROOT}/lib") - FIND_PATH(HDF5_INCLUDE_DIR H5FDcore.h PATHS ${HDF5_INC_ROOT} NO_DEFAULT_PATH) - FIND_LIBRARY(HDF5 NAMES hdf5 PATHS ${HDF5_LIB_ROOT} NO_DEFAULT_PATH) - FIND_LIBRARY(HDF5_HL NAMES hdf5_hl PATHS ${HDF5_LIB_ROOT} NO_DEFAULT_PATH) + FIND_PATH(HDF5_INCLUDE_DIR hdf5.h PATHS ${HDF5_ROOT}) + FIND_LIBRARY(HDF5 NAMES hdf5 PATHS ${HDF5_ROOT}) + FIND_LIBRARY(HDF5_HL NAMES hdf5_hl PATHS ${HDF5_ROOT}) ELSE() - SET(HDF5_INC_ROOT "/usr/local/include /opt/local/include /usr/include") - SET(HDF5_LIB_ROOT "/usr/local/lib /opt/local/lib /usr/lib") - FIND_PATH(HDF5_INCLUDE_DIR H5FDcore.h PATHS ${HDF5_INC_ROOT}) - FIND_LIBRARY(HDF5 NAMES hdf5 PATHS ${HDF5_LIB_ROOT}) - FIND_LIBRARY(HDF5_HL NAMES hdf5_hl PATHS ${HDF5_LIB_ROOT}) + MESSAGE(STATUS "Looking for HDF5 in standard locations") + FIND_PATH(HDF5_INCLUDE_DIR hdf5.h PATHS /usr/include /usr/local/include) + FIND_LIBRARY(HDF5 NAMES hdf5 PATHS /usr/lib /usr/local/lib) + FIND_LIBRARY(HDF5_HL NAMES hdf5_hl /usr/lib /usr/local/lib) ENDIF() IF (HDF5 AND HDF5_HL) @@ -43,4 +38,4 @@ ELSE (HDF5_FOUND) IF (HDF5_FIND_REQUIRED) MESSAGE(FATAL_ERROR "Could not find HDF5") ENDIF (HDF5_FIND_REQUIRED) -ENDIF (HDF5_FOUND) \ No newline at end of file +ENDIF (HDF5_FOUND) diff --git a/python/README.txt b/python/README.txt index ccc55a7..9f38a00 100644 --- a/python/README.txt +++ b/python/README.txt @@ -3,7 +3,7 @@ Project Name: meanie3D-python Version: 1.6.0 Url: http://git.meteo.uni-bonn.de/projects/meanie3d Author: Juergen Simon -Email: juergen.simon@uni-bonn.de +Email: tachyonimpulse@gmail.com License: MIT License #----------------------------------------------------------------------------# diff --git a/python/setup.py b/python/setup.py index ca590bc..a91344e 100644 --- a/python/setup.py +++ b/python/setup.py @@ -7,7 +7,7 @@ url='http://git.meteo.uni-bonn.de/projects/meanie3d', license='MIT License', author='Juergen Simon', - author_email='juergen.simon@uni-bonn.de', + author_email='tachyonimpulse@gmail.com', zip_safe = False, description='A python script for running the meanie3D clustering and tracking algorithms as well as processing and visualising the results.', entry_points = { From 981ad55ff26bfe385de80588f4f464619559a85f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Thu, 18 Jun 2020 16:03:30 +0100 Subject: [PATCH 07/65] Adding radolan submodule. --- radolan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/radolan b/radolan index 03e8107..2b954d9 160000 --- a/radolan +++ b/radolan @@ -1 +1 @@ -Subproject commit 03e8107a51515230f603f91ec8a031f9b7035749 +Subproject commit 2b954d9e0fa1f252fcbe739cb957c68c0a8aca08 From 0f278d80cee2d5e743a7671df40b54a7dc0e6181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sat, 20 Jun 2020 16:01:03 +0100 Subject: [PATCH 08/65] Overhaul documentation. OpenMP integration for modern compilers (built-in) Code formatting instructions (albeit not applied) CMAKE fixes --- .dockerignore | 1 + .gitignore | 1 + .gitmodules | 3 - CMakeLists.txt | 1228 ++++++++--------- README.md | 118 +- cmake_modules/FindFLANN.cmake | 1 + cmake_modules/Findlibradolan.cmake | 29 + include/meanie3D/clustering/cluster_op_impl.h | 83 +- include/meanie3D/clustering/conrad_cluster.h | 3 +- include/meanie3D/parallel.h | 8 +- python/meanie3D/__init__.py | 4 +- radolan | 1 - src/adaptors/Radolan.cpp | 3 +- src/executables/meanie3D-copydims.cpp | 55 +- src/executables/meanie3D-minmax.cpp | 44 +- .../meanie3D-parallax_correction.cpp | 1 - src/utils/.editorconfig | 16 + 17 files changed, 809 insertions(+), 790 deletions(-) create mode 100644 cmake_modules/Findlibradolan.cmake delete mode 160000 radolan create mode 100644 src/utils/.editorconfig diff --git a/.dockerignore b/.dockerignore index 3b7a2bd..363ba1e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .idea .vscode +build CMakeFiles cmake-build-debug Makefile.docker diff --git a/.gitignore b/.gitignore index a6b145f..0382436 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ netbeans-8.0 html cmake-build-debug .vscode +build diff --git a/.gitmodules b/.gitmodules index 6f537ce..5531394 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,6 +2,3 @@ [submodule "numericalrecipes"] path = numericalrecipes url = http://simon:mutter20panir@git.meteo.uni-bonn.de/git/numericalrecipes -[submodule "radolan"] - path = radolan - url = https://github.com/meteo-ubonn/radolan.git diff --git a/CMakeLists.txt b/CMakeLists.txt index f2f5021..c1b45d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,10 +35,10 @@ SET(PROJECT_AUTHOR_EMAIL "tachyonimpulse@gmail.com") SET(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake_modules") CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/config_h.in - ${PROJECT_SOURCE_DIR}/${PROJECT_NAME}/config.h) + ${PROJECT_SOURCE_DIR}/${PROJECT_NAME}/config.h) CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/include/meanie3D/version.h.in - ${PROJECT_SOURCE_DIR}/include/meanie3D/version.h) + ${PROJECT_SOURCE_DIR}/include/meanie3D/version.h) # Set default cmake build type to Debug # (None, Debug, Release, RelWithDebInfo or MinSizeRel) @@ -58,73 +58,108 @@ OPTION(FOR_DOCKER "Build for dockerized version" OFF) OPTION(WITH_OPENMP "Enable/Disable OpenMP" ON) OPTION(WITH_TESTS "Enable/Disable building of tests" OFF) OPTION(WITH_VTK "Enable/Disable visualisation code (requires VTK)" OFF) -OPTION(WITH_OPENCV "Legacy option. Leave at OFF" OFF) +OPTION(WITH_RADOLAN_UTILS "Compile with radolan adaptor (requires libradolan)" OFF) +OPTION(WITH_SATELLITE_UTILS "Compile with tools for satellite data" OFF) +OPTION(WITH_KONRAD_UTILS "Compile with binaries for KONRAD tracks" OFF) +OPTION(WITH_DOCS "Compile source code documentation" OFF) # ------------------------------------- # Submodules # ------------------------------------- ADD_SUBDIRECTORY(python) -ADD_SUBDIRECTORY(radolan) -INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/radolan/include) +IF (WITH_RADOLAN_UTILS) + ADD_SUBDIRECTORY(radolan) + INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/radolan/include) +ENDIF() # ------------------------------------- # PRESETS # ------------------------------------- if (FOR_DOCKER) - MESSAGE("Setting preset options for building docker container") - SET(CMAKE_BUILD_TYPE "Release") - ADD_DEFINITIONS(-DWITH_OPENMP=1) + MESSAGE("Setting preset options for building docker container") + SET(CMAKE_BUILD_TYPE "Release") + ADD_DEFINITIONS(-DWITH_OPENMP=1) + ADD_DEFINITIONS(-DWITH_RADOLAN=0) + ADD_DEFINITIONS(-DWITH_SHAPELIB=0) + ADD_DEFINITIONS(-DWITH_VTK=0) + ADD_DEFINITIONS(-DWITH_TESTS=0) + ADD_DEFINITIONS(-DWITH_KONRAD_UTILS=0) + ADD_DEFINITIONS(-DWITH_SATELLITE=0) ELSE() - # ------------------------------------- - # Multithreading / Parallelism Choices - # ------------------------------------- - - SET(OPENMP_ENABLED ${WITH_OPENMP}) - IF (OPENMP_ENABLED) - MESSAGE(STATUS "OpenMP is enabled") - ADD_DEFINITIONS(-DWITH_OPENMP=1) - ELSE () - MESSAGE(STATUS "OpenMP is disabled") - ENDIF () - - # ------------------------------------- - # Visualization on/off - # ------------------------------------- - - SET(VTK_ENABLED ${WITH_VTK}) - IF (VTK_ENABLED) - MESSAGE(STATUS "VTK is enabled") - ADD_DEFINITIONS(-DWITH_VTK=1) - ELSE () - MESSAGE(STATUS "VTK is disabled") - ENDIF () - - # ------------------------------------- - # Automated tests yes/no - # ------------------------------------- - - SET(TESTS_ENABLED ${WITH_TESTS}) - IF (TESTS_ENABLED) - MESSAGE(STATUS "Tests are enabled.") - ENABLE_TESTING() - ADD_SUBDIRECTORY(gtest-1.7.0) - INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0) - INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0/include) - ELSE () - MESSAGE(STATUS "Tests are disabled. Use -DWITH_TESTS=ON/OFF to switch tests on or off.") - ENDIF () - - # ------------------------------------- - # OpenCV - # ------------------------------------- - - SET(OPENCV_ENABLED ${WITH_OPENCV}) - IF (OPENCV_ENABLED) - MESSAGE(STATUS "OpenCV is enabled.") - ADD_DEFINITIONS(-DWITH_OPENCV=1) - ENDIF () + # ------------------------------------- + # Multithreading / Parallelism Choices + # ------------------------------------- + SET(OPENMP_ENABLED ${WITH_OPENMP}) + IF (OPENMP_ENABLED) + MESSAGE(STATUS "OpenMP is enabled") + ADD_DEFINITIONS(-DWITH_OPENMP=1) + ELSE () + MESSAGE(STATUS "OpenMP is disabled") + ENDIF () + + # ------------------------------------- + # Visualization on/off + # ------------------------------------- + SET(VTK_ENABLED ${WITH_VTK}) + IF (VTK_ENABLED) + MESSAGE(STATUS "VTK is enabled") + ADD_DEFINITIONS(-DWITH_VTK=1) + ELSE () + MESSAGE(STATUS "VTK is disabled") + ENDIF () + + # ------------------------------------- + # Automated tests yes/no + # ------------------------------------- + SET(TESTS_ENABLED ${WITH_TESTS}) + IF (TESTS_ENABLED) + MESSAGE(STATUS "Tests are enabled.") + ENABLE_TESTING() + ADD_SUBDIRECTORY(gtest-1.7.0) + INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0) + INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0/include) + ELSE () + MESSAGE(STATUS "Tests are disabled. Use -DWITH_TESTS=ON/OFF to switch tests on or off.") + ENDIF () + + # ------------------------------------- + # Radolan Adaptor + # ------------------------------------- + SET(RADOLAN_ENABLED ${WITH_RADOLAN_UTILS}) + IF (RADOLAN_ENABLED) + MESSAGE(STATUS "Compiling with radolan adaptor.") + ADD_DEFINITIONS(-DWITH_RADOLAN=1) + ENDIF () + + # ------------------------------------- + # KONRAD + # ------------------------------------- + SET(KONRAD_ENABLED ${WITH_KONRAD_UTILS}) + IF (KONRAD_ENABLED) + MESSAGE(STATUS "Adding binaries for working with KONRAD data") + ADD_DEFINITIONS(-DWITH_KONRAD_UTILS=1) + ENDIF () + + # ------------------------------------- + # SATELLITE + # ------------------------------------- + SET(SATELLITE_ENABLED ${WITH_SATELLITE_UTILS}) + IF (SATELLITE_ENABLED) + MESSAGE(STATUS "Adding binaries for working with EUMETSAT satellite data") + ADD_DEFINITIONS(-DWITH_SATELLITE=1) + ENDIF () + + # ------------------------------------- + # OpenCV (legacy) + # ------------------------------------- + + SET(OPENCV_ENABLED ${WITH_OPENCV}) + IF (OPENCV_ENABLED) + MESSAGE(STATUS "OpenCV is enabled.") + ADD_DEFINITIONS(-DWITH_OPENCV=1) + ENDIF () ENDIF() # ------------------------------------- @@ -135,31 +170,31 @@ MESSAGE(STATUS "CMAKE_C_COMPILER = ${CMAKE_C_COMPILER}") MESSAGE(STATUS "CMAKE_CXX_COMPILER = ${CMAKE_CXX_COMPILER}") MESSAGE(STATUS "Compiler ID = ${CMAKE_CXX_COMPILER_ID}") IF (CMAKE_COMPILER_IS_GNUCXX) - MESSAGE(STATUS "Compiler is GNU") + MESSAGE(STATUS "Compiler is GNU") ENDIF () IF (APPLE) - IF (CMAKE_CXX_COMPILER MATCHES ".*clang") - # Boost storage type problem - ADD_DEFINITIONS(-DBOOST_HAS_INT128=1) - ENDIF () + IF (CMAKE_CXX_COMPILER MATCHES ".*clang") + # Boost storage type problem + ADD_DEFINITIONS(-DBOOST_HAS_INT128=1) + ENDIF () ENDIF () IF ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") - SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -pipe -ggdb -msse3 -g") - SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -pipe -msse3") - SET(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL} -pipe -msse3") - SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -pipe -g -ggdb -msse3") - IF (WITH_LTO) - MESSAGE(STATUS "Link-Time Optimization switched on") - SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -flto") - SET(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL} -flto") - ENDIF () + SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -pipe -ggdb -msse3 -g") + SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -pipe -msse3") + SET(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL} -pipe -msse3") + SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -pipe -g -ggdb -msse3") + IF (WITH_LTO) + MESSAGE(STATUS "Link-Time Optimization switched on") + SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -flto") + SET(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL} -flto") + ENDIF () ELSEIF ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") - SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb -g") - SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -ggdb") - SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -msse3") - SET(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL} -msse3") + SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -ggdb -g") + SET(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -ggdb") + SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -msse3") + SET(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL} -msse3") ENDIF () # ------------------------------------- @@ -171,85 +206,39 @@ MESSAGE(STATUS "Resolving dependencies for Meanie3D") # netcdf FIND_PACKAGE(NETCDF) IF (NETCDF_FOUND) - INCLUDE_DIRECTORIES(${NETCDF_INCLUDE_DIR}) + INCLUDE_DIRECTORIES(${NETCDF_INCLUDE_DIR}) ELSE (NETCDF_FOUND) - MESSAGE(FATAL_ERROR "netcdf not found INCLUDE_DIR=${NETCDF_INCLUDE_DIR} LIBRARIES=${NETCDF_LIBRARIES}") + MESSAGE(FATAL_ERROR "netcdf not found INCLUDE_DIR=${NETCDF_INCLUDE_DIR} LIBRARIES=${NETCDF_LIBRARIES}") ENDIF (NETCDF_FOUND) # HDF5 FIND_PACKAGE(HDF5) IF (HDF5_FOUND) - INCLUDE_DIRECTORIES(${HDF5_INCLUDE_DIR}) + INCLUDE_DIRECTORIES(${HDF5_INCLUDE_DIR}) ELSE (HDF5_FOUND) - MESSAGE(FATAL_ERROR "HDF5 not found INCLUDE_DIR=${HDF5_INCLUDE_DIR} LIBRARIES=${HDF5_LIBRARIES}") + MESSAGE(FATAL_ERROR "HDF5 not found INCLUDE_DIR=${HDF5_INCLUDE_DIR} LIBRARIES=${HDF5_LIBRARIES}") ENDIF (HDF5_FOUND) # Boost FIND_PACKAGE(Boost COMPONENTS program_options thread filesystem system) IF (Boost_FOUND) - INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIR}) + INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIR}) ELSE (Boost_FOUND) - MESSAGE(FATAL_ERROR ${Boost_ERROR_REASON}) + MESSAGE(FATAL_ERROR ${Boost_ERROR_REASON}) ENDIF () # FLANN FIND_PACKAGE(FLANN) IF (FLANN_FOUND) - INCLUDE_DIRECTORIES(${FLANN_INCLUDE_DIR}) + INCLUDE_DIRECTORIES(${FLANN_INCLUDE_DIR}) ELSE (FLANN_FOUND) - MESSAGE(FATAL_ERROR "libflann not found (http://www.cs.ubc.ca/research/flann/)") + MESSAGE(FATAL_ERROR "libflann not found (http://www.cs.ubc.ca/research/flann/)") ENDIF (FLANN_FOUND) -# OpenMP -# IF (OPENMP_ENABLED) -# FIND_PACKAGE(OpenMP REQUIRED) -# IF (OPENMP_FOUND) -# MESSAGE(STATUS "Found OpenMP (C_FLAGS=${OpenMP_C_FLAGS}, CXX_FLAGS=${OpenMP_CXX_FLAGS})") -# SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") -# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") -# SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}") -# ELSE (OPENMP_FOUND) -# MESSAGE(ERROR "-- OpenMP is not enabled in your compiler") -# ENDIF (OPENMP_FOUND) -# -# IF (NOT CMAKE_COMPILER_IS_GNUCXX) -# FIND_PACKAGE(OpenMP_RT REQUIRED) -# IF (OpenMP_RT_FOUND) -# INCLUDE_DIRECTORIES(${OpenMP_RT_INCLUDE_DIRS}) -# ELSE () -# MESSAGE(FATAL_ERROR "OpenMP Runtime not found (https://www.openmprtl.org)") -# ENDIF () -# ENDIF () -# -# ENDIF (OPENMP_ENABLED) - -# OpenCV -IF (OPENCV_ENABLED) - FIND_PACKAGE(OpenCV) - IF (OpenCV_FOUND) - MESSAGE(STATUS "OpenCV FOUND") - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenCV_C_FLAGS}") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenCV_CXX_FLAGS}") - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenCV_EXE_LINKER_FLAGS}") - ENDIF (OpenCV_FOUND) -ENDIF () - -# VTK -IF (VTK_ENABLED) - FIND_PACKAGE(VTK 6.0 REQUIRED NO_MODULE) - IF (VTK_FOUND) - MESSAGE(STATUS "VTK found") - INCLUDE(${VTK_USE_FILE}) - INCLUDE_DIRECTORIES(${VTK_INCLUDE_DIRS}) - ELSE () - MESSAGE(FATAL_ERROR "VTK not found (http://www.vtk.org/VTK/resources/software.html)") - ENDIF () -ENDIF () - # Blitz++ FIND_PACKAGE(Blitz REQUIRED) IF (Blitz_FOUND) - INCLUDE_DIRECTORIES(${Blitz_INCLUDE_DIR}) + INCLUDE_DIRECTORIES(${Blitz_INCLUDE_DIR}) ELSE () MESSAGE(FATAL_ERROR "blitz++ not found") ENDIF () @@ -257,565 +246,554 @@ ENDIF () # Shapefile FIND_PACKAGE(SHP) IF (NOT SHP_FOUND) - ADD_DEFINITIONS(-DWITH_SHAPELIB=0) - MESSAGE(WARNING "shapelib not found (http://shapelib.maptools.org/). Disabling shapelib code.") + ADD_DEFINITIONS(-DWITH_SHAPELIB=0) + MESSAGE(STATUS "Shapelib disabled") ELSE () - ADD_DEFINITIONS(-DWITH_SHAPELIB=1) - MESSAGE(STATUS "Shapefile found") - INCLUDE_DIRECTORIES(${SHP_INCLUDE_DIR}) + ADD_DEFINITIONS(-DWITH_SHAPELIB=1) + MESSAGE(STATUS "Shapelib enabled") + INCLUDE_DIRECTORIES(${SHP_INCLUDE_DIR}) +ENDIF () + +# OpenMP +IF (OPENMP_DISABLED) + IF (CMAKE_COMPILER_IS_GNUCXX) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + ELSE(CMAKE_COMPILER_IS_GNUCXX) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") + ENDIF (CMAKE_COMPILER_IS_GNUCXX) +ENDIF (OPENMP_DISABLED) + +# VTK +IF (VTK_ENABLED) + FIND_PACKAGE(VTK 6.0 REQUIRED NO_MODULE) + IF (VTK_FOUND) + MESSAGE(STATUS "VTK found") + INCLUDE(${VTK_USE_FILE}) + INCLUDE_DIRECTORIES(${VTK_INCLUDE_DIRS}) + ELSE () + MESSAGE(FATAL_ERROR "VTK not found (http://www.vtk.org/VTK/resources/software.html)") + ENDIF () ENDIF () +FIND_PACKAGE(libradolan REQUIRED) +IF (libradolan_FOUND) + MESSAGE(STATUS "libradolan enabled") + INCLUDE_DIRECTORIES(${libradolan_INCLUDE_DIR}) +ELSE() + MESSAGE(FATAL_ERROR "libradolan not found") +ENDIF() + +IF (KONRAD_ENABLED) + MESSAGE(STATUS "KONRAD utilities enabled") +ELSE() + MESSAGE(STATUS "KONRAD utilities disabled") +ENDIF() + +IF (SATELLITE_ENABLED) + MESSAGE(STATUS "OASE satellite data utilities enabled") +ELSE() + MESSAGE(STATUS "OASE satellite data utilities disabled") +ENDIF() # ------------------------------------- # Binaries # ------------------------------------- # Library - ADD_LIBRARY(meanie3D SHARED - meanie3D/config.h - include/meanie3D/adaptors/Radolan.h - include/meanie3D/adaptors.h - include/meanie3D/array/array_index.h - include/meanie3D/array/array_index_impl.h - include/meanie3D/array/linear_index_mapping.h - include/meanie3D/array/multiarray.h - include/meanie3D/array/multiarray_blitz.h - include/meanie3D/array/multiarray_boost.h - include/meanie3D/array/multiarray_recursive.h - include/meanie3D/array.h - include/meanie3D/clustering/cluster.h - include/meanie3D/clustering/cluster_impl.h - include/meanie3D/clustering/cluster_list.h - include/meanie3D/clustering/cluster_list_impl.h - include/meanie3D/clustering/cluster_op.h - include/meanie3D/clustering/cluster_op_impl.h - include/meanie3D/clustering/cluster_utils.h - include/meanie3D/clustering/cluster_utils_impl.h - include/meanie3D/clustering/conrad_cluster.h - include/meanie3D/clustering/histogram.h - include/meanie3D/clustering/histogram_impl.h - include/meanie3D/clustering/detection.h - include/meanie3D/clustering/detection_impl.h - include/meanie3D/clustering/id.h - include/meanie3D/clustering.h - include/meanie3D/defines.h - include/meanie3D/exceptions/CFFileConversionException.h - include/meanie3D/exceptions.h - include/meanie3D/featurespace/coordinate_system.h - include/meanie3D/featurespace/coordinate_system_impl.h - include/meanie3D/featurespace/data_store.h - include/meanie3D/featurespace/featurespace.h - include/meanie3D/featurespace/featurespace_impl.h - include/meanie3D/featurespace/netcdf_data_store.h - include/meanie3D/featurespace/point.h - include/meanie3D/featurespace/point_default_factory.h - include/meanie3D/featurespace/point_factory.h - include/meanie3D/featurespace/point_impl.h - include/meanie3D/featurespace/timestamp.h - include/meanie3D/featurespace.h - include/meanie3D/filters/convection_filter.h - include/meanie3D/filters/convection_filter_impl.h - include/meanie3D/filters/filter.h - include/meanie3D/filters/scalespace_filter.h - include/meanie3D/filters/scalespace_filter_impl.h - include/meanie3D/filters/scalespace_kernel.h - include/meanie3D/filters/scalespace_kernel_impl.h - include/meanie3D/filters/replacement_filter.h - include/meanie3D/filters/replacement_filter_impl.h - include/meanie3D/filters/threshold_filter.h - include/meanie3D/filters/threshold_filter_impl.h - include/meanie3D/filters/weight_filter.h - include/meanie3D/filters/weight_filter_impl.h - include/meanie3D/filters.h - include/meanie3D/implementations.h - include/meanie3D/index/index.h - include/meanie3D/index/index_impl.h - include/meanie3D/index/index_whitening.h - include/meanie3D/index/indexed_flann.h - include/meanie3D/index/indexed_kdtree.h - include/meanie3D/index/linear.h - include/meanie3D/index/rectilinear_grid_index.h - include/meanie3D/index/search_parameters.h - include/meanie3D/index.h - include/meanie3D/kdtree/kdtree.h - include/meanie3D/meanie3D.h - include/meanie3D/namespaces.h - include/meanie3D/numericalrecipes/nrutil.h - include/meanie3D/numericalrecipes/numericalrecipes.h - include/meanie3D/operations/iterate_op.h - include/meanie3D/operations/iterate_op_impl.h - include/meanie3D/operations/kernels.h - include/meanie3D/operations/kernels_impl.h - include/meanie3D/operations/meanshift_op.h - include/meanie3D/operations/meanshift_op_impl.h - include/meanie3D/operations/operation.h - include/meanie3D/operations.h - include/meanie3D/parallel.h - include/meanie3D/tracking/track.h - include/meanie3D/tracking/track_cluster.h - include/meanie3D/tracking/tracking.h - include/meanie3D/tracking/tracking_commandline.h - include/meanie3D/tracking/tracking_impl.h - include/meanie3D/tracking.h - include/meanie3D/utils/array_utils.h - include/meanie3D/utils/cluster_index.h - include/meanie3D/utils/cluster_index_impl.h - include/meanie3D/utils/commandline.h - include/meanie3D/utils/file_utils.h - include/meanie3D/utils/gaussian_normal.h - include/meanie3D/utils/map_utils.h - include/meanie3D/utils/matrix.h - include/meanie3D/utils/matrix_impl.h - include/meanie3D/utils/netcdf_utils.h - include/meanie3D/utils/opencv_utils.h - include/meanie3D/utils/rand_utils.h - include/meanie3D/utils/set_utils.h - include/meanie3D/utils/time_utils.h - include/meanie3D/utils/vector_utils.h - include/meanie3D/utils/verbosity.h - include/meanie3D/utils/visit.h - include/meanie3D/utils/visit_impl.h - include/meanie3D/utils.h - include/meanie3D/weights/brightband_evidence.h - include/meanie3D/weights/ci_weights.h - include/meanie3D/weights/default_weights.h - include/meanie3D/weights/exp10_weight.h - include/meanie3D/weights/inverse_default.h - include/meanie3D/weights/oase_weights.h - include/meanie3D/weights/weight_function.h - include/meanie3D/weights/weight_function_factory.h - include/meanie3D/weights/weight_function_factory_impl.h - include/meanie3D/weights.h - src/adaptors/Radolan.cpp - src/exceptions/CFFileConversionException.cpp - src/kdtree/kdtree.c - src/numericalrecipes/betai.c - src/numericalrecipes/betacf.c - src/numericalrecipes/crank.c - src/numericalrecipes/erfcc.c - src/numericalrecipes/gammln.c - src/numericalrecipes/kendl1.c - src/numericalrecipes/nrutil.c - src/numericalrecipes/spear.c - src/numericalrecipes/sort2.c - src/utils/rand_utils.cpp - ) - -SOURCE_GROUP("meanie3d/adaptors" FILES - include/meanie3D/adaptors/Radolan.h - ) + meanie3D/config.h + include/meanie3D/adaptors/Radolan.h + include/meanie3D/adaptors.h + include/meanie3D/array/array_index.h + include/meanie3D/array/array_index_impl.h + include/meanie3D/array/linear_index_mapping.h + include/meanie3D/array/multiarray.h + include/meanie3D/array/multiarray_blitz.h + include/meanie3D/array/multiarray_boost.h + include/meanie3D/array/multiarray_recursive.h + include/meanie3D/array.h + include/meanie3D/clustering/cluster.h + include/meanie3D/clustering/cluster_impl.h + include/meanie3D/clustering/cluster_list.h + include/meanie3D/clustering/cluster_list_impl.h + include/meanie3D/clustering/cluster_op.h + include/meanie3D/clustering/cluster_op_impl.h + include/meanie3D/clustering/cluster_utils.h + include/meanie3D/clustering/cluster_utils_impl.h + include/meanie3D/clustering/conrad_cluster.h + include/meanie3D/clustering/histogram.h + include/meanie3D/clustering/histogram_impl.h + include/meanie3D/clustering/detection.h + include/meanie3D/clustering/detection_impl.h + include/meanie3D/clustering/id.h + include/meanie3D/clustering.h + include/meanie3D/defines.h + include/meanie3D/exceptions/CFFileConversionException.h + include/meanie3D/exceptions.h + include/meanie3D/featurespace/coordinate_system.h + include/meanie3D/featurespace/coordinate_system_impl.h + include/meanie3D/featurespace/data_store.h + include/meanie3D/featurespace/featurespace.h + include/meanie3D/featurespace/featurespace_impl.h + include/meanie3D/featurespace/netcdf_data_store.h + include/meanie3D/featurespace/point.h + include/meanie3D/featurespace/point_default_factory.h + include/meanie3D/featurespace/point_factory.h + include/meanie3D/featurespace/point_impl.h + include/meanie3D/featurespace/timestamp.h + include/meanie3D/featurespace.h + include/meanie3D/filters/convection_filter.h + include/meanie3D/filters/convection_filter_impl.h + include/meanie3D/filters/filter.h + include/meanie3D/filters/scalespace_filter.h + include/meanie3D/filters/scalespace_filter_impl.h + include/meanie3D/filters/scalespace_kernel.h + include/meanie3D/filters/scalespace_kernel_impl.h + include/meanie3D/filters/replacement_filter.h + include/meanie3D/filters/replacement_filter_impl.h + include/meanie3D/filters/threshold_filter.h + include/meanie3D/filters/threshold_filter_impl.h + include/meanie3D/filters/weight_filter.h + include/meanie3D/filters/weight_filter_impl.h + include/meanie3D/filters.h + include/meanie3D/implementations.h + include/meanie3D/index/index.h + include/meanie3D/index/index_impl.h + include/meanie3D/index/index_whitening.h + include/meanie3D/index/indexed_flann.h + include/meanie3D/index/indexed_kdtree.h + include/meanie3D/index/linear.h + include/meanie3D/index/rectilinear_grid_index.h + include/meanie3D/index/search_parameters.h + include/meanie3D/index.h + include/meanie3D/kdtree/kdtree.h + include/meanie3D/meanie3D.h + include/meanie3D/namespaces.h + include/meanie3D/numericalrecipes/nrutil.h + include/meanie3D/numericalrecipes/numericalrecipes.h + include/meanie3D/operations/iterate_op.h + include/meanie3D/operations/iterate_op_impl.h + include/meanie3D/operations/kernels.h + include/meanie3D/operations/kernels_impl.h + include/meanie3D/operations/meanshift_op.h + include/meanie3D/operations/meanshift_op_impl.h + include/meanie3D/operations/operation.h + include/meanie3D/operations.h + include/meanie3D/parallel.h + include/meanie3D/tracking/track.h + include/meanie3D/tracking/track_cluster.h + include/meanie3D/tracking/tracking.h + include/meanie3D/tracking/tracking_commandline.h + include/meanie3D/tracking/tracking_impl.h + include/meanie3D/tracking.h + include/meanie3D/utils/array_utils.h + include/meanie3D/utils/cluster_index.h + include/meanie3D/utils/cluster_index_impl.h + include/meanie3D/utils/commandline.h + include/meanie3D/utils/file_utils.h + include/meanie3D/utils/gaussian_normal.h + include/meanie3D/utils/map_utils.h + include/meanie3D/utils/matrix.h + include/meanie3D/utils/matrix_impl.h + include/meanie3D/utils/netcdf_utils.h + include/meanie3D/utils/opencv_utils.h + include/meanie3D/utils/rand_utils.h + include/meanie3D/utils/set_utils.h + include/meanie3D/utils/time_utils.h + include/meanie3D/utils/vector_utils.h + include/meanie3D/utils/verbosity.h + include/meanie3D/utils/visit.h + include/meanie3D/utils/visit_impl.h + include/meanie3D/utils.h + include/meanie3D/weights/brightband_evidence.h + include/meanie3D/weights/ci_weights.h + include/meanie3D/weights/default_weights.h + include/meanie3D/weights/exp10_weight.h + include/meanie3D/weights/inverse_default.h + include/meanie3D/weights/oase_weights.h + include/meanie3D/weights/weight_function.h + include/meanie3D/weights/weight_function_factory.h + include/meanie3D/weights/weight_function_factory_impl.h + include/meanie3D/weights.h + src/exceptions/CFFileConversionException.cpp + src/kdtree/kdtree.c + src/numericalrecipes/betai.c + src/numericalrecipes/betacf.c + src/numericalrecipes/crank.c + src/numericalrecipes/erfcc.c + src/numericalrecipes/gammln.c + src/numericalrecipes/kendl1.c + src/numericalrecipes/nrutil.c + src/numericalrecipes/spear.c + src/numericalrecipes/sort2.c + src/utils/rand_utils.cpp +) + +SOURCE_GROUP("meanie3d/adaptors" FILES + include/meanie3D/adaptors/Radolan.h +) SOURCE_GROUP("meanie3d/array" FILES - include/meanie3D/array/array_index.h - include/meanie3D/array/array_index_impl.h - include/meanie3D/array/linear_index_mapping.h - include/meanie3D/array/multiarray.h - include/meanie3D/array/multiarray_blitz.h - include/meanie3D/array/multiarray_boost.h - include/meanie3D/array/multiarray_recursive.h - ) + include/meanie3D/array/array_index.h + include/meanie3D/array/array_index_impl.h + include/meanie3D/array/linear_index_mapping.h + include/meanie3D/array/multiarray.h + include/meanie3D/array/multiarray_blitz.h + include/meanie3D/array/multiarray_boost.h + include/meanie3D/array/multiarray_recursive.h +) SOURCE_GROUP("meanie3d/clustering" FILES - include/meanie3D/clustering/cluster.h - include/meanie3D/clustering/cluster_impl.h - include/meanie3D/clustering/cluster_list.h - include/meanie3D/clustering/cluster_list_impl.h - include/meanie3D/clustering/cluster_op.h - include/meanie3D/clustering/cluster_op_impl.h - include/meanie3D/clustering/cluster_utils.h - include/meanie3D/clustering/cluster_utils_impl.h - include/meanie3D/clustering/conrad_cluster.h - include/meanie3D/clustering/histogram.h - include/meanie3D/clustering/histogram_impl.h - include/meanie3D/clustering/id.h - ) + include/meanie3D/clustering/cluster.h + include/meanie3D/clustering/cluster_impl.h + include/meanie3D/clustering/cluster_list.h + include/meanie3D/clustering/cluster_list_impl.h + include/meanie3D/clustering/cluster_op.h + include/meanie3D/clustering/cluster_op_impl.h + include/meanie3D/clustering/cluster_utils.h + include/meanie3D/clustering/cluster_utils_impl.h + include/meanie3D/clustering/conrad_cluster.h + include/meanie3D/clustering/histogram.h + include/meanie3D/clustering/histogram_impl.h + include/meanie3D/clustering/id.h +) SOURCE_GROUP("meanie3d/exceptions" FILES - include/meanie3D/exceptions/CFFileConversionException.h - ) + include/meanie3D/exceptions/CFFileConversionException.h +) SOURCE_GROUP("meanie3d/featurespace" FILES - include/meanie3D/featurespace/coordinate_system.h - include/meanie3D/featurespace/coordinate_system_impl.h - include/meanie3D/featurespace/data_store.h - include/meanie3D/featurespace/featurespace.h - include/meanie3D/featurespace/featurespace_impl.h - include/meanie3D/featurespace/netcdf_data_store.h - include/meanie3D/featurespace/point.h - include/meanie3D/featurespace/point_default_factory.h - include/meanie3D/featurespace/point_factory.h - include/meanie3D/featurespace/point_impl.h - include/meanie3D/featurespace/timestamp.h - ) + include/meanie3D/featurespace/coordinate_system.h + include/meanie3D/featurespace/coordinate_system_impl.h + include/meanie3D/featurespace/data_store.h + include/meanie3D/featurespace/featurespace.h + include/meanie3D/featurespace/featurespace_impl.h + include/meanie3D/featurespace/netcdf_data_store.h + include/meanie3D/featurespace/point.h + include/meanie3D/featurespace/point_default_factory.h + include/meanie3D/featurespace/point_factory.h + include/meanie3D/featurespace/point_impl.h + include/meanie3D/featurespace/timestamp.h +) SOURCE_GROUP("meanie3d/filters" FILES - include/meanie3D/filters/convection_filter.h - include/meanie3D/filters/convection_filter_impl.h - include/meanie3D/filters/filter.h - include/meanie3D/filters/scalespace_filter.h - include/meanie3D/filters/scalespace_filter_impl.h - include/meanie3D/filters/scalespace_kernel.h - include/meanie3D/filters/scalespace_kernel_impl.h - include/meanie3D/filters/replacement_filter.h - include/meanie3D/filters/replacement_filter_impl.h - include/meanie3D/filters/threshold_filter.h - include/meanie3D/filters/threshold_filter_impl.h - include/meanie3D/filters/weight_filter.h - include/meanie3D/filters/weight_filter_impl.h - ) + include/meanie3D/filters/convection_filter.h + include/meanie3D/filters/convection_filter_impl.h + include/meanie3D/filters/filter.h + include/meanie3D/filters/scalespace_filter.h + include/meanie3D/filters/scalespace_filter_impl.h + include/meanie3D/filters/scalespace_kernel.h + include/meanie3D/filters/scalespace_kernel_impl.h + include/meanie3D/filters/replacement_filter.h + include/meanie3D/filters/replacement_filter_impl.h + include/meanie3D/filters/threshold_filter.h + include/meanie3D/filters/threshold_filter_impl.h + include/meanie3D/filters/weight_filter.h + include/meanie3D/filters/weight_filter_impl.h +) SOURCE_GROUP("meanie3d/index" FILES - include/meanie3D/index/index.h - include/meanie3D/index/index_impl.h - include/meanie3D/index/index_whitening.h - include/meanie3D/index/indexed_flann.h - include/meanie3D/index/indexed_kdtree.h - include/meanie3D/index/linear.h - include/meanie3D/index/rectilinear_grid_index.h - include/meanie3D/index/search_parameters.h) + include/meanie3D/index/index.h + include/meanie3D/index/index_impl.h + include/meanie3D/index/index_whitening.h + include/meanie3D/index/indexed_flann.h + include/meanie3D/index/indexed_kdtree.h + include/meanie3D/index/linear.h + include/meanie3D/index/rectilinear_grid_index.h + include/meanie3D/index/search_parameters.h +) SOURCE_GROUP("meanie3d/operations" FILES - include/meanie3D/operations/iterate_op.h - include/meanie3D/operations/iterate_op_impl.h - include/meanie3D/operations/kernels.h - include/meanie3D/operations/kernels_impl.h - include/meanie3D/operations/meanshift_op.h - include/meanie3D/operations/meanshift_op_impl.h - include/meanie3D/operations/operation.h - ) + include/meanie3D/operations/iterate_op.h + include/meanie3D/operations/iterate_op_impl.h + include/meanie3D/operations/kernels.h + include/meanie3D/operations/kernels_impl.h + include/meanie3D/operations/meanshift_op.h + include/meanie3D/operations/meanshift_op_impl.h + include/meanie3D/operations/operation.h +) SOURCE_GROUP("meanie3d/tracking" FILES - include/meanie3D/tracking/track.h - include/meanie3D/tracking/track_cluster.h - include/meanie3D/tracking/tracking.h - include/meanie3D/tracking/tracking_impl.h - ) + include/meanie3D/tracking/track.h + include/meanie3D/tracking/track_cluster.h + include/meanie3D/tracking/tracking.h + include/meanie3D/tracking/tracking_impl.h +) SOURCE_GROUP("meanie3d/utils" FILES - include/meanie3D/utils/array_utils.h - include/meanie3D/utils/cluster_index.h - include/meanie3D/utils/cluster_index_impl.h - include/meanie3D/utils/file_utils.h - include/meanie3D/utils/gaussian_normal.h - include/meanie3D/utils/map_utils.h - include/meanie3D/utils/matrix.h - include/meanie3D/utils/matrix_impl.h - include/meanie3D/utils/netcdf_utils.h - include/meanie3D/utils/opencv_utils.h - include/meanie3D/utils/rand_utils.h - include/meanie3D/utils/set_utils.h - include/meanie3D/utils/time_utils.h - include/meanie3D/utils/vector_utils.h - include/meanie3D/utils/verbosity.h - include/meanie3D/utils/visit.h - include/meanie3D/utils/visit_impl.h - ) + include/meanie3D/utils/array_utils.h + include/meanie3D/utils/cluster_index.h + include/meanie3D/utils/cluster_index_impl.h + include/meanie3D/utils/file_utils.h + include/meanie3D/utils/gaussian_normal.h + include/meanie3D/utils/map_utils.h + include/meanie3D/utils/matrix.h + include/meanie3D/utils/matrix_impl.h + include/meanie3D/utils/netcdf_utils.h + include/meanie3D/utils/opencv_utils.h + include/meanie3D/utils/rand_utils.h + include/meanie3D/utils/set_utils.h + include/meanie3D/utils/time_utils.h + include/meanie3D/utils/vector_utils.h + include/meanie3D/utils/verbosity.h + include/meanie3D/utils/visit.h + include/meanie3D/utils/visit_impl.h +) SOURCE_GROUP("meanie3d/weights" FILES - include/meanie3D/weights/brightband_evidence.h - include/meanie3D/weights/ci_weights.h - include/meanie3D/weights/default_weights.h - include/meanie3D/weights/exp10_weight.h - include/meanie3D/weights/inverse_default.h - include/meanie3D/weights/oase_weights.h - include/meanie3D/weights/weight_function.h - ) + include/meanie3D/weights/brightband_evidence.h + include/meanie3D/weights/ci_weights.h + include/meanie3D/weights/default_weights.h + include/meanie3D/weights/exp10_weight.h + include/meanie3D/weights/inverse_default.h + include/meanie3D/weights/oase_weights.h + include/meanie3D/weights/weight_function.h +) # ------------------------------------------------------------- # # Library and binaries # ------------------------------------------------------------- # TARGET_LINK_LIBRARIES(meanie3D - radolan - ${Boost_LIBRARIES} - ${VTK_LIBRARIES} - ${Blitz_LIBRARY} - ${OpenCV_LIBRARIES} - ${OpenMP_RT_LIBRARIES}) -#SET_TARGET_PROPERTIES(meanie3D PROPERTIES LINKER_LANGUAGE CXX) - -# Conversion tool - -ADD_EXECUTABLE(meanie3D-radolan2cfm - src/executables/meanie3D-radolan2cfm.cpp - ) - -TARGET_LINK_LIBRARIES(meanie3D-radolan2cfm - meanie3D - radolan - ${Boost_LIBRARIES} - ${VTK_LIBRARIES} - ${HDF5_LIBRARIES} - ${NETCDF_LIBRARIES}) -SET_TARGET_PROPERTIES(meanie3D-radolan2cfm PROPERTIES LINKER_LANGUAGE CXX) - -# detection tool - -ADD_EXECUTABLE(meanie3D-detect - src/executables/meanie3D-detect.cpp) + ${libradolan_LIBRARIES} + ${Boost_LIBRARIES} + ${VTK_LIBRARIES} + ${Blitz_LIBRARY} + ${OpenCV_LIBRARIES} + ${OpenMP_RT_LIBRARIES}) +SET_TARGET_PROPERTIES(meanie3D PROPERTIES LINKER_LANGUAGE CXX) +# Detection tool + +ADD_EXECUTABLE(meanie3D-detect src/executables/meanie3D-detect.cpp) TARGET_LINK_LIBRARIES(meanie3D-detect - meanie3D - ${Boost_LIBRARIES} - ${VTK_LIBRARIES} - ${HDF5_LIBRARIES} - ${NETCDF_LIBRARIES} - ${OpenMP_RT_LIBRARIES}) + meanie3D + ${FLANN_LIBRARIES} + ${LZ4} + ${Boost_LIBRARIES} + ${VTK_LIBRARIES} + ${HDF5_LIBRARIES} + ${NETCDF_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-detect PROPERTIES LINKER_LANGUAGE CXX) # tracking tool -ADD_EXECUTABLE(meanie3D-track - meanie3D - src/executables/meanie3D-track.cpp - ) - +ADD_EXECUTABLE(meanie3D-track meanie3D src/executables/meanie3D-track.cpp) TARGET_LINK_LIBRARIES(meanie3D-track - meanie3D - ${Boost_LIBRARIES} - ${VTK_LIBRARIES} - ${HDF5_LIBRARIES} - ${NETCDF_LIBRARIES} - ${OpenMP_RT_LIBRARIES}) + meanie3D + ${Boost_LIBRARIES} + ${VTK_LIBRARIES} + ${HDF5_LIBRARIES} + ${NETCDF_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-track PROPERTIES LINKER_LANGUAGE CXX) # trackstats -ADD_EXECUTABLE(meanie3D-trackstats - meanie3D - src/executables/meanie3D-trackstats.cpp - ) - +ADD_EXECUTABLE(meanie3D-trackstats meanie3D src/executables/meanie3D-trackstats.cpp) TARGET_LINK_LIBRARIES(meanie3D-trackstats - meanie3D - ${VTK_LIBRARIES} - ${Boost_LIBRARIES} - ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES} - ${OpenMP_RT_LIBRARIES}) + meanie3D + ${VTK_LIBRARIES} + ${Boost_LIBRARIES} + ${NETCDF_LIBRARIES} + ${HDF5_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-trackstats PROPERTIES LINKER_LANGUAGE CXX) -# trackstats CONRAD - -ADD_EXECUTABLE(meanie3D-trackstats-conrad - meanie3D - src/executables/meanie3D-trackstats-conrad.cpp - ) - -TARGET_LINK_LIBRARIES(meanie3D-trackstats-conrad - meanie3D - ${VTK_LIBRARIES} - ${Boost_LIBRARIES} - ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES} - ${OpenMP_RT_LIBRARIES}) -SET_TARGET_PROPERTIES(meanie3D-trackstats-conrad PROPERTIES LINKER_LANGUAGE CXX) - # copydims -ADD_EXECUTABLE(meanie3D-copydims - meanie3D - src/executables/meanie3D-copydims.cpp - ) - +ADD_EXECUTABLE(meanie3D-copydims meanie3D src/executables/meanie3D-copydims.cpp) TARGET_LINK_LIBRARIES(meanie3D-copydims - meanie3D - ${Boost_LIBRARIES} - ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES}) + meanie3D + ${Boost_LIBRARIES} + ${NETCDF_LIBRARIES} + ${HDF5_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-copydims PROPERTIES LINKER_LANGUAGE CXX) -# Parallax correction for satellite - -ADD_EXECUTABLE(meanie3D-parallax_correction - src/executables/meanie3D-parallax_correction.cpp - ) +# valid_min/valid_max checker -TARGET_LINK_LIBRARIES(meanie3D-parallax_correction - meanie3D - ${Boost_LIBRARIES} - ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES}) -SET_TARGET_PROPERTIES(meanie3D-parallax_correction PROPERTIES LINKER_LANGUAGE CXX) +ADD_EXECUTABLE(meanie3D-minmax src/executables/meanie3D-minmax.cpp) +TARGET_LINK_LIBRARIES(meanie3D-minmax + meanie3D + ${Boost_LIBRARIES} + ${NETCDF_LIBRARIES} + ${HDF5_LIBRARIES}) +SET_TARGET_PROPERTIES(meanie3D-minmax PROPERTIES LINKER_LANGUAGE CXX) # map data -ADD_EXECUTABLE(meanie3D-mapdata - meanie3D - src/executables/meanie3D-mapdata.cpp - ) - +ADD_EXECUTABLE(meanie3D-mapdata meanie3D src/executables/meanie3D-mapdata.cpp) TARGET_LINK_LIBRARIES(meanie3D-mapdata - meanie3D - ${Boost_LIBRARIES} - ${VTK_LIBRARIES} - ${HDF5_LIBRARIES} - ${NETCDF_LIBRARIES}) + meanie3D + ${Boost_LIBRARIES} + ${VTK_LIBRARIES} + ${HDF5_LIBRARIES} + ${NETCDF_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-mapdata PROPERTIES LINKER_LANGUAGE CXX) # timestamp update -ADD_EXECUTABLE(meanie3D-timestamp - src/executables/meanie3D-timestamp.cpp - ) - +ADD_EXECUTABLE(meanie3D-timestamp src/executables/meanie3D-timestamp.cpp) TARGET_LINK_LIBRARIES(meanie3D-timestamp - meanie3D - ${Boost_LIBRARIES} - ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES}) + meanie3D + ${Boost_LIBRARIES} + ${NETCDF_LIBRARIES} + ${HDF5_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-timestamp PROPERTIES LINKER_LANGUAGE CXX) -# -# VTK-converter -# -IF (VTK_ENABLED) - ADD_EXECUTABLE(meanie3D-cfm2vtk - src/executables/meanie3D-cfm2vtk.cpp) - - TARGET_LINK_LIBRARIES(meanie3D-cfm2vtk - meanie3D - ${Boost_LIBRARIES} - ${VTK_LIBRARIES} - ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES} - ${OpenMP_RT_LIBRARIES}) - SET_TARGET_PROPERTIES(meanie3D-cfm2vtk PROPERTIES LINKER_LANGUAGE CXX) -ENDIF () - -# valid_min/valid_max checker - -ADD_EXECUTABLE(meanie3D-minmax - src/executables/meanie3D-minmax.cpp - ) +# Radolan conversion utility + +IF (RADOLAN_ENABLED) + ADD_EXECUTABLE(meanie3D-radolan2cfm src/executables/meanie3D-radolan2cfm.cpp) + TARGET_LINK_LIBRARIES(meanie3D-radolan2cfm + meanie3D + ${Boost_LIBRARIES} + ${VTK_LIBRARIES} + ${HDF5_LIBRARIES} + ${NETCDF_LIBRARIES}) + SET_TARGET_PROPERTIES(meanie3D-radolan2cfm PROPERTIES LINKER_LANGUAGE CXX) +ENDIF(RADOLAN_ENABLED) + +# trackstats KONRAD + +IF (WITH_KONRAD_UTILS) + ADD_EXECUTABLE(meanie3D-trackstats-conrad meanie3D src/executables/meanie3D-trackstats-conrad.cpp) + TARGET_LINK_LIBRARIES(meanie3D-trackstats-conrad + meanie3D + ${VTK_LIBRARIES} + ${Boost_LIBRARIES} + ${NETCDF_LIBRARIES} + ${HDF5_LIBRARIES} + ${OpenMP_RT_LIBRARIES}) + SET_TARGET_PROPERTIES(meanie3D-trackstats-conrad PROPERTIES LINKER_LANGUAGE CXX) +ENDIF() -TARGET_LINK_LIBRARIES(meanie3D-minmax - meanie3D - ${Boost_LIBRARIES} - ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES}) -SET_TARGET_PROPERTIES(meanie3D-minmax PROPERTIES LINKER_LANGUAGE CXX) +# Satellite data binaries -ADD_EXECUTABLE(meanie3D-satconv - src/executables/meanie3D-satconv.cpp - ) +IF (WITH_SATELLITE_UTILS) +# Parallax correction + ADD_EXECUTABLE(meanie3D-parallax_correction src/executables/meanie3D-parallax_correction.cpp) + TARGET_LINK_LIBRARIES(meanie3D-parallax_correction + meanie3D + ${Boost_LIBRARIES} + ${NETCDF_LIBRARIES} + ${HDF5_LIBRARIES}) + SET_TARGET_PROPERTIES(meanie3D-parallax_correction PROPERTIES LINKER_LANGUAGE CXX) +# satconv tool +ADD_EXECUTABLE(meanie3D-satconv src/executables/meanie3D-satconv.cpp) TARGET_LINK_LIBRARIES(meanie3D-satconv - meanie3D - ${Boost_LIBRARIES} - ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES}) + meanie3D + ${Boost_LIBRARIES} + ${NETCDF_LIBRARIES} + ${HDF5_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-satconv PROPERTIES LINKER_LANGUAGE CXX) +ENDIF() +# VTK-converter + +IF (VTK_ENABLED) + ADD_EXECUTABLE(meanie3D-cfm2vtk src/executables/meanie3D-cfm2vtk.cpp) + TARGET_LINK_LIBRARIES(meanie3D-cfm2vtk + meanie3D + ${Boost_LIBRARIES} + ${VTK_LIBRARIES} + ${NETCDF_LIBRARIES} + ${HDF5_LIBRARIES} + ${OpenMP_RT_LIBRARIES}) + SET_TARGET_PROPERTIES(meanie3D-cfm2vtk PROPERTIES LINKER_LANGUAGE CXX) +ENDIF () # ------------------------------------------------------------- # # Tests # ------------------------------------------------------------- # IF (TESTS_ENABLED) - - ADD_EXECUTABLE(m3D-test-kernel - test/kernels/testcases.h - test/kernels/test.cpp) - - TARGET_LINK_LIBRARIES(m3D-test-kernel - meanie3D - gtest - ${Boost_LIBRARIES}) - SET_TARGET_PROPERTIES(m3D-test-kernel PROPERTIES LINKER_LANGUAGE CXX) - - # Unit tests for vector utils - - ADD_EXECUTABLE(m3D-test-collections - test/collections/tests_arrayindex.h - test/collections/tests_map.h - test/collections/tests_multiarray.h - test/collections/tests_set.h - test/collections/tests_vector.h - test/collections/test.cpp) - - TARGET_LINK_LIBRARIES(m3D-test-collections - gtest - meanie3D - ${Boost_LIBRARIES}) - - SET_TARGET_PROPERTIES(m3D-test-collections PROPERTIES LINKER_LANGUAGE CXX) - - # Unit tests for class FeatureSpace - - ADD_EXECUTABLE(m3D-test-featurespace - test/settings.cpp - test/settings.h - test/testcase_base.h - test/testcase_base_impl.h - test/featurespace/circular_pattern.h - test/featurespace/circular_pattern_impl.h - test/featurespace/uniform.h - test/featurespace/uniform_impl.h - test/featurespace/weighed.h - test/featurespace/weighed_impl.h - test/featurespace/iteration.h - test/featurespace/iteration_impl.h - test/featurespace/testcases.h - test/featurespace/test.cpp) - - TARGET_LINK_LIBRARIES(m3D-test-featurespace - gtest - meanie3D - ${Boost_LIBRARIES} - ${NETCDF_LIBRARIES} - ${FLANN_LIBRARIES} - ${OpenMP_RT_LIBRARIES} - ${HDF5_LIBRARIES} - ${VTK_LIBRARIES} - ${Blitz_LIBRARY}) - - SET_TARGET_PROPERTIES(m3D-test-featurespace PROPERTIES LINKER_LANGUAGE CXX) - - # KDTree - - ADD_EXECUTABLE(m3D-test-kdtree - test/kdtree/test.c) - - TARGET_LINK_LIBRARIES(m3D-test-kdtree - meanie3D - ${Boost_LIBRARIES}) - SET_TARGET_PROPERTIES(m3D-test-kdtree PROPERTIES LINKER_LANGUAGE CXX) - - # Detection tests - - ADD_EXECUTABLE(m3D-test-detection - test/settings.cpp - test/settings.h - test/testcase_base.h - test/testcase_base_impl.h - test/detection/clustering.h - test/detection/clustering_impl.h - test/detection/detection_test.h - test/detection/test.cpp - test/detection/testcases.h - test/detection/variable_weighed.h - test/detection/variable_weighed_impl.h) - - TARGET_LINK_LIBRARIES(m3D-test-detection - gtest - meanie3D - ${Boost_LIBRARIES} - ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES} - ${OpenMP_RT_LIBRARIES} - ${VTK_LIBRARIES}) - SET_TARGET_PROPERTIES(m3D-test-detection PROPERTIES LINKER_LANGUAGE CXX) - - ADD_TEST(KernelTest m3D-test-kernel) - ADD_TEST(CollectionTest m3D-test-collections) - ADD_TEST(KDTreeTest m3D-test-kdtree) - ADD_TEST(MeanshiftTest m3D-test-featurespace) - ADD_TEST(ClusteringTest m3D-test-detection) - - ADD_CUSTOM_TARGET(check COMMAND make test) - + # Test Suite + ADD_EXECUTABLE(m3D-test-kernel test/kernels/testcases.h test/kernels/test.cpp) + TARGET_LINK_LIBRARIES(m3D-test-kernel + meanie3D + gtest + ${Boost_LIBRARIES}) + SET_TARGET_PROPERTIES(m3D-test-kernel PROPERTIES LINKER_LANGUAGE CXX) + + # Unit tests for vector utils + ADD_EXECUTABLE(m3D-test-collections + test/collections/tests_arrayindex.h + test/collections/tests_map.h + test/collections/tests_multiarray.h + test/collections/tests_set.h + test/collections/tests_vector.h + test/collections/test.cpp) + TARGET_LINK_LIBRARIES(m3D-test-collections + gtest + meanie3D + ${Boost_LIBRARIES}) + SET_TARGET_PROPERTIES(m3D-test-collections PROPERTIES LINKER_LANGUAGE CXX) + + # Unit tests for class FeatureSpace + ADD_EXECUTABLE(m3D-test-featurespace + test/settings.cpp + test/settings.h + test/testcase_base.h + test/testcase_base_impl.h + test/featurespace/circular_pattern.h + test/featurespace/circular_pattern_impl.h + test/featurespace/uniform.h + test/featurespace/uniform_impl.h + test/featurespace/weighed.h + test/featurespace/weighed_impl.h + test/featurespace/iteration.h + test/featurespace/iteration_impl.h + test/featurespace/testcases.h + test/featurespace/test.cpp) + TARGET_LINK_LIBRARIES(m3D-test-featurespace + gtest + meanie3D + ${Boost_LIBRARIES} + ${NETCDF_LIBRARIES} + ${FLANN_LIBRARIES} + ${OpenMP_RT_LIBRARIES} + ${HDF5_LIBRARIES} + ${VTK_LIBRARIES} + ${Blitz_LIBRARY}) + SET_TARGET_PROPERTIES(m3D-test-featurespace PROPERTIES LINKER_LANGUAGE CXX) + + # KDTree + ADD_EXECUTABLE(m3D-test-kdtree test/kdtree/test.c) + TARGET_LINK_LIBRARIES(m3D-test-kdtree meanie3D ${Boost_LIBRARIES}) + SET_TARGET_PROPERTIES(m3D-test-kdtree PROPERTIES LINKER_LANGUAGE CXX) + + # Detection tests + ADD_EXECUTABLE(m3D-test-detection + test/settings.cpp + test/settings.h + test/testcase_base.h + test/testcase_base_impl.h + test/detection/clustering.h + test/detection/clustering_impl.h + test/detection/detection_test.h + test/detection/test.cpp + test/detection/testcases.h + test/detection/variable_weighed.h + test/detection/variable_weighed_impl.h) + + TARGET_LINK_LIBRARIES(m3D-test-detection + gtest + meanie3D + ${Boost_LIBRARIES} + ${NETCDF_LIBRARIES} + ${HDF5_LIBRARIES} + ${OpenMP_RT_LIBRARIES} + ${VTK_LIBRARIES}) + SET_TARGET_PROPERTIES(m3D-test-detection PROPERTIES LINKER_LANGUAGE CXX) + + ADD_TEST(KernelTest m3D-test-kernel) + ADD_TEST(CollectionTest m3D-test-collections) + ADD_TEST(KDTreeTest m3D-test-kdtree) + ADD_TEST(MeanshiftTest m3D-test-featurespace) + ADD_TEST(ClusteringTest m3D-test-detection) + ADD_CUSTOM_TARGET(check COMMAND make test) ENDIF (TESTS_ENABLED) # ------------------------------------------------------------- # @@ -835,29 +813,37 @@ INSTALL(TARGETS meanie3D LIBRARY DESTINATION "/usr/local/lib") INSTALL(TARGETS meanie3D-detect RUNTIME DESTINATION "/usr/local/bin") INSTALL(TARGETS meanie3D-track RUNTIME DESTINATION "/usr/local/bin") INSTALL(TARGETS meanie3D-trackstats RUNTIME DESTINATION "/usr/local/bin") -INSTALL(TARGETS meanie3D-trackstats-conrad RUNTIME DESTINATION "/usr/local/bin") INSTALL(TARGETS meanie3D-timestamp RUNTIME DESTINATION "/usr/local/bin") -INSTALL(TARGETS meanie3D-parallax_correction RUNTIME DESTINATION "/usr/local/bin") INSTALL(TARGETS meanie3D-minmax RUNTIME DESTINATION "/usr/local/bin") -INSTALL(TARGETS meanie3D-satconv RUNTIME DESTINATION "/usr/local/bin") -INSTALL(TARGETS meanie3D-radolan2cfm RUNTIME DESTINATION "/usr/local/bin") +IF (WITH_SATELLITE_UTILS) + INSTALL(TARGETS meanie3D-satconv RUNTIME DESTINATION "/usr/local/bin") + INSTALL(TARGETS meanie3D-parallax_correction RUNTIME DESTINATION "/usr/local/bin") +ENDIF() +IF (WITH_KONRAD_UTILS) + INSTALL(TARGETS meanie3D-trackstats-conrad RUNTIME DESTINATION "/usr/local/bin") +ENDIF() +IF (WITH_RADOLAN_UTILS) + INSTALL(TARGETS meanie3D-radolan2cfm RUNTIME DESTINATION "/usr/local/bin") +ENDIF() IF (WITH_VTK) - INSTALL(TARGETS meanie3D-cfm2vtk RUNTIME DESTINATION "/usr/local/bin") + INSTALL(TARGETS meanie3D-cfm2vtk RUNTIME DESTINATION "/usr/local/bin") ENDIF (WITH_VTK) # ------------------------------------------------------------- # # Documentation # ------------------------------------------------------------- # - -SET(FULL_VERSION_FOR_DOXYGEN 0.1.0) -INCLUDE(FindDoxygen) -IF (DOXYGEN_EXECUTABLE) +IF (WITH_DOCS) + SET(FULL_VERSION_FOR_DOXYGEN 0.1.0) + INCLUDE(FindDoxygen) + IF (DOXYGEN_EXECUTABLE) IF (DOXYGEN_DOT_EXECUTABLE) - SET(UBIQ_DOXYGEN_HAVE_DOT "YES") + SET(UBIQ_DOXYGEN_HAVE_DOT "YES") ELSE (DOXYGEN_DOT_EXECUTABLE) - SET(UBIQ_DOXYGEN_HAVE_DOT "NO") + SET(UBIQ_DOXYGEN_HAVE_DOT "NO") ENDIF (DOXYGEN_DOT_EXECUTABLE) CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/doxy.in ${PROJECT_BINARY_DIR}/Doxyfile) FILE(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/html/radolan/doxygen) ADD_CUSTOM_TARGET(meanie3D-docs COMMAND ${DOXYGEN_EXECUTABLE} ${PROJECT_BINARY_DIR}/Doxyfile) -ENDIF (DOXYGEN_EXECUTABLE) + MESSAGE(STATUS "Doxygen source code documentation will be available") + ENDIF (DOXYGEN_EXECUTABLE) +ENDIF() \ No newline at end of file diff --git a/README.md b/README.md index 0221168..85ea58d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ # meanie3D -[//]: # {#mainpage} This project provides a very generic implementation of mean-shift clustering on NetCDF data sets. The data sets need to follow the "CF-Metadata":http://cfconventions.org. The mean-shift algorithm @@ -26,14 +25,13 @@ Meanie3D comes under [[MIT license]]. ## Version -The latest stable version is v1.6.0. Versions are tagged. +The latest stable version is v1.6.1. Versions are tagged. ## Build Instructions ### Dependencies Meanie3D comes with a number of dependencies that need to be installed prior to attempting installation: - * Boost 1.56 or better (http://www.boost.org) * FLANN 1.8.0 or better (http://www.cs.ubc.ca/research/flann/) * Blitz++ (http://sourceforge.net/projects/blitz/) @@ -43,12 +41,11 @@ Meanie3D comes with a number of dependencies that need to be installed prior to * Python 2.5 or better (https://www.python.org) * numpy (try running @pip install numpy@ or download and install from http://www.numpy.org) * NetCDF4-python (try running @pip install netCDF4@ or download and install from http://unidata.github.io/netcdf4-python) +* libradolan (http://meteo-ubonn.github.io/radolan/) Optionally, the following libraries may be used, if they are switched on with the appropriate flags: - -* Intel's Thread Building Blocks (-DWITH_TBB=1) version 4.3 (https://www.threadingbuildingblocks.org). This is mutually exclusive with OpenMP (-DWITH_OPENMP=1). * VTK 6.0 or better (http://www.vtk.org/VTK/resources/software.html) -* OpenCV 2.4.x or better (http://opencv.org/downloads.html) +* doxygen (https://www.doxygen.nl/download.html) ### Compiler Prerequisites @@ -64,32 +61,24 @@ flags -DWITH_OPENMP=0 to your cmake call. Meanie3D uses CMAKE to generate makefiles. You can use CMAKE's abilities to generate IDE files if you prefer. Start by cloning the master branch (for an up-to date but possibly unstable version) or one of the stable releases. -
-git clone https://github.com/meteo-ubonn/meanie3D.git
-cd meanie3D
-git submodule init
-git submodule update radolan
-
+ git clone https://github.com/meteo-ubonn/meanie3D.git + +TODO: revise handling of map data If you want to download the OASE topology and mapdata file for visualisation, you can obtain this file by adding: -
-git clone http://git.meteo.uni-bonn.de/git/oase-mapdata
-
+ + git clone http://git.meteo.uni-bonn.de/git/oase-mapdata Create a build directory: -
-cd ..
-mkdir meanie3D-make
-cd meanie3D-make
-cmake ../meanie3D
-
+ mkdir meanie3D-make + cd meanie3D-make + cmake ../meanie3D +### Available build types In order to switch optimizations on, tell cmake to use the release build type: -
-cmake -DCMAKE_BUILD_TYPE=Release ../meanie3D
-
+ cmake -DCMAKE_BUILD_TYPE=Release ../meanie3D Note that there have been some problems on Linux with aggressive optimization and NetCDF. Your mileage may vary. You should try the release build in any event, since it speeds up performance a lot. If you observe unexpected @@ -97,44 +86,61 @@ problems in reading/writing NetCDF files, you may have fallen victim to the prob (leave the -DCMAKE_BUILD_TYPE=Release). Once all dependencies are successfully resolved, install the product by calling the following: -
-make install
-
- -There are a number of flags you can use to customize your installation: - -### VTK/ visualization code - -Because of the heavyweight of the VTK package, the visualization code is disabled by default. If you have VTK6 -installed, you can enable visualizations by adding the following flag to your cmake call: -
--D WITH_VTK=YES
-
- -This will result in some binaries and command line options for visualization to become available to you. - -### Add tests - + make install + +There are a number of options to customize your installation: + +### -DWITH_VTK=YES +Because of the large footprint of the VTK package, the visualization code is disabled by default. While +visualization is not necessary to run the algorithm, it can be useful to develop your parameters to +have visual queues as to what is happening. Setting this flag will result in the following changes: +* The `meanie3D-cfm2vtk` binary will be compiled. This tool can visualize netCDF files in Visit/VTK +* In several places in the code, visualizable output for intermediary steps becomes available: + * Cluster boundaries/outlines + * Cluster 'modes' (algorithmic center of a cluster) + * Cluster geometrical centers + * Weight function used in detection + * Mean-shift vectors + * Visualization of search window for mean shift. + * Cluster weight function response + * Individual variables of the original netCDF (by cluster) + * Cluster tracks + +*Important Notes*: Visualization is switched off in the Docker version. The visualiation code uses +libradolan. If you do switch this on, you will be required to install libradolan as well. + +### -DWITH_OPENMP=YES +In order to speed the process up, meanie3D uses OpenMP to parallelize it's computation. This option +is switched on by default. + +### -D WITH_TESTS=YES Meanie3D has a number of regression tests, that cover the core algorithms and collection classes. This will become important to you if you should decide to work on the core algorithms yourself. The tests are a good -method of making sure you haven't broken anything critical. In order to enable unit tests, use the flag: -
--D WITH_TESTS=YES
-
- -The unit tests can then be run by calling -
-make test
-
- -### Source code level documentation +method of making sure you haven't broken anything critical. The unit tests can then be run by calling + + make test +### -DWITH_DOCS=YES In order to start developing your own Meanie3D code, it might be useful to have API documentation of the various classes in the project. If you have doxygen installed, you can call the following make command to create a browsable HTML documentation in doc/html (open the file index.html). -
-make docs
-
+ + make docs + +### -DWITH_RADOLAN_UTILS=YES +This will result in compilation of the `meanie3D-radolan2cfm` utility, which converts files in RADOLAN +format to a cf-metadata compliant netCDF file, which then can be used to run the tracking. + +### -DWITH_SATELLITE_UTILS=YES +The package comes with binaries to perform some conversion on satellite data. Those binaries were +provided in the context of research work for the OASE project. The following binaries will be provided +if this flag is set: +* `meanie3D-satconv` - Converts spectral radiance to equivalent brightness temperature or vice versa +* `meanie3D-parallax_correction` - Applies parallax correction to mseviri satellite data in OASE composite files. + +### -DWITH_KONRAD_UTILS=YES +The package comes with a tool `meanie3D-trackstats-conrad` which analyses KONRAD tracks in a way that makes +the data comparable to meanie3D data. This is a specialized tool developed in the context of the OASE project. ## Frequently Asked Quesions diff --git a/cmake_modules/FindFLANN.cmake b/cmake_modules/FindFLANN.cmake index 64419fe..b36ea3d 100644 --- a/cmake_modules/FindFLANN.cmake +++ b/cmake_modules/FindFLANN.cmake @@ -1,5 +1,6 @@ FIND_PATH(FLANN_INCLUDE_DIR flann PATHS /usr/include /usr/local/include/ opt/local/include) FIND_LIBRARY(FLANN NAMES flann PATHS /usr/lib /usr/local/lib /opt/local/lib) +FIND_LIBRARY(LZ4 NAMES lz4 PATHS /usr/lib /usr/local/lib /opt/local/lib) IF (FLANN_INCLUDE_DIR AND FLANN) SET(FLANN_LIBRARIES ${FLANN}) diff --git a/cmake_modules/Findlibradolan.cmake b/cmake_modules/Findlibradolan.cmake new file mode 100644 index 0000000..084a0e9 --- /dev/null +++ b/cmake_modules/Findlibradolan.cmake @@ -0,0 +1,29 @@ +# +# - Find Radolan library +# +# The libradolan library is required if the adaptor for +# radolan data should be created. +# +# Output Variables: +# libradolan_INCLUDE_DIR where to find radolan.h +# libradolan_LIBRARIES location of libradolan.so/.dylib +# libradolan_FOUND flag indicating if both include and library have been located + +FIND_PATH(libradolan_INCLUDE_DIR radolan PATHS /usr/include /usr/local/include /opt/local/include) +FIND_LIBRARY(libradolan_LIBRARY NAMES radolan) + +IF (libradolan_INCLUDE_DIR AND libradolan_LIBRARY) + SET(libradolan_FOUND TRUE) +ENDIF (libradolan_INCLUDE_DIR AND libradolan_LIBRARY) + +# handle the QUIETLY and REQUIRED arguments and set libradolan_FOUND to TRUE if +# all listed variables are TRUE +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(libradolan DEFAULT_MSG libradolan_LIBRARY libradolan_INCLUDE_DIR) + +IF(libradolan_FOUND) + SET(libradolan_LIBRARIES ${libradolan_LIBRARY} ) + SET(libradolan_INCLUDE_DIRS ${libradolan_INCLUDE_DIR} ) +ENDIF(libradolan_FOUND) + +MARK_AS_ADVANCED(libradolan_INCLUDE_DIR libradolan_LIBRARY) \ No newline at end of file diff --git a/include/meanie3D/clustering/cluster_op_impl.h b/include/meanie3D/clustering/cluster_op_impl.h index 952d488..758b62c 100644 --- a/include/meanie3D/clustering/cluster_op_impl.h +++ b/include/meanie3D/clustering/cluster_op_impl.h @@ -33,7 +33,8 @@ #include -namespace m3D { +namespace m3D +{ #if WRITE_MODES static size_t s_pass_counter = 0; @@ -63,57 +64,67 @@ namespace m3D { #pragma mark - #pragma mark Clustering Code - template + template ClusterList * - ClusterOperation::cluster() { + ClusterOperation::cluster() + { using namespace m3D::utils::vectors; const CoordinateSystem *cs = m_context.coord_system; vector resolution; - if (m_context.search_params->search_type() == SearchTypeRange) { - RangeSearchParams *p = (RangeSearchParams *) m_context.search_params; + if (m_context.search_params->search_type() == SearchTypeRange) + { + RangeSearchParams *p = (RangeSearchParams *)m_context.search_params; // Physical grid resolution in the // spatial range resolution = cs->resolution(); - resolution = ((T) 4.0) * resolution; + resolution = ((T)4.0) * resolution; // Supplement with bandwidth values for // the value range - for (size_t i = resolution.size(); i < p->bandwidth.size(); i++) { + for (size_t i = resolution.size(); i < p->bandwidth.size(); i++) + { resolution.push_back(p->bandwidth[i]); } - } else { - KNNSearchParams *p = (KNNSearchParams *) m_context.search_params; + } + else + { + KNNSearchParams *p = (KNNSearchParams *)m_context.search_params; resolution = p->resolution; } - if (m_context.show_progress) { - cout << endl << "Creating meanshift vector graph ..."; + if (m_context.show_progress) + { + cout << endl + << "Creating meanshift vector graph ..."; start_timer(); m_progress_bar = new boost::progress_display(this->feature_space->size()); } // Create an empty cluster list ClusterList *cluster_list = new ClusterList( - m_params.filename, - m_params.variables, - m_params.dimensions, - m_params.dimension_variables, - m_params.time_index); + m_params.filename, + m_params.variables, + m_params.dimensions, + m_params.dimension_variables, + m_params.time_index); // Guard against empty feature-space - if (this->feature_space->points.size() == 0) { + if (this->feature_space->points.size() == 0) + { cout << "Feature space is empty" << endl; return cluster_list; } - MeanshiftOperation meanshiftOperator(this->feature_space, this->point_index); + MeanshiftOperation meanshiftOperator(this->feature_space, this->point_index); meanshiftOperator.prime_index(m_context.search_params); #if WITH_OPENMP #pragma omp parallel for schedule(dynamic) #endif - for (size_t index = 0; index < this->feature_space->size(); index++) { - if (m_context.show_progress) { + for (size_t index = 0; index < this->feature_space->size(); index++) + { + if (m_context.show_progress) + { #if WITH_OPENMP #pragma omp critical #endif @@ -132,7 +143,8 @@ namespace m3D { x->gridded_shift = this->feature_space->coordinate_system->to_gridpoints(spatial_shift); } - if (m_context.show_progress) { + if (m_context.show_progress) + { cout << "done. (" << stop_timer() << "s)" << endl; delete m_progress_bar; m_progress_bar = NULL; @@ -140,10 +152,10 @@ namespace m3D { // Analyse the graph and create clusters cluster_list->aggregate_cluster_graph( - this->feature_space, - m_context.weight_function, - m_params.coalesceWithStrongestNeighbour, - m_context.show_progress); + this->feature_space, + m_context.weight_function, + m_params.coalesceWithStrongestNeighbour, + m_context.show_progress); // Provide fresh ids right away m3D::uuid_t uuid = 0; @@ -151,13 +163,13 @@ namespace m3D { m3D::id_t id = 0; ClusterUtils::provideIds(cluster_list, id); -// cout << "Cluster list after aggregation:" << endl; -// cluster_list.print(); + // cout << "Cluster list after aggregation:" << endl; + // cluster_list.print(); // Replace points with original data () ClusterUtils::replace_points_from_datastore(cluster_list, m_context.data_store); -// cout << "Cluster list after filtering points:" << endl; -// cluster_list.print(); + // cout << "Cluster list after filtering points:" << endl; + // cluster_list.print(); // Find margin points (#325) ClusterUtils::obtain_margin_flag(cluster_list, this->feature_space); @@ -170,16 +182,19 @@ namespace m3D { #if WITH_VTK size_t min_size = std::numeric_limits::max(); size_t max_size = std::numeric_limits::min(); - for (size_t i = 0; i < cluster_list.size(); i++) { - if (cluster_list[i]->size() < min_size) { + for (size_t i = 0; i < cluster_list.size(); i++) + { + if (cluster_list[i]->size() < min_size) + { min_size = cluster_list[i]->size(); } - if (cluster_list[i]->size() > max_size) { + if (cluster_list[i]->size() > max_size) + { max_size = cluster_list[i]->size(); } } - NetCDFDataStore *ds = (NetCDFDataStore *) this->feature_space->data_store(); + NetCDFDataStore *ds = (NetCDFDataStore *)this->feature_space->data_store(); std::string fn = ds->filename() + "-modes-" + boost::lexical_cast(pass_counter()) + ".vtk"; VisitUtils::write_cluster_modes_vtk(fn, cluster_list.clusters); @@ -190,6 +205,6 @@ namespace m3D { #endif return cluster_list; } -} +} // namespace m3D #endif diff --git a/include/meanie3D/clustering/conrad_cluster.h b/include/meanie3D/clustering/conrad_cluster.h index b9bb12d..b8485cf 100644 --- a/include/meanie3D/clustering/conrad_cluster.h +++ b/include/meanie3D/clustering/conrad_cluster.h @@ -21,7 +21,7 @@ * SOFTWARE. */ - +#ifdef WITH_KONRAD_UTILS #ifndef M3D_CONRAD_CLUSTER_H #define M3D_CONRAD_CLUSTER_H @@ -241,3 +241,4 @@ namespace m3D { } #endif +#endif // KONRAD \ No newline at end of file diff --git a/include/meanie3D/parallel.h b/include/meanie3D/parallel.h index 8777130..8072352 100644 --- a/include/meanie3D/parallel.h +++ b/include/meanie3D/parallel.h @@ -21,12 +21,8 @@ * SOFTWARE. */ -#ifndef M3D_PARALL_H -#define M3D_PARALL_H - -#if WITH_OPENMP -#include -#endif +#ifndef M3D_PARALLEL_H +#define M3D_PARALLEL_H #define PROVIDE_MUTEX ! #define PROVIDE_THREADSAFETY ! diff --git a/python/meanie3D/__init__.py b/python/meanie3D/__init__.py index 4931b5a..ee2c0c4 100644 --- a/python/meanie3D/__init__.py +++ b/python/meanie3D/__init__.py @@ -1,5 +1,5 @@ __author__ = 'Juergen Simon' -__email__ = 'juergen.simon@uni-bonn.de' +__email__ = 'tachyonimpulse@gmail.com' __version__ = '1.6.0' __url__ = 'http://git.meteo.uni-bonn.de/projects/meanie3d' __all__ = ['app', 'visualisation', 'resources'] @@ -25,7 +25,7 @@ def appendSystemPythonPath(): ''' Returns the system's python path to import outside modules. ''' - system_python_path = ":/Users/simon/anaconda/lib/python35.zip:/Users/simon/anaconda/lib/python3.5:/Users/simon/anaconda/lib/python3.5/plat-darwin:/Users/simon/anaconda/lib/python3.5/lib-dynload:/Users/simon/anaconda/lib/python3.5/site-packages:/Users/simon/anaconda/lib/python3.5/site-packages/Sphinx-1.4.1-py3.5.egg:/Users/simon/anaconda/lib/python3.5/site-packages/aeosa:/Users/simon/anaconda/lib/python3.5/site-packages/setuptools-23.0.0-py3.5.egg".strip() + system_python_path = ":/Library/Frameworks/SQLite3.framework/Versions/C/Python/2.7:/Library/Frameworks/GEOS.framework/Versions/3/Python/2.7:/Library/Python/2.7/site-packages/numpy-override:/Library/Python/2.7/site-packages/matplotlib-override:/Library/Frameworks/GDAL.framework/Versions/1.11/Python/2.7/site-packages:/Library/Python/2.7/site-packages/pip-1.5.2-py2.7.egg:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python27.zip:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-darwin:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-mac:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-mac/lib-scriptpackages:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-tk:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-old:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-dynload:/Library/Python/2.7/site-packages:/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python:/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/PyObjC".strip() paths = system_python_path.split(':') for path in paths: if path: diff --git a/radolan b/radolan deleted file mode 160000 index 2b954d9..0000000 --- a/radolan +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2b954d9e0fa1f252fcbe739cb957c68c0a8aca08 diff --git a/src/adaptors/Radolan.cpp b/src/adaptors/Radolan.cpp index f197143..9e5fbe2 100644 --- a/src/adaptors/Radolan.cpp +++ b/src/adaptors/Radolan.cpp @@ -1,4 +1,3 @@ - #include #include @@ -449,4 +448,4 @@ namespace m3D { } #endif } -} +} \ No newline at end of file diff --git a/src/executables/meanie3D-copydims.cpp b/src/executables/meanie3D-copydims.cpp index 9c634d5..2df381a 100644 --- a/src/executables/meanie3D-copydims.cpp +++ b/src/executables/meanie3D-copydims.cpp @@ -38,11 +38,11 @@ typedef double FS_TYPE; static const double NO_SCALE = numeric_limits::min(); void parse_commmandline(program_options::variables_map vm, - NcFile **filePtr, - string &source_filename, - string &root_directory, - vector &dimensions, - vector &dimension_variables) + NcFile **filePtr, + string &source_filename, + string &root_directory, + vector &dimensions, + vector &dimension_variables) { // Version @@ -65,7 +65,8 @@ void parse_commmandline(program_options::variables_map vm, try { root_directory = vm["root"].as(); - } catch (const boost::exception& e) + } + catch (const boost::exception &e) { cerr << "Missing 'root' argument" << endl; @@ -80,7 +81,8 @@ void parse_commmandline(program_options::variables_map vm, try { file = new NcFile(source_filename, NcFile::read); - } catch (const netCDF::exceptions::NcException &e) + } + catch (const netCDF::exceptions::NcException &e) { cerr << "ERROR opening file '" << source_filename << "' : " << e.what() << endl; exit(EXIT_FAILURE); @@ -100,7 +102,7 @@ void parse_commmandline(program_options::variables_map vm, // parse dimension list - typedef boost::tokenizer > tokenizer; + typedef boost::tokenizer> tokenizer; boost::char_separator sep(","); @@ -108,7 +110,7 @@ void parse_commmandline(program_options::variables_map vm, for (tokenizer::iterator tok_iter = dim_tokens.begin(); tok_iter != dim_tokens.end(); ++tok_iter) { - const char* name = (*tok_iter).c_str(); + const char *name = (*tok_iter).c_str(); dimensions.push_back(file->getDim(name)); @@ -116,20 +118,14 @@ void parse_commmandline(program_options::variables_map vm, } } -int main(int argc, char** argv) +int main(int argc, char **argv) { using namespace m3D; // Declare the supported options. program_options::options_description desc("Options"); - desc.add_options() - ("help", "produce this help message") - ("version", "print version information and exit") - ("source", program_options::value(), "File to use as a template") - ("root", program_options::value()->default_value("."), "Root directory to recursively iterate over files and replace the dimensions and vars in all \"*-clusters.nc\" files") - ("dimensions", program_options::value(), "Comma-separatred list of the dimensions to be used. The program expects dimension variables with identical names.") - ; + desc.add_options()("help", "produce this help message")("version", "print version information and exit")("source", program_options::value(), "File to use as a template")("root", program_options::value()->default_value("."), "Root directory to recursively iterate over files and replace the dimensions and vars in all \"*-clusters.nc\" files")("dimensions", program_options::value(), "Comma-separatred list of the dimensions to be used. The program expects dimension variables with identical names."); program_options::variables_map vm; @@ -137,7 +133,8 @@ int main(int argc, char** argv) { program_options::store(program_options::parse_command_line(argc, argv, desc), vm); program_options::notify(vm); - } catch (std::exception &e) + } + catch (std::exception &e) { cerr << "Error parsing command line: " << e.what() << endl; cerr << "Check meanie3D-trackplot --help for command line options" << endl; @@ -167,7 +164,8 @@ int main(int argc, char** argv) try { parse_commmandline(vm, &source, source_filename, root_directory, dimensions, dimension_variables); - } catch (const std::exception &e) + } + catch (const std::exception &e) { cerr << e.what() << endl; exit(EXIT_FAILURE); @@ -185,9 +183,7 @@ int main(int argc, char** argv) { fs::path f = dir_iter->path(); - if (fs::is_regular_file(f) - && boost::algorithm::ends_with(f.filename().generic_string(), "-clusters.nc") - && f.generic_string() != source_filename) + if (fs::is_regular_file(f) && boost::algorithm::ends_with(f.filename().generic_string(), "-clusters.nc") && f.generic_string() != source_filename) { try { @@ -220,7 +216,7 @@ int main(int argc, char** argv) multimap attributes = original.getAtts(); - multimap::iterator at; + multimap::iterator at; for (at = attributes.begin(); at != attributes.end(); at++) { @@ -228,7 +224,7 @@ int main(int argc, char** argv) size_t size = a.getAttLength(); - void *data = (void *) malloc(size); + void *data = (void *)malloc(size); a.getValues(data); @@ -285,7 +281,7 @@ int main(int argc, char** argv) // read data - FS_TYPE *var_data = (FS_TYPE*) malloc(sizeof (FS_TYPE) * size); + FS_TYPE *var_data = (FS_TYPE *)malloc(sizeof(FS_TYPE) * size); original_var.getVar(var_data); @@ -297,9 +293,9 @@ int main(int argc, char** argv) // copy attributes - map< string, NcVarAtt > attributes = original_var.getAtts(); + map attributes = original_var.getAtts(); - map< string, NcVarAtt >::iterator at; + map::iterator at; for (at = attributes.begin(); at != attributes.end(); at++) { @@ -307,7 +303,7 @@ int main(int argc, char** argv) size_t size = a.getAttLength(); - void *attr_data = (void *) malloc(size); + void *attr_data = (void *)malloc(size); a.getValues(attr_data); @@ -318,7 +314,8 @@ int main(int argc, char** argv) } cout << " done." << endl; - } catch (const netCDF::exceptions::NcException &e) + } + catch (const netCDF::exceptions::NcException &e) { cerr << "ERROR opening file '" << f.generic_string() << "' : " << e.what() << endl; exit(EXIT_FAILURE); diff --git a/src/executables/meanie3D-minmax.cpp b/src/executables/meanie3D-minmax.cpp index 7a63a9d..47d2cfa 100644 --- a/src/executables/meanie3D-minmax.cpp +++ b/src/executables/meanie3D-minmax.cpp @@ -40,6 +40,7 @@ using namespace m3D::utils; /** Feature-space data type */ typedef double T; +typedef unsigned char Byte; /** Filename formats */ @@ -98,79 +99,54 @@ void get_limits(NcVar variable, T& min, T& max) { min = std::numeric_limits::max(); max = std::numeric_limits::min(); - T fill_value = 0.0; - bool have_fill_value = false; - - try - { + try { NcVarAtt fillValue = variable.getAtt("_FillValue"); - - if (!fillValue.isNull()) - { + if (!fillValue.isNull()) { fillValue.getValues(&fill_value); - have_fill_value = true; } - } catch (::netCDF::exceptions::NcException &e) - { + } catch (::netCDF::exceptions::NcException &e) { + std::cerr << e.what() << std::endl; } - T *values = NULL; size_t numElements = 0; - - if (variable.getDimCount() == 1) - { + if (variable.getDimCount() == 1) { // 1D - numElements = variable.getDim(0).getSize(); values = allocate(numElements); variable.getVar(values); - } - - else if (variable.getDimCount() == 2) - { + } else if (variable.getDimCount() == 2) { // 2D - size_t N = variable.getDim(0).getSize(); size_t M = variable.getDim(1).getSize(); numElements = N * M; - values = allocate(numElements); variable.getVar(values); - - } else if (variable.getDimCount() == 3) - { + } else if (variable.getDimCount() == 3) { // 3D - size_t N = variable.getDim(0).getSize(); size_t M = variable.getDim(1).getSize(); size_t K = variable.getDim(2).getSize(); numElements = N * M * K; - values = allocate(numElements); - vector count(3, 1); count[0] = 1; count[1] = M; count[2] = K; - vector start(3, 0); start[0] = 0; start[1] = 0; start[2] = 0; - variable.getVar(start, count, values); - } else - { + } else { cerr << "ERROR:Variables with " << variable.getDimCount() << " dimensions are not currently handled" << endl; return; } - for (size_t j = 0; j < numElements; j++) - { + for (size_t j = 0; j < numElements; j++) { T value = values[j]; if (have_fill_value && value == fill_value) continue; if (value < min) diff --git a/src/executables/meanie3D-parallax_correction.cpp b/src/executables/meanie3D-parallax_correction.cpp index 121cc1f..cdd8bba 100644 --- a/src/executables/meanie3D-parallax_correction.cpp +++ b/src/executables/meanie3D-parallax_correction.cpp @@ -413,7 +413,6 @@ void correct_parallax(boost::filesystem::path in_path, const ShiftedProperties s } #if WITH_VTK #if WRITE_PARALLAX_VECTORS - string vector_path = in_path.filename().stem().string() + "-parallax.vtk"; VisitUtils::write_vectors_vtk(vector_path, origins, correction_vectors, "parallax"); #endif diff --git a/src/utils/.editorconfig b/src/utils/.editorconfig new file mode 100644 index 0000000..f46ec8f --- /dev/null +++ b/src/utils/.editorconfig @@ -0,0 +1,16 @@ +# top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +charset = utf-8 +trim_trailing_whitespace = true +end_of_line = lf +insert_final_newline = true + +# Tab indentation (no size specified) +[Makefile] +indent_style = tab + +[*.{c,h,cpp,hpp}] +indent_size = 2 \ No newline at end of file From e4681b7e59e0a6acee84ddfa669f03bc742bd42a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sat, 20 Jun 2020 16:25:44 +0100 Subject: [PATCH 09/65] Updated units.hpp --- include/meanie3D/units/units.hpp | 1341 +++++++++++++++--------------- 1 file changed, 660 insertions(+), 681 deletions(-) diff --git a/include/meanie3D/units/units.hpp b/include/meanie3D/units/units.hpp index 663dacb..89ab0d5 100644 --- a/include/meanie3D/units/units.hpp +++ b/include/meanie3D/units/units.hpp @@ -1,205 +1,232 @@ /* - Written by Calum Grant - Copyright (C) Calum Grant 2007 - - Home page: http://calumgrant.net/units - File location: http://calumgrant.net/units/units.hpp - - Copying permitted under the terms of the Boost software license. - - This library is distributed as a single file for convenience. - Normally it should be split up! - - Library name: - units - - Purpose: - Decorate values with units. - Provides safety and automatic conversion between values of different - units. - */ + Written by Calum Grant + Copyright (C) Calum Grant 2007 + Copyright (C) r-lyeh 2015 + Home page: http://github.com/r-lyeh/units + + Copying permitted under the terms of the Boost software license. + + This library is distributed as a single file for convenience. + Normally it should be split up! + + Library name: + units + + Purpose: + Decorate values with units. + Provides safety and automatic conversion between values of different + units. + + Changelog: + - v1.1.0 + - Binary SI units for information (byte, KiB, MiB, GiB, TiB, PiB) (@r-lyeh) + - Binary SI units for bandwidth (bps, KiBps, MiBps, GiBps, TiBps, PiBps) (@r-lyeh) + - v1.0.0 + - Initial commit (Calum Grant) +*/ #ifndef CG_UNITS_HPP_INCLUDED #define CG_UNITS_HPP_INCLUDED #include -namespace units { +#define UNITS_VERSION "v1.1.0" + +namespace units +{ namespace internal // Boost would call this "detail" { // Forward - template + template struct convert; // Allows construction of no units struct none; // Forward - template + template struct fixed_power; - } - + } // namespace internal // Construct a unit equivalent to Unit1*Unit2 - template + template struct compose; - // Constructs a unit equivalent to Unit*Num/Den - template + template struct scale; - // Constructs a unit equivalent to Unit+Num/Den - template + template struct translate; - // Constructs a unit equivalent to Unit^(Num/Den) - template + template struct pow; - // A unit which is effectively no units at all. typedef pow unit; - // A value with a unit. - // Value is the type you are storing + // Value is the type you are storing // Units is the units of the value - - template + template class value { public: typedef Value value_type; typedef Units unit; - value() : m_rep() { + value() : m_rep() + { } - explicit value(const value_type &v) : m_rep(v) { + explicit value(const value_type &v) : m_rep(v) + { } - template - value(const value &v) : - m_rep(internal::convert::fn(v.get())) { + template + value(const value &v) : m_rep(internal::convert::fn(v.get())) + { } - const value_type &get() const { + const value_type &get() const + { return m_rep; } - template - value &operator=(const value &other) { + template + value &operator=(const value &other) + { m_rep = value(other).get(); return *this; } - template - value operator+(const value &other) const { + template + value operator+(const value &other) const + { return value(get() + value(other).get()); } - template - value &operator+=(const value &other) { + template + value &operator+=(const value &other) + { m_rep += value(other).get(); return *this; } - template - value &operator-=(const value &other) { + template + value &operator-=(const value &other) + { m_rep -= value(other).get(); return *this; } - template - value operator-(const value &other) const { + template + value operator-(const value &other) const + { return value(get() - value(other).get()); } - value operator-() const { + value operator-() const + { return value(-get()); } - template + template value > - operator*(const value &other) const { - return value >(get() * other.get()); + operator*(const value &other) const + { + return value > (get() * other.get()); } - value operator*(const value_type &v) const { + value operator*(const value_type &v) const + { return value(get() * v); } - value &operator*=(const value_type &v) { + value &operator*=(const value_type &v) + { m_rep *= v; return *this; } - template + template value > > - operator/(const value &other) const { + operator/(const value &other) const + { return value > >(get() / other.get()); } - value operator/(const value_type &v) const { + value operator/(const value_type &v) const + { return value(get() / v); } - value &operator/=(const value_type &v) { + value &operator/=(const value_type &v) + { m_rep /= v; return *this; } - template - bool operator==(const value &other) const { + template + bool operator==(const value &other) const + { return get() == value(other).get(); } - template - bool operator!=(const value &other) const { + template + bool operator!=(const value &other) const + { return get() != value(other).get(); } - template - bool operator<(const value &other) const { + template + bool operator<(const value &other) const + { return get() < value(other).get(); } - template - bool operator<=(const value &other) const { + template + bool operator<=(const value &other) const + { return get() <= value(other).get(); } - template - bool operator>(const value &other) const { + template + bool operator>(const value &other) const + { return get() > value(other).get(); } - template - bool operator>=(const value &other) const { + template + bool operator>=(const value &other) const + { return get() >= value(other).get(); } - value &operator++() { + value &operator++() + { ++m_rep; return *this; } - value operator++(int) { + value operator++(int) + { value v = *this; ++m_rep; return v; } - value &operator--() { + value &operator--() + { --m_rep; return *this; } - value operator--(int) { + value operator--(int) + { value v = *this; --m_rep; return v; @@ -209,547 +236,502 @@ namespace units { value_type m_rep; }; - template - value > operator/(const Value &a, const value &b) { - return value >(a / b.get()); + template + value > operator/(const Value &a, const value &b) + { + return value > (a / b.get()); } - template - value operator*(const Value &a, const value &b) { + template + value operator*(const Value &a, const value &b) + { return value(a * b.get()); } - template - value > sqrt(const value &a) { - return value >(std::sqrt(a.get())); + template + value > sqrt(const value &a) + { + return value > (std::sqrt(a.get())); } - template - value > raise(const value &a) { - return value >(internal::fixed_power::pow(a.get())); + template + value > raise(const value &a) + { + return value > (internal::fixed_power::pow(a.get())); } -} - +} // namespace units /*****************************************************************************/ // Implementation -namespace units { - namespace internal { - // Ensures (at compile-time) that the template argument is true. - template struct static_assert; - - template<> struct static_assert { - }; - - - // Forward - template - struct convertible; - - - // Forward - template - struct scaling_factor; - - - // Converts T1 to T2. - // Stage 3 - performed after Stage 1 and Stage 2. - // The reason we perform convert in stages is so that the compiler - // can resolve templates in the order we want it to. - - template - struct convert3 +namespace units +{ + namespace internal { - // The default implementation assumes that the two quantities are in compatible - // units up to some scaling factor. Find the scaling factor and apply it. - - template - static V fn(const V &v) { - return v * scaling_factor::template fn() / scaling_factor::template fn(); - } - }; - - - // Converts T1 to T2. - // Template matches the first argument (T1), - // this is the fall-through to convert3. - - template - struct convert2 - { - - template - static V fn(const V &v) { - return convert3::fn(v); - } - }; - - - // Converts T1 to T2. - // If you really want to implement your own conversion routine, - // specialise this template. - // The default implementation falls through to convert2. - - template - struct convert - { - // If this fails, then T1 is not convertible to T2: - static_assert - ::value> check_convertible; - - template - static V fn(const V &v) { - return convert2::fn(v); - } - }; - - - // If we convert to the same type, the conversion function - // is trivial. - - template - struct convert - { - - template - static const U &fn(const U &u) { - return u; - } - }; - - - // Convert to same type. - - template - struct convert3 - { - - template - static const U &fn(const U &u) { - return u; - } - }; - - - // Convert from a scaled unit - - template - struct convert2, U> - { - - template - static V fn(const V &v) { - return convert::fn((v * Den) / Num); - } - }; - - - // Convert to a scaled unit - - template - struct convert3 > - { - - template - static V fn(const V &v) { - return (convert::fn(v) * Num) / Den; - } - }; - - - // Convert from a translated unit - - template - struct convert2, U> - { - - template - static V fn(const V &v) { - return convert::fn(v - static_cast (Num) / static_cast (Den)); - } - }; - + // Ensures (at compile-time) that the template argument is true. + template + struct static_assert_; + template <> + struct static_assert_ + { + }; - // Convert to a translated unit + // Forward + template + struct convertible; - template - struct convert3 > - { + // Forward + template + struct scaling_factor; + + // Converts T1 to T2. + // Stage 3 - performed after Stage 1 and Stage 2. + // The reason we perform convert in stages is so that the compiler + // can resolve templates in the order we want it to. + template + struct convert3 + { + // The default implementation assumes that the two quantities are in compatible + // units up to some scaling factor. Find the scaling factor and apply it. + template + static V fn(const V &v) + { + return v * scaling_factor::template fn() / scaling_factor::template fn(); + } + }; - template - static V fn(const V &v) { - return convert::fn(v) + static_cast (Num) / static_cast (Den); - } - }; + // Converts T1 to T2. + // Template matches the first argument (T1), + // this is the fall-through to convert3. + template + struct convert2 + { + template + static V fn(const V &v) + { + return convert3::fn(v); + } + }; + // Converts T1 to T2. + // If you really want to implement your own conversion routine, + // specialise this template. + // The default implementation falls through to convert2. + template + struct convert + { + // If this fails, then T1 is not convertible to T2: + static_assert_::value> check_convertible; - // Count the power to which unit Term is raised in the unit List. - // Returns a rational num/den of the power of term Term in List. - // The default assumes that Term is not found (num/den=0) + template + static V fn(const V &v) + { + return convert2::fn(v); + } + }; - template - struct count_terms - { - static const int num = 0; - static const int den = 1; - }; + // If we convert to the same type, the conversion function + // is trivial. + template + struct convert + { + template + static const U &fn(const U &u) { return u; } + }; - template - struct count_terms - { - static const int num = 1; - static const int den = 1; - }; + // Convert to same type. + template + struct convert3 + { + template + static const U &fn(const U &u) { return u; } + }; + // Convert from a scaled unit + template + struct convert2, U> + { + template + static V fn(const V &v) + { + return convert::fn((v * Den) / Num); + } + }; - // count_terms ignores scaling factors - that is taken care of by scaling_factor. + // Convert to a scaled unit + template + struct convert3 > + { + template + static V fn(const V &v) + { + return (convert::fn(v) * Num) / Den; + } + }; - template - struct count_terms > - { - typedef count_terms result; - static const int num = result::num; - static const int den = result::den; - }; + // Convert from a translated unit + template + struct convert2, U> + { + template + static V fn(const V &v) + { + return convert::fn(v - static_cast(Num) / static_cast(Den)); + } + }; + // Convert to a translated unit + template + struct convert3 > + { + template + static V fn(const V &v) + { + return convert::fn(v) + static_cast(Num) / static_cast(Den); + } + }; - // count_terms ignores translation. + // Count the power to which unit Term is raised in the unit List. + // Returns a rational num/den of the power of term Term in List. + // The default assumes that Term is not found (num/den=0) + template + struct count_terms + { + static const int num = 0; + static const int den = 1; + }; - template - struct count_terms > - { - typedef count_terms result; - static const int num = result::num; - static const int den = result::den; - }; + template + struct count_terms + { + static const int num = 1; + static const int den = 1; + }; + // count_terms ignores scaling factors - that is taken care of by scaling_factor. + template + struct count_terms > + { + typedef count_terms result; + static const int num = result::num; + static const int den = result::den; + }; - // Addition of fractions. + // count_terms ignores translation. + template + struct count_terms > + { + typedef count_terms result; + static const int num = result::num; + static const int den = result::den; + }; - template - struct count_terms > - { - typedef count_terms result1; - typedef count_terms result2; - static const int num = + // Addition of fractions. + template + struct count_terms > + { + typedef count_terms result1; + typedef count_terms result2; + static const int num = result1::num * result2::den + result1::den * result2::num; - static const int den = + static const int den = result1::den * result2::den; - }; - - - // Multiplication of fractions. - - template - struct count_terms > - { - typedef count_terms result; - static const int num = N * result::num; - static const int den = D * result::den; - }; - + }; - // Counts the power of the unit Term in units T1 and T2. - // Reports if they are equal, using equality of fractions. - // Does a depth-first search of the unit "Term", - // or counts the terms in the default case. + // Multiplication of fractions. + template + struct count_terms > + { + typedef count_terms result; + static const int num = N * result::num; + static const int den = D * result::den; + }; - template - struct check_terms_equal - { - typedef count_terms count1; - typedef count_terms count2; + // Counts the power of the unit Term in units T1 and T2. + // Reports if they are equal, using equality of fractions. + // Does a depth-first search of the unit "Term", + // or counts the terms in the default case. + template + struct check_terms_equal + { + typedef count_terms count1; + typedef count_terms count2; - static const bool value = + static const bool value = count1::num * count2::den == count1::den * count2::num; - }; + }; - template - struct check_terms_equal, T1, T2> - { - static const bool value = check_terms_equal::value; - }; + template + struct check_terms_equal, T1, T2> + { + static const bool value = check_terms_equal::value; + }; - template - struct check_terms_equal, T1, T2> - { - static const bool value = check_terms_equal::value; - }; + template + struct check_terms_equal, T1, T2> + { + static const bool value = check_terms_equal::value; + }; - template - struct check_terms_equal, T1, T2> - { - static const bool value = check_terms_equal::value; - }; + template + struct check_terms_equal, T1, T2> + { + static const bool value = check_terms_equal::value; + }; - template - struct check_terms_equal, T3, T4> - { - static const bool value = + template + struct check_terms_equal, T3, T4> + { + static const bool value = check_terms_equal::value && check_terms_equal::value; - }; - - - // Determines whether two types are convertible - // Counts the powers in the LHS and RHS and ensures they are equal. + }; - template - struct convertible - { - static const bool value = + // Determines whether two types are convertible + // Counts the powers in the LHS and RHS and ensures they are equal. + template + struct convertible + { + static const bool value = check_terms_equal::value && check_terms_equal::value; - }; - - - // A functor that raises a value to the power Num/Den. - // The template is specialised for efficiency - // so that we don't always have to call the std::pow function. - - template - struct fixed_power - { - - template - static T pow(const T &t) { - return std::pow(t, static_cast (Num) / static_cast (Den)); - } - }; - - template - struct fixed_power - { - - template - static const T &pow(const T &t) { - return t; - } - }; - - template - struct fixed_power - { - - template - static T pow(const T &t) { - return t * t; - } - }; - - template - struct fixed_power - { - - template - static T pow(const T &t) { - return t * t * t; - } - }; - - template - struct fixed_power - { - - template - static const T &pow(const T &t) { - T u = t * t; - return u * u; - } - }; - - template - struct fixed_power - { - - template - static T pow(const T &t) { - return 1 / t; - } - }; - - template - struct fixed_power - { - - template - static T pow(const T &t) { - return 1 / (t * t); - } - }; - - template - struct fixed_power - { - - template - static T pow(const T &t) { - return 1; - } - }; + }; + // A functor that raises a value to the power Num/Den. + // The template is specialised for efficiency + // so that we don't always have to call the std::pow function. + template + struct fixed_power + { + template + static T pow(const T &t) + { + return std::pow(t, static_cast(Num) / static_cast(Den)); + } + }; - // Determine the scaling factor of a unit in relation to its "base" units. - // Default is that U is a primitive unit and is not scaled. + template + struct fixed_power + { + template + static const T &pow(const T &t) + { + return t; + } + }; - template - struct scaling_factor - { + template + struct fixed_power + { + template + static T pow(const T &t) + { + return t * t; + } + }; - template - static T fn() { - return 1; - } - }; + template + struct fixed_power + { + template + static T pow(const T &t) + { + return t * t * t; + } + }; - template - struct scaling_factor > - { + template + struct fixed_power + { + template + static const T &pow(const T &t) + { + T u = t * t; + return u * u; + } + }; - template - static T fn() { - return - scaling_factor::template fn() * - scaling_factor::template fn(); - } - }; + template + struct fixed_power + { + template + static T pow(const T &t) + { + return 1 / t; + } + }; - template - struct scaling_factor > - { + template + struct fixed_power + { + template + static T pow(const T &t) + { + return 1 / (t * t); + } + }; - template - static T fn() { - return - scaling_factor::template fn() * - static_cast (N) / static_cast (D); - } - }; + template + struct fixed_power + { + template + static T pow(const T &t) + { + return 1; + } + }; - template - struct scaling_factor > - { + // Determine the scaling factor of a unit in relation to its "base" units. + // Default is that U is a primitive unit and is not scaled. + template + struct scaling_factor + { + template + static T fn() { return 1; } + }; - template - static T fn() { - return fixed_power::pow(scaling_factor::template fn()); - } - }; + template + struct scaling_factor > + { + template + static T fn() + { + return scaling_factor::template fn() * + scaling_factor::template fn(); + } + }; - template - struct scaling_factor > - { + template + struct scaling_factor > + { + template + static T fn() + { + return scaling_factor::template fn() * + static_cast(N) / static_cast(D); + } + }; - template - static T fn() { - return scaling_factor::template fn(); - } - }; -} -} + template + struct scaling_factor > + { + template + static T fn() + { + return fixed_power::pow(scaling_factor::template fn()); + } + }; + template + struct scaling_factor > + { + template + static T fn() + { + return scaling_factor::template fn(); + } + }; + } // namespace internal +} // namespace units /*****************************************************************************/ // Display - -#define UNIT_DISPLAY_NAME(unit, string) \ -template<> struct output_unit { \ - template static void fn(Stream&os) { os << string; } \ -}; - +#define UNIT_DISPLAY_NAME(unit, string) \ + template <> \ + struct output_unit \ + { \ + template \ + static void fn(Stream &os) { os << string; } \ + }; #define UNITS_DISPLAY_NAME(unit, string) \ -namespace units { UNIT_DISPLAY_NAME( unit, string ) } - + namespace units \ + { \ + UNIT_DISPLAY_NAME(unit, string) \ + } -namespace units { - namespace internal { +namespace units +{ + namespace internal + { // This is the default unit formatting mechanism - - template + template struct output_unit2 { - - template - static void fn(Stream &os) { - os << "units"; - } + template + static void fn(Stream &os) { os << "units"; } }; - } - + } // namespace internal // Functor to write unit text to stream - - template + template struct output_unit { - - template - static void fn(Stream &os) { - internal::output_unit2::fn(os); - } + template + static void fn(Stream &os) { internal::output_unit2::fn(os); } }; - UNIT_DISPLAY_NAME(unit, "1"); - - namespace internal { - - template - struct output_unit2 > + namespace internal + { + template + struct output_unit2 > { - - template - static void fn(Stream &os) { + template + static void fn(Stream &os) + { output_unit::fn(os); os << '.'; output_unit::fn(os); } }; - template - struct output_unit2 > + template + struct output_unit2 > { - - template - static void fn(Stream &os) { - if (Num != Den) os << '('; + template + static void fn(Stream &os) + { + if (Num != Den) + os << '('; output_unit::fn(os); - if (Num != Den) { + if (Num != Den) + { os << ')'; os << '^' << Num; - if (Num % Den) { + if (Num % Den) + { os << '/' << Den; } } } }; - template - struct output_unit2 > + template + struct output_unit2 > { - - template - static void fn(Stream &os) { + template + static void fn(Stream &os) + { os << '('; output_unit::fn(os); os << '+' << Num; - if (Den != 1) os << '/' << Den; + if (Den != 1) + os << '/' << Den; os << ')'; } }; - template - struct output_unit2 > + template + struct output_unit2 > { - - template - static void fn(Stream &os) { + template + static void fn(Stream &os) + { os << Den; if (Num != 1) os << '/' << Num; @@ -757,232 +739,197 @@ namespace units { output_unit::fn(os); } }; - } + } // namespace internal - template - Str &operator<<(Str &os, const value &value) { + template + Str &operator<<(Str &os, const value &value) + { os << value.get() << ' '; output_unit::fn(os); return os; } -} - +} // namespace units /*****************************************************************************/ // Additional units -namespace units { - namespace units { +namespace units +{ + namespace units + { typedef ::units::unit unit; // SI base units: - struct m; // meter - struct kg; // kilogram - struct s; // second - struct K; // Kelvin - struct A; // Ampere - struct mol; // mole - struct cd; // candela - } + struct m; // meter + struct kg; // kilogram + struct s; // second + struct K; // Kelvin + struct A; // Ampere + struct mol; // mole + struct cd; // candela + struct byte; // information + } // namespace units UNIT_DISPLAY_NAME(units::m, "m"); - UNIT_DISPLAY_NAME(units::kg, "kg"); - UNIT_DISPLAY_NAME(units::s, "s"); - UNIT_DISPLAY_NAME(units::K, "K"); - UNIT_DISPLAY_NAME(units::A, "A"); - UNIT_DISPLAY_NAME(units::mol, "mol"); - UNIT_DISPLAY_NAME(units::cd, "cd"); + UNIT_DISPLAY_NAME(units::byte, "bytes"); - namespace units { + namespace units + { // SI derived units: - typedef compose > rad; - typedef compose, pow > sr; + typedef compose > rad; + typedef compose, pow > sr; typedef pow Hz; typedef compose > > N; - typedef compose > Pa; + typedef compose > Pa; typedef compose J; - typedef compose > W; + typedef compose > W; typedef compose C; - typedef compose > V; - typedef compose > F; - typedef compose > Ohm; - typedef compose > S; + typedef compose > V; + typedef compose > F; + typedef compose > Ohm; + typedef compose > S; typedef compose Wb; - typedef compose > T; - typedef compose > H; + typedef compose > T; + typedef compose > H; typedef cd lm; - typedef compose > lx; + typedef compose > lx; typedef pow Bq; - typedef compose > Gy; + typedef compose > Gy; typedef Gy Sv; typedef compose, mol> kat; - } + } // namespace units UNIT_DISPLAY_NAME(units::rad, "rad"); - UNIT_DISPLAY_NAME(units::sr, "sr"); - // UNIT_DISPLAY_NAME( units::Hz, "Hz" ); // Too problematic + // UNIT_DISPLAY_NAME( units::Hz, "Hz" ); // Too problematic UNIT_DISPLAY_NAME(units::N, "N"); - UNIT_DISPLAY_NAME(units::Pa, "Pa"); - UNIT_DISPLAY_NAME(units::J, "J"); - UNIT_DISPLAY_NAME(units::W, "W"); - UNIT_DISPLAY_NAME(units::C, "C"); - UNIT_DISPLAY_NAME(units::V, "V"); - UNIT_DISPLAY_NAME(units::F, "F"); - UNIT_DISPLAY_NAME(units::Ohm, "Ohm"); - UNIT_DISPLAY_NAME(units::S, "S"); - UNIT_DISPLAY_NAME(units::Wb, "Wb"); - UNIT_DISPLAY_NAME(units::T, "T"); - UNIT_DISPLAY_NAME(units::H, "H"); - UNIT_DISPLAY_NAME(units::lx, "lx"); - UNIT_DISPLAY_NAME(units::Gy, "Gy"); - UNIT_DISPLAY_NAME(units::kat, "kat"); - namespace units { + namespace units + { // SI prefixes: - - template + template struct deca { typedef scale type; }; - - template + template struct hecto { typedef scale type; }; - - template + template struct kilo { typedef scale type; }; - - template + template struct mega { typedef scale::type, 1, 1000> type; }; - - template + template struct giga { typedef scale::type, 1, 1000> type; }; - - template + template struct tera { typedef scale::type, 1, 1000> type; }; - - template + template struct peta { typedef scale::type, 1, 1000> type; }; - - template + template struct exa { typedef scale::type, 1, 1000> type; }; - - template + template struct zetta { typedef scale::type, 1, 1000> type; }; - - template + template struct yotta { typedef scale::type, 1, 1000> type; }; - template + template struct deci { typedef scale type; }; - - template + template struct centi { typedef scale type; }; - - template + template struct milli { typedef scale type; }; - - template + template struct micro { typedef scale::type, 1000> type; }; - - template + template struct nano { typedef scale::type, 1000> type; }; - - template + template struct pico { typedef scale::type, 1000> type; }; - - template + template struct femto { typedef scale::type, 1000> type; }; - - template + template struct atto { typedef scale::type, 1000> type; }; - - template + template struct zepto { typedef scale::type, 1000> type; }; - - template + template struct yocto { typedef scale::type, 1000> type; }; - // Some prefixed SI units: typedef centi::type cm; typedef milli::type mm; @@ -990,21 +937,60 @@ namespace units { typedef milli::type g; typedef milli::type mg; typedef milli::type ms; - } + typedef micro::type us; + typedef nano::type ns; - UNIT_DISPLAY_NAME(units::cm, "cm"); + // binary SI prefixes (capital letter): + template + struct Kilo + { + typedef scale type; + }; + template + struct Mega + { + typedef scale::type, 1, 1024> type; + }; + template + struct Giga + { + typedef scale::type, 1, 1024> type; + }; + template + struct Tera + { + typedef scale::type, 1, 1024> type; + }; + template + struct Peta + { + typedef scale::type, 1, 1024> type; + }; - UNIT_DISPLAY_NAME(units::mm, "mm"); + typedef Kilo::type KiB; + typedef Mega::type MiB; + typedef Giga::type GiB; + typedef Tera::type TiB; + typedef Peta::type PiB; + } // namespace units + UNIT_DISPLAY_NAME(units::cm, "cm"); + UNIT_DISPLAY_NAME(units::mm, "mm"); UNIT_DISPLAY_NAME(units::km, "km"); - UNIT_DISPLAY_NAME(units::g, "g"); - UNIT_DISPLAY_NAME(units::mg, "mg"); - UNIT_DISPLAY_NAME(units::ms, "ms"); + UNIT_DISPLAY_NAME(units::us, "us"); + UNIT_DISPLAY_NAME(units::ns, "ns"); - namespace units { + UNIT_DISPLAY_NAME(units::KiB, "KiB"); + UNIT_DISPLAY_NAME(units::MiB, "MiB"); + UNIT_DISPLAY_NAME(units::GiB, "GiB"); + UNIT_DISPLAY_NAME(units::TiB, "TiB"); + UNIT_DISPLAY_NAME(units::PiB, "PiB"); + + namespace units + { // Non-SI mass typedef scale lb; typedef scale oz; @@ -1047,12 +1033,20 @@ namespace units { typedef pow m3; // Non-SI velocity - typedef compose > mph; - typedef compose > kph; - typedef compose > meters_per_second; - typedef compose > knot; + typedef compose > mph; + typedef compose > kph; + typedef compose > meters_per_second; + typedef compose > knot; typedef scale mach; + // Non-SI velocity (transfer/bandwidth) + typedef compose > bps; + typedef compose > KiBps; + typedef compose > MiBps; + typedef compose > GiBps; + typedef compose > TiBps; + typedef compose > PiBps; + // Angles typedef scale degree; typedef scale grad; @@ -1069,89 +1063,56 @@ namespace units { typedef scale percent; typedef scale dozen; typedef scale bakers_dozen; - } + } // namespace units UNIT_DISPLAY_NAME(units::lb, "lb"); - UNIT_DISPLAY_NAME(units::oz, "oz"); - UNIT_DISPLAY_NAME(units::tonne, "tonnes"); - UNIT_DISPLAY_NAME(units::Celsius, "'C"); - UNIT_DISPLAY_NAME(units::Fahrenheit, "'F"); - UNIT_DISPLAY_NAME(units::minute, "minutes"); - UNIT_DISPLAY_NAME(units::hour, "hours"); - UNIT_DISPLAY_NAME(units::day, "days"); - UNIT_DISPLAY_NAME(units::week, "weeks"); - UNIT_DISPLAY_NAME(units::month, "months"); - UNIT_DISPLAY_NAME(units::year, "years"); - UNIT_DISPLAY_NAME(units::century, "centuries"); - UNIT_DISPLAY_NAME(units::millennium, "millennia"); - UNIT_DISPLAY_NAME(units::inch, "inches"); - UNIT_DISPLAY_NAME(units::foot, "foot"); - UNIT_DISPLAY_NAME(units::yard, "yards"); - UNIT_DISPLAY_NAME(units::mile, "miles"); - UNIT_DISPLAY_NAME(units::nautical_mile, "nautical miles"); - UNIT_DISPLAY_NAME(units::hectare, "ha"); - UNIT_DISPLAY_NAME(units::are, "are"); - UNIT_DISPLAY_NAME(units::acre, "acres"); - UNIT_DISPLAY_NAME(units::ml, "ml"); - UNIT_DISPLAY_NAME(units::liter, "l"); - UNIT_DISPLAY_NAME(units::dl, "dl"); - UNIT_DISPLAY_NAME(units::cl, "cl"); - UNIT_DISPLAY_NAME(units::mph, "mph"); - UNIT_DISPLAY_NAME(units::kph, "km/h"); - UNIT_DISPLAY_NAME(units::knot, "knots"); - - UNIT_DISPLAY_NAME(units::mach, "mach"); - + UNIT_DISPLAY_NAME(units::bps, "bytes/s"); + UNIT_DISPLAY_NAME(units::KiBps, "KiB/s"); + UNIT_DISPLAY_NAME(units::MiBps, "MiB/s"); + UNIT_DISPLAY_NAME(units::GiBps, "GiB/s"); + UNIT_DISPLAY_NAME(units::TiBps, "TiB/s"); + UNIT_DISPLAY_NAME(units::PiBps, "PiB/s"); UNIT_DISPLAY_NAME(units::degree, "deg"); - UNIT_DISPLAY_NAME(units::grad, "grad"); - UNIT_DISPLAY_NAME(units::degree_minute, "'"); - UNIT_DISPLAY_NAME(units::degree_second, "\""); - UNIT_DISPLAY_NAME(units::kPa, "kPa"); - UNIT_DISPLAY_NAME(units::psi, "PSI"); - UNIT_DISPLAY_NAME(units::millibar, "millibars"); - UNIT_DISPLAY_NAME(units::percent, "%"); - UNIT_DISPLAY_NAME(units::rpm, "rpm"); - UNIT_DISPLAY_NAME(units::dozen, "dozen"); - UNIT_DISPLAY_NAME(units::bakers_dozen, "bakers dozen"); - namespace values { + namespace values + { typedef value unit; // SI units @@ -1186,6 +1147,14 @@ namespace units { typedef value Sv; typedef value kat; + // SI binary + typedef value byte; + typedef value KiB; + typedef value MiB; + typedef value GiB; + typedef value TiB; + typedef value PiB; + // Prefixed units typedef value cm; typedef value mm; @@ -1193,6 +1162,8 @@ namespace units { typedef value g; typedef value mg; typedef value ms; + typedef value us; + typedef value ns; // Non-SI typedef value lb; @@ -1236,6 +1207,13 @@ namespace units { typedef value knot; typedef value mach; + typedef value bps; + typedef value KiBps; + typedef value MiBps; + typedef value GiBps; + typedef value TiBps; + typedef value PiBps; + typedef value degree; typedef value grad; typedef value degree_minute; @@ -1249,14 +1227,14 @@ namespace units { typedef value rpm; typedef value dozen; typedef value bakers_dozen; - } - + } // namespace values - namespace constants { + namespace constants + { // Physical constants: const value > > k(1.3806504e-23); const value mu(1.660538782e-27); - const value > NA(6.02214179e23); + const value > NA(6.02214179e23); const value G0(7.7480917004e-5); const value > > e0(8.854187817e-12); const value me(9.10938215e-31); @@ -1267,40 +1245,41 @@ namespace units { const value inv_alpha(137.035999679); const value > > u0(12.566370614); const value phi0(2.067833667e-15); // ?? - const value, pow > > > R(8.314472); - const value, compose, pow > > > G( - 6.67428e-11); - const value > h(6.62606896e-34); - const value > h_bar(1.054571628e-34); + const value, pow > > > R(8.314472); + const value, compose, pow > > > G(6.67428e-11); + const value > h(6.62606896e-34); + const value > h_bar(1.054571628e-34); const value mp(1.672621637e-27); const value mpme(1836.15267247); - const value > Rinf(10973731.568527); + const value > Rinf(10973731.568527); const value > > c(299792458); - const value, pow > > > rho(5.6704e-8); + const value, pow > > > rho(5.6704e-8); // Other constants: const value pi(3.141592653589793); const value lightyear(9.4605284e15); const value > > g(9.80665); - } - + } // namespace constants // Trigonometry - template - Value sin(const value &angle) { + template + Value sin(const value &angle) + { return std::sin(value(angle).get()); } - template - Value cos(const value &angle) { + template + Value cos(const value &angle) + { return std::cos(value(angle).get()); } - template - Value tan(const value &angle) { + template + Value tan(const value &angle) + { return std::tan(value(angle).get()); } -} +} // namespace units -#endif +#endif \ No newline at end of file From 5ccd787e3d992d5071b4b3de056c2723130d1626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sat, 20 Jun 2020 16:30:52 +0100 Subject: [PATCH 10/65] Syntax fixes for gcc --- include/meanie3D/units/units.hpp | 2 +- src/executables/meanie3D-copydims.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/meanie3D/units/units.hpp b/include/meanie3D/units/units.hpp index 89ab0d5..b982243 100644 --- a/include/meanie3D/units/units.hpp +++ b/include/meanie3D/units/units.hpp @@ -322,7 +322,7 @@ namespace units struct convert { // If this fails, then T1 is not convertible to T2: - static_assert_::value> check_convertible; + static_assert_::value > check_convertible; template static V fn(const V &v) diff --git a/src/executables/meanie3D-copydims.cpp b/src/executables/meanie3D-copydims.cpp index 2df381a..8b56f0a 100644 --- a/src/executables/meanie3D-copydims.cpp +++ b/src/executables/meanie3D-copydims.cpp @@ -102,7 +102,7 @@ void parse_commmandline(program_options::variables_map vm, // parse dimension list - typedef boost::tokenizer> tokenizer; + typedef boost::tokenizer > tokenizer; boost::char_separator sep(","); From 36295f76b9b602d40f7e985cc9b985cdf4f7634d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sat, 20 Jun 2020 16:39:36 +0100 Subject: [PATCH 11/65] Upgrade to Python3 --- python/CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 240806e..cf92d9d 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -18,9 +18,9 @@ GET_DIRECTORY_PROPERTY(PROJECT_URL DIRECTORY .. DEFINITION PROJECT_URL) GET_DIRECTORY_PROPERTY(PROJECT_LICENSE DIRECTORY .. DEFINITION PROJECT_LICENSE) # Python -FIND_PACKAGE(PythonInterp REQUIRED) -IF (PYTHONINTERP_FOUND) - MESSAGE(STATUS "Found Python interpreter (${PYTHON_VERSION_STRING}) ${PYTHON_EXECUTABLE}") +FIND_PACKAGE (Python3 COMPONENTS Interpreter) +IF (Python3_FOUND) + MESSAGE(STATUS "Found Python interpreter (${Python3_VERSION}) ${Python3_EXECUTABLE}") ELSE () MESSAGE(FATAL_ERROR "python not found") ENDIF () @@ -28,11 +28,11 @@ ENDIF () # ------------------------------------------------------------- # # Python packaging # ------------------------------------------------------------- # -IF (PYTHONINTERP_FOUND) +IF (Python3_FOUND) # Use the system's python path to append it to visit's python # path to enable importing modules to visit's python. - EXECUTE_PROCESS(COMMAND python -c "import sys; sys.stdout.write(':'.join(sys.path))" OUTPUT_VARIABLE SYSTEM_PYTHON_PATH) + EXECUTE_PROCESS(COMMAND ${Python3_EXECUTABLE} -c "import sys; sys.stdout.write(':'.join(sys.path))" OUTPUT_VARIABLE SYSTEM_PYTHON_PATH) # Configure package file FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/meanie3D DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) @@ -48,9 +48,9 @@ IF (PYTHONINTERP_FOUND) ADD_CUSTOM_TARGET(copy-python-files ALL COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/meanie3D ${CMAKE_CURRENT_BINARY_DIR} DEPENDS meanie3D) - ADD_CUSTOM_TARGET(${PROJECT_NAME} ALL COMMAND ${PYTHON_EXECUTABLE} setup.py build) + ADD_CUSTOM_TARGET(${PROJECT_NAME} ALL COMMAND ${Python3_EXECUTABLE} setup.py build) ADD_CUSTOM_TARGET(${PROJECT_NAME}-clean COMMAND rm -rf build dist *.egg-info setup.py meanie3D/__init__.py README.txt) - INSTALL(CODE "EXECUTE_PROCESS(COMMAND \"${PYTHON_EXECUTABLE}\" setup.py install WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})") + INSTALL(CODE "EXECUTE_PROCESS(COMMAND \"${Python3_EXECUTABLE}\" setup.py install WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})") ENDIF () From 7823edf12d7bc5f89a55392f15871444c3be66de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sat, 20 Jun 2020 17:17:02 +0100 Subject: [PATCH 12/65] Upgrade to Python3 (fixes) --- Dockerfile | 38 +++-------- gtest-1.7.0/xcode/Scripts/versiongenerate.py | 2 +- python/CMakeLists.txt | 1 - python/meanie2tic/meanie2tic.py | 22 +++--- python/meanie3D/app/analysis.py | 12 ++-- python/meanie3D/app/external.py | 27 ++++++-- python/meanie3D/app/postprocessing.py | 72 ++++++++++---------- python/meanie3D/app/tracking.py | 26 +++---- python/meanie3D/app/utils.py | 4 +- python/meanie3D/meanie3D_run.py | 64 ++++++++--------- python/meanie3D/visualisation/clusters.py | 51 +++++++------- python/meanie3D/visualisation/tracks.py | 12 ++-- python/meanie3D/visualisation/utils.py | 26 +++---- 13 files changed, 176 insertions(+), 181 deletions(-) diff --git a/Dockerfile b/Dockerfile index b8856b9..6be24ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,32 +7,11 @@ RUN apt-get -y dist-upgrade RUN apt-get -y install software-properties-common RUN apt-get -y update RUN apt-get -y install \ -git \ -gcc g++ \ -python3 python3-pip \ -cmake \ -doxygen \ -libflann1.9 libflann-dev \ -libboost-all-dev \ -blitz++ \ -shapelib \ -libhdf5-dev \ -netcdf-bin libnetcdf-dev libnetcdf-c++4 python3-netcdf4 \ -zlib1g zlib1g-dev -RUN pip3 install setuptools - -# Build NetCDF-CXX (always an extra bloody sausage with this package...) -RUN apt-get -y install wget -RUN wget --quiet https://github.com/Unidata/netcdf-cxx4/archive/v4.2.1.tar.gz -RUN tar xvzf v4.2.1.tar.gz -RUN cd netcdf-cxx4-4.2.1 && ./configure && make install && cd .. -RUN rm -rf netcdf-cxx4-4.2.1 && rm v4.2.1.tar.gz - -# Install python-netcdf4 from source -#RUN wget https://pypi.python.org/packages/source/n/netCDF4/netCDF4-1.2.3.1.tar.gz#md5=24fc0101c7c441709c230e76af611d53 -#RUN tar xvzf netCDF4-1.2.3.1.tar.gz -#RUN cd netCDF4-1.2.3.1 && python setup.py build && python setup.py build && cd .. -#RUN rm -rf netCDF4* +wget git cmake \ +gcc g++ python3 python3-pip \ +libboost-all-dev libflann1.9 libflann-dev blitz++ \ +shapelib libhdf5-dev netcdf-bin libnetcdf-dev libnetcdf-c++4 libnetcdf-c++4-dev zlib1g zlib1g-dev +RUN pip3 install setuptools netcdf4 external utils # Visualisation # RUN pip3 install Cython h5py netcdf4 @@ -44,16 +23,21 @@ RUN rm -rf netcdf-cxx4-4.2.1 && rm v4.2.1.tar.gz # ENV VISIT_EXECUTABLE=/usr/local/visit/bin/visit # RUN rm -rf visit* +# libradolan +RUN git clone https://github.com/JuergenSimon/radolan.git +RUN cd radolan && cmake . && make install && cd .. && rm -rf radolan + # Meanie3D RUN git clone --recurse-submodules --depth=1 https://github.com/JuergenSimon/meanie3D WORKDIR /meanie3D RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch --all RUN git checkout --track remotes/origin/dockerize && git pull -RUN cmake -DFOR_DOCKER=1 . && make install +RUN git pull && cmake -DFOR_DOCKER=YES . && make install # Cleanup WORKDIR / RUN rm -rf meanie3D +RUN apt-get remove -y wget git cmake RUN apt autoremove -y # Prepare for runtime diff --git a/gtest-1.7.0/xcode/Scripts/versiongenerate.py b/gtest-1.7.0/xcode/Scripts/versiongenerate.py index 81de8c9..ea24f96 100755 --- a/gtest-1.7.0/xcode/Scripts/versiongenerate.py +++ b/gtest-1.7.0/xcode/Scripts/versiongenerate.py @@ -54,7 +54,7 @@ # Read the command line argument (the output directory for Version.h) if (len(sys.argv) < 3): - print "Usage: versiongenerate.py input_dir output_dir" + print("Usage: versiongenerate.py input_dir output_dir") sys.exit(1) else: input_dir = sys.argv[1] diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index cf92d9d..e9ece72 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -52,5 +52,4 @@ IF (Python3_FOUND) ADD_CUSTOM_TARGET(${PROJECT_NAME}-clean COMMAND rm -rf build dist *.egg-info setup.py meanie3D/__init__.py README.txt) INSTALL(CODE "EXECUTE_PROCESS(COMMAND \"${Python3_EXECUTABLE}\" setup.py install WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})") - ENDIF () diff --git a/python/meanie2tic/meanie2tic.py b/python/meanie2tic/meanie2tic.py index 2d2d10c..d0d9c32 100755 --- a/python/meanie2tic/meanie2tic.py +++ b/python/meanie2tic/meanie2tic.py @@ -59,7 +59,7 @@ def write_csv(json): objects = csv.writer(objects_file, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL) - print "Writing object information:\n" + print("Writing object information:\n") tracks = json['tracks']; for ti in range(0,len(tracks)): track = tracks[ti] @@ -70,7 +70,7 @@ def write_csv(json): header = ['timestep','id','centerx','centery','volume', 'boxxmin','boxxmax','boxymin','boxymax', 'lwpmean','lwpmin','lwpmax'] - print header + print(header) objects.writerow(header) for ci in range(0,len(clusters)): @@ -111,14 +111,14 @@ def write_csv(json): # (12) Max LWP of the object (=maximal LWP value included) row.append(cluster['max'][0]) - print row + print(f"{row}") objects.writerow(row) # # Write the tracks # - print "Writing tracking information:\n" + print("Writing tracking information:\n") tracks_file = open('tracks.csv', 'wb') if (not tracks_file): @@ -143,7 +143,7 @@ def write_csv(json): # if the first row, write a header if li == 0: header = ['timestep','idstep1','idstep2','weight'] - print header + print(f"{header}") tracks.writerow(header) row = [] @@ -159,17 +159,17 @@ def write_csv(json): # (4) Weight factor to decide object relevance after split/merge event for life row.append(0) - print row + print(f"{row}") tracks.writerow(row) # ---------------------------------------------------------------------------- ## Prints usage and exits # def usage(): - print "meanie2tic.py [--file,-f] [--help,-h]" - print "Converts meanie3D track-dictionary.json into CSV files for Tracking Intercomparison project" - print "--file,-f : track dictionary file (defaults to 'track-dictionary.json' if omitted)" - print "--help, -h : print this message and exit." + print("meanie2tic.py [--file,-f] [--help,-h]") + print("Converts meanie3D track-dictionary.json into CSV files for Tracking Intercomparison project") + print("--file,-f : track dictionary file (defaults to 'track-dictionary.json' if omitted)") + print("--help, -h : print this message and exit.") sys.exit(1) return # ---------------------------------------------------------------------------- @@ -182,7 +182,7 @@ def main(): argv = sys.argv[1:] opts, args = getopt.getopt(argv, "f:h", ["file","help"]) except getopt.GetoptError as detail: - print detail + print(f"{detail}") sys.exit(2) if len(sys.argv) < 2: diff --git a/python/meanie3D/app/analysis.py b/python/meanie3D/app/analysis.py index 7a45d05..7aa93db 100644 --- a/python/meanie3D/app/analysis.py +++ b/python/meanie3D/app/analysis.py @@ -56,11 +56,11 @@ def usage(): Prints help and exits :return: ''' - print "meanie3D-trackgraph -f " - print "Analyses a track dictionary and shows a track graph with splits, merges etc." - print "-f : tracking dictionary file" - print "--help, -h : print this message and exits." - print "--version : prints the version information and exits" + print("meanie3D-trackgraph -f ") + print("Analyses a track dictionary and shows a track graph with splits, merges etc.") + print("-f : tracking dictionary file") + print("--help, -h : print this message and exits.") + print("--version : prints the version information and exits") sys.exit(1) return @@ -76,7 +76,7 @@ def run(): argv = sys.argv[1:] opts, args = getopt.getopt(argv, "c:f:s:o:r:h", ["resume","help","version","start=","end="]) except getopt.GetoptError as detail: - print detail + print(f"{detail}") sys.exit(2) num_params = 0 diff --git a/python/meanie3D/app/external.py b/python/meanie3D/app/external.py index 1ff83a5..349e3e0 100644 --- a/python/meanie3D/app/external.py +++ b/python/meanie3D/app/external.py @@ -101,14 +101,27 @@ def locateCommandInPath(command, path, recurse): return result -## Attempts to locate the given executables in the given filesystem paths. -# -# \param command_list - list of strings containing the commands -# \param path_list - list of strings containing the search paths -# \param recurse - boolean. If the method recurses into each path. -# \return a dictionary containing the command names mapping to the command paths -# \throws IOError if a command can not be located or is not executable +def hasCommand(command): + """ + Tests if the command is available + @param command name + @returns True or False + """ + try: + locateCommands([command]) + except IOError: + return False + + def locateCommandsInPaths(command_list, path_list, recurse): + """ + Attempts to locate the given executables in the given filesystem paths. + @param command_list - list of strings containing the commands + @param path_list - list of strings containing the search paths + @param recurse - boolean. If the method recurses into each path. + @returns a dictionary containing the command names mapping to the command paths + @throws IOError if a command can not be located or is not executable + """ for command in command_list: result = COMMAND_NOT_FOUND for path in path_list: diff --git a/python/meanie3D/app/postprocessing.py b/python/meanie3D/app/postprocessing.py index 0d56fce..0083205 100644 --- a/python/meanie3D/app/postprocessing.py +++ b/python/meanie3D/app/postprocessing.py @@ -39,11 +39,11 @@ # ---------------------------------------------------------------------------- -external.locateCommands(['meanie3D-cfm2vtk', 'meanie3D-trackstats', 'gnuplot']) if __have_visit__: import visit external.locateCommands(['visit']) + # ---------------------------------------------------------------------------- def check_configuration(configuration): @@ -56,10 +56,10 @@ def check_configuration(configuration): # Check the configuration dimensions = utils.getValueForKeyPath(configuration, 'data.dimensions') if (not dimensions): - print "ERROR:configuration must contain 'dimensions'" + print (".*") return -1 if (not len(dimensions) in [2, 3]): - print "ERROR:Can only process 2D or 3D" + print (".*") return -1 # Check that visualiseTracks has .vtk to work from @@ -69,7 +69,7 @@ def check_configuration(configuration): if tracks: if utils.getValueForKeyPath(tracks, 'visualiseTracks') and not utils.getValueForKeyPath(tracks, 'meanie3D-trackstats.vtk_tracks'): - print "WARNING: tracks.visualiseTracks = True but tracks.meanie3D-trackstats.vtk_tracks = False. Correcting." + print (".*") utils.setValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.vtk_tracks', True) # Complement vtk_dimensions to make our life a little easier down the road. @@ -83,7 +83,7 @@ def check_configuration(configuration): # Make sure that plotStats has gnuplot files to work with if utils.getValueForKeyPath(tracks, 'plotStats') and not utils.getValueForKeyPath(tracks, 'meanie3D-trackstats.gnuplot'): - print "WARNING: track.plot_stats = True but tracks.meanie3D-trackstats.gnuplot = False. Correcting." + print (".*") utils.setValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.gnuplot', True) @@ -98,19 +98,19 @@ def run_trackstats(configuration, directory): ''' pconf = configuration['postprocessing'] if not 'tracks' in pconf: - print "Skipping meanie3D-trackstats because 'tracks' configuration is missing" + print (".*") return False tracks = pconf['tracks'] if not 'meanie3D-trackstats' in tracks: - print "Skipping meanie3D-trackstats because 'tracks.meanie3D-trackstats' configuration is missing" + print (".*") return False if utils.getValueForKeyPath(configuration,'skip_trackstats'): - print "Skipping meanie3D-trackstats because --skip-trackstats is present" + print (".*") return False - print "Running meanie3D-trackstats for %s" % directory + print (f"{directory}") conf = tracks['meanie3D-trackstats'] os.chdir(directory) @@ -161,16 +161,16 @@ def run_trackstats(configuration, directory): params.append("-s netcdf") if configuration['resume'] and haveFiles: - print "Skipping meanie3D-trackstats" + print (".*") os.chdir("..") return True return_code = -1 try: - print "meanie3D-trackstats %s" % (" ".join(params)) + print(f'{".*".join(params)}') return_code = external.execute_command("meanie3D-trackstats", " ".join(params), silent=True) except: - print "ERROR:%s" % sys.exc_info()[0] + print(f'{(".*") % sys.exc_info()[0]}') raise os.chdir("..") @@ -194,7 +194,7 @@ def plot_trackstats(configuration, directory): return False conf = tracks['meanie3D-trackstats'] - print "Plotting .eps files for %s" % directory + print (".*") % directory os.chdir(directory) @@ -249,7 +249,7 @@ def plot_trackstats(configuration, directory): try: return_code = external.execute_command("gnuplot", "plot_stats.gp") except: - print "ERROR:%s" % sys.exc_info()[0] + print (".*") % sys.exc_info()[0] os.chdir("..") return (return_code == 0) @@ -266,7 +266,7 @@ def plot_stats_comparison(configuration): ''' scales = configuration['scales'] if not scales: - print "No scales found, no comparison is done." + print (".*") return False pconf = configuration['postprocessing'] @@ -278,7 +278,7 @@ def plot_stats_comparison(configuration): conf = utils.getValueForKeyPath(pconf, 'tracks.meanie3D-trackstats') - print "Plotting .eps files for comparison" + print (".*") # create a tempfile for gnuplot f = open('plot_stats_comparison.gp', 'w') @@ -359,7 +359,7 @@ def plot_stats_comparison(configuration): try: return_code = external.execute_command("gnuplot", "plot_stats_comparison.gp") except: - print "ERROR:%s" % sys.exc_info()[0] + print (".*") % sys.exc_info()[0] os.chdir("..") return (return_code == 0) @@ -371,7 +371,7 @@ def run_comparison(configuration): :param configuration: :return: ''' - print "Running cross-scale comparison" + print (".*") return plot_stats_comparison(configuration) @@ -389,13 +389,13 @@ def compile_and_run_template(templatePath, configuration, replacements, director ''' scriptFilename = tempfile.mktemp() + ".py" # scriptFilename = os.path.abspath("generated.py") - # print "\tWriting python script for visualisation to: " + scriptFilename + # print (".*") + scriptFilename with open(templatePath) as infile, open(scriptFilename, 'w') as outfile: for line in infile: for src, target in replacements.iteritems(): line = line.replace(src, target) outfile.write(line) - # print "\tDone." + # print (".*") # Compile command line params for visualisation params = "-s %s" % scriptFilename @@ -408,10 +408,10 @@ def compile_and_run_template(templatePath, configuration, replacements, director # Run visualisation returnCode = -1 try: - print "Executing visit: " + params + print (".*") + params returnCode = external.execute_command('visit', params, silent=True) except: - print "ERROR:%s" % sys.exc_info()[0] + print (".*") % sys.exc_info()[0] # Change back out os.chdir('..') @@ -427,9 +427,9 @@ def visualise_tracks(configuration, directory): :param directory: :return: ''' - print "Visualising tracks for %s" % directory + print (".*") % directory if not __have_visit__: - print "Visit not found. Skipping." + print (".*") return home = os.path.abspath(os.path.dirname(meanie3D.__file__) + os.path.sep + os.path.pardir) templatePath = home + os.path.sep + os.path.sep.join(("meanie3D", "resources", "tracks_visit.py")) @@ -452,9 +452,9 @@ def visualise_clusters(configuration, directory): :param directory: :return: ''' - print "Visualising clusters for %s" % directory + print (".*") % directory if not __have_visit__: - print "Visit not found. Skipping." + print (".*") return home = os.path.abspath(os.path.dirname(meanie3D.__file__) + os.path.sep + os.path.pardir) templatePath = home + os.path.sep + os.path.sep.join(("meanie3D", "resources", "clusters_visit.py")) @@ -500,7 +500,7 @@ def cleanup(configuration, directory): :param directory: :return: ''' - print "Cleaning temporary files for %s" % directory + print (".*") % directory os.chdir(directory) if utils.getValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.vtk_tracks'): for filename in glob.glob('*.vtk'): @@ -541,23 +541,23 @@ def run(configuration): else: directories = ['clustering'] - print "Processing directories: %s" % str(directories) + print (".*") % str(directories) for directory in directories: if os.path.isdir(directory): - print "Processing %s" % directory + print (".*") % directory else: - print "ERROR:%s is not a directory!" % directory + print (".*") % directory continue # run the track statistics if configuration['time_operations']: - print "Running trackstats ..." + print (".*") start_time = time.time() if (run_trackstats(configuration, directory)): if configuration['time_operations']: - print "Finished. (%.2f seconds)" % (time.time()-start_time) + print (".*") % (time.time()-start_time) # Copy HTML files copy_html_files(configuration, directory) @@ -570,24 +570,24 @@ def run(configuration): if __have_visit__ and utils.getValueForKeyPath(configuration, 'postprocessing.tracks.visualiseTracks'): if configuration['time_operations']: - print "Visualising tracks ..." + print (".*") start_time = time.time() visualise_tracks(configuration, directory) if configuration['time_operations']: - print "Finished. (%.2f seconds)" % (time.time()-start_time) + print (".*") % (time.time()-start_time) if __have_visit__ and utils.getValueForKeyPath(configuration, 'postprocessing.clusters.visualiseClusters'): if configuration['time_operations']: - print "Visualising clusters ..." + print (".*") start_time = time.time() visualise_clusters(configuration, directory) if configuration['time_operations']: - print "Finished. (%.2f seconds)" % (time.time()-start_time) + print (".*") % (time.time()-start_time) cleanup(configuration, directory) diff --git a/python/meanie3D/app/tracking.py b/python/meanie3D/app/tracking.py index d0bbfef..56e0595 100644 --- a/python/meanie3D/app/tracking.py +++ b/python/meanie3D/app/tracking.py @@ -179,13 +179,13 @@ def run(config,time_index): # the first time step if time_index <= 0: # TODO: consider scenario where user opted not to remove previous results? - print "Removing results from previous runs" + print (".*") utils.create_ouput_directories(output_dir) else: resume_at_index = utils.number_of_netcdf_files(output_dir+"/netcdf") if time_index >= 0 and time_index < resume_at_index: return - print "Resuming at index " + str(resume_at_index) + print (".*") + str(resume_at_index) # Get a list of the files we need to process. source = utils.getValueForKeyPath(config,'source_directory') @@ -214,9 +214,9 @@ def run(config,time_index): cluster_file = cluster_file + "-" + str(time_index) cluster_file += "-clusters.nc" - print "-------------------------------------------------------------------------------------" - print "Processing " + netcdf_file - print "-------------------------------------------------------------------------------------" + print (".*") + print (".*") + netcdf_file + print (".*") # ---------------------------------------------- # Clustering @@ -231,7 +231,7 @@ def run(config,time_index): run_count = run_count + 1 continue - print "-- Clustering --" + print (".*") # build the clustering command params = detect_params @@ -280,11 +280,11 @@ def run(config,time_index): if config['time_operations']: start_time = time.time() - print "meanie3D-detect " + params + print (".*") + params external.execute_command("meanie3D-detect",params,True) if config['time_operations']: - print "Finished. (%.2f seconds)" % (time.time()-start_time) + print (".*") % (time.time()-start_time) # ---------------------------------------------- @@ -300,7 +300,7 @@ def run(config,time_index): else: logfile = output_dir+"/log/tracking_" + str(time_index)+".log" - print "-- Tracking --" + print (".*") params = tracking_params + " --previous %s --current %s " \ % (os.path.abspath(last_cluster_file),os.path.abspath(cluster_file)) params = params + " > " + os.path.abspath(logfile) @@ -309,11 +309,11 @@ def run(config,time_index): if config['time_operations']: start_time = time.time() - print "meanie3D-track" + params + print (".*") + params external.execute_command("meanie3D-track",params,True) if config['time_operations']: - print "Finished. (%.2f seconds)" % (time.time()-start_time) + print (".*") % (time.time()-start_time) # keep track last_cluster_file = cluster_file @@ -323,8 +323,8 @@ def run(config,time_index): # Clean out the trash if there is any if config['cleanup_vtk']: - print "Cleaning up *.vt*" + print (".*") call("rm -f *.vt*", shell=True) - print "Done." + print (".*") return \ No newline at end of file diff --git a/python/meanie3D/app/utils.py b/python/meanie3D/app/utils.py index a1cfb63..7357141 100644 --- a/python/meanie3D/app/utils.py +++ b/python/meanie3D/app/utils.py @@ -177,7 +177,7 @@ def setValueForKeyPath(object,keypath,value): # in visualisation objects. Try to set an enumerated value when hitting # this combination if type(value) is str and type(getattr(object,keypath)) is int: - print "Attempting to resolve constant" + print (".*") value = getattr(object,value) if type(object) is dict: @@ -271,7 +271,7 @@ def find(path,filename,requiredComponent=None): if result: return result except OSError as err: - print err + print(f"{err}") return None diff --git a/python/meanie3D/meanie3D_run.py b/python/meanie3D/meanie3D_run.py index 53a9228..6dc431a 100755 --- a/python/meanie3D/meanie3D_run.py +++ b/python/meanie3D/meanie3D_run.py @@ -36,29 +36,29 @@ import meanie3D.app.postprocessing # Make sure the C++ executables are installed -meanie3D.app.external.locateCommands(["meanie3D-detect","meanie3D-track","meanie3D-cfm2vtk","meanie3D-trackstats","rm"]) +meanie3D.app.external.locateCommands(["meanie3D-detect","meanie3D-track","meanie3D-trackstats","rm"]) # ---------------------------------------------------------------------------- ## Prints usage and exits # def usage(): - print "meanie3D -c= -f= [-s=] [--start=t1 --end=t2] [--resume,-r] [--help,-h] [--version] [--time-operations]" - print "runs a complete set of netcdf files through the clustering/tracking" - print "-c : json config file specifying variables etc." - print "-f : directory containing the files. It is assumed that" - print " the files are in the correct order when sorted alphabetically." - print "--scales,-s : comma separated list of scale parameters." - print "--ranges,-r : comma separated list of bandwidths, one for each spatial and value range variable." - print "--json-example : prints an example .json configuration and exits" - print "--start index of time step to start with in files with time dimension" - print "--end index of time step to end at in files with time dimension" - print "--resume : if this flag is present, the algorithm assumes to resume" - print " operations where it last left off. If not present, previous" - print " results will be erased before starting" - print "--help, -h : print this message and exit." - print "--version : prints the version information and exits" - print "--time-operations: time operations and print out their elapsed time" - print "--skip-trackstats: skip meanie3D-trackstats, even if the configuration is present" + print("meanie3D -c= -f= [-s=] [--start=t1 --end=t2] [--resume,-r] [--help,-h] [--version] [--time-operations]") + print("runs a complete set of netcdf files through the clustering/tracking") + print("-c : json config file specifying variables etc.") + print("-f : directory containing the files. It is assumed that") + print(" the files are in the correct order when sorted alphabetically.") + print("--scales,-s : comma separated list of scale parameters.") + print("--ranges,-r : comma separated list of bandwidths, one for each spatial and value range variable.") + print("--json-example : prints an example .json configuration and exits") + print("--start index of time step to start with in files with time dimension") + print("--end index of time step to end at in files with time dimension") + print("--resume : if this flag is present, the algorithm assumes to resume") + print(" operations where it last left off. If not present, previous") + print(" results will be erased before starting") + print("--help, -h : print this message and exit.") + print("--version : prints the version information and exits") + print("--time-operations: time operations and print out their elapsed time") + print("--skip-trackstats: skip meanie3D-trackstats, even if the configuration is present") sys.exit(1) return # ---------------------------------------------------------------------------- @@ -67,13 +67,13 @@ def usage(): ## Prints the configuration file .json file description and exits # def print_configuration_format(): - print '''{''' - print ''' "description" : "...", /* Description of file content */''' - print ''' "data" : {...}, /* call --help config.data for help on this section */''' - print ''' "detection" : {...}, /* call --help config.detection for help on this section */''' - print ''' "tracking" : {...}, /* call --help config.tracking for help on this section */''' - print ''' "postprocessing" : {...} /* call --help config.postprocessing for help on this section */''' - print '''}''' + print('{') + print(' "description" : "...", /* Description of file content */') + print(' "data" : {...}, /* call --help config.data for help on this section */') + print(' "detection" : {...}, /* call --help config.detection for help on this section */') + print(' "tracking" : {...}, /* call --help config.tracking for help on this section */') + print(' "postprocessing" : {...} /* call --help config.postprocessing for help on this section */') + print('}') sys.exit(1) # ---------------------------------------------------------------------------- @@ -81,7 +81,7 @@ def print_configuration_format(): ## Prints version info and exits # def print_version(): - print "meanie3D: " + meanie3D.getVersion() + "\n" + print("meanie3D: " + meanie3D.getVersion() + "\n") sys.exit(1) return # ---------------------------------------------------------------------------- @@ -105,7 +105,7 @@ def main(): "scales="] opts, args = getopt.getopt(argv, "c:f:s:o:r:h", long_args) except getopt.GetoptError as detail: - print detail + print(f"{detail}") sys.exit(2) scales = [] @@ -182,11 +182,11 @@ def main(): if start_time_index != -1 or end_time_index != -1 : if (start_time_index == -1 and end_time_index != -1) or (start_time_index != -1 and end_time_index == -1): - print "start-time-index and end-time-index must both be set" + print("start-time-index and end-time-index must both be set") sys.exit(2) if start_time_index > end_time_index: - print "start-time-index is after end-time-index!" + print("start-time-index is after end-time-index!") sys.exit(2) uses_time = True @@ -195,9 +195,9 @@ def main(): configuration = meanie3D.app.utils.load_configuration(config_file); - print "-------------------------------------------------------------------------------------" - print "Configuration: " + configuration['description'] - print "-------------------------------------------------------------------------------------" + print("-------------------------------------------------------------------------------------") + print("Configuration: ") + configuration['description'] + print("-------------------------------------------------------------------------------------") # Enrich the configuration with env/command line stuff configuration["source_directory"] = os.path.abspath(netcdf_dir) diff --git a/python/meanie3D/visualisation/clusters.py b/python/meanie3D/visualisation/clusters.py index ffb3966..880a996 100644 --- a/python/meanie3D/visualisation/clusters.py +++ b/python/meanie3D/visualisation/clusters.py @@ -34,8 +34,7 @@ import utils from meanie3D.app import external -external.locateCommands(['meanie3D-cfm2vtk']) - +__have_vtk__ = external.hasCommand('meanie3D-cfm2vtk') # ------------------------------------------------------------------------------ # Adds clusters with names "*_infix_*.vtk" to the current visualisation window. @@ -57,7 +56,7 @@ def add_clusters(cluster_vtk_file,configuration): sys.stderr.write("ERROR:no value found for postprocessing.clusters.visit.cluster") return # plot the clusters - print "Adding clusters from file %s" % cluster_vtk_file + print (".*") % cluster_vtk_file utils.addPseudocolorPlot(cluster_vtk_file, clusterOptions) return @@ -77,12 +76,12 @@ def run(conf): clusterConf = utils.getValueForKeyPath(conf,'postprocessing.clusters') if not clusterConf: - print "No configuration for cluster postprocessing. Nothing to do." + print (".*") return 0 visitConf = utils.getValueForKeyPath(clusterConf,'visit') if not visitConf: - print "No configuration for visuals. Nothing to do." + print (".*") return 0 views = utils.getValueForKeyPath(visitConf,'views') @@ -94,10 +93,10 @@ def run(conf): utils.setAnnotations(conf,'postprocessing.clusters.visit.annotations') if not utils.getValueForKeyPath(conf,'resume'): - print "Removing results from previous runs" + print (".*") subprocess.call("rm -rf images movies *.vtk *.vtr *tracking*.png *source*.png", shell=True) else: - print "Removing intermediary files from previous runs" + print (".*") subprocess.call("rm -f *.vtk *.vtr *.vtu", shell=True) # Figure out the ending for the cluster vtk files @@ -110,12 +109,12 @@ def run(conf): # Glob the netcdf directory or find the single file uses_time = utils.getValueForKeyPath(conf,'uses_time') - print "Current work directory: " + os.path.abspath(os.getcwd()) + print (".*") + os.path.abspath(os.getcwd()) if uses_time: - print "Processing file " + conf['source_directory'] + print (".*") + conf['source_directory'] netcdf_file = conf['source_directory'] else: - print "Processing files in directory " + conf['source_directory'] + print (".*") + conf['source_directory'] netcdf_files = sorted(glob.glob(conf['source_directory']+"/*.nc")) # Keep track of number of images to allow @@ -154,12 +153,12 @@ def run(conf): label_vtk_file = basename + "-clusters-centers.vtk" displacement_vtk_file = basename + "-clusters-displacements.vtk" - print "netcdf_file = " + netcdf_file - print "cluster_file = " + cluster_file + print (".*") + netcdf_file + print (".*") + cluster_file # check if the files both exist if not os.path.exists(cluster_file): - print "Cluster file does not exist. Skipping." + print (".*") continue # predict the filenames for checking on resume @@ -171,10 +170,10 @@ def run(conf): if conf['resume'] == True: exists = utils.images_exist(visitConf['views'],"source",image_count) if exists == "all": - print "Source visualization " + number_postfix + " exists. Skipping." + print (".*") skip_source = True elif exists == "partial": - print "Deleting partial visualization " + number_postfix + print (".*") + number_postfix utils.delete_images(conf,"source",image_count) if skip_source == False: @@ -189,7 +188,7 @@ def run(conf): utils.add_datetime(clusterConf,netcdf_file,time_index) # Add source data and threshold it - print "Plotting source data ..." + print (".*") start_time = time.time() @@ -202,25 +201,25 @@ def run(conf): visit.DeleteAllPlots() visit.ClearWindow() - print " done. (%.2f seconds)" % (time.time()-start_time) + print (".*") % (time.time()-start_time) - if utils.getValueForKeyPath(clusterConf,'visualiseClusters'): + if utils.getValueForKeyPath(clusterConf,'visualiseClusters') and __have_vtk__: skip = False if conf['resume'] == True: exists = utils.images_exist(visitConf['views'],"tracking",image_count) if exists == "all": - print "Cluster visualization "+number_postfix+" exists. Skipping." + print (".*") skip = True elif exists == "partial": - print "Deleting partial cluster visualization " + number_postfix + print (".*") + number_postfix utils.delete_images(conf,"tracking",image_count) if skip == False: # Run the conversion - print "-- Converting clusters to .vtr --" + print (".*") start_time = time.time() params = "-f %s %s" \ % (cluster_file,utils.getValueForKeyPath(conf,'postprocessing.clusters.meanie3D-cfm2vtk')) @@ -232,9 +231,9 @@ def run(conf): params += " --vtk-dimensions=%s" % vtkDimString # pdb.set_trace(); - print "meanie3D-cfm2vtk %s" % params + print (".*") % params meanie3D.app.external.execute_command('meanie3D-cfm2vtk', params) - print " done. (%.2f seconds)" % (time.time()-start_time) + print (".*") % (time.time()-start_time) # Move cluster output file to individual file if uses_time: @@ -242,7 +241,7 @@ def run(conf): os.rename(cluster_vtk_file,cluster_vtk_file_dst) cluster_vtk_file = cluster_vtk_file_dst - print "-- Rendering cluster scene --" + print (".*") start_time = time.time() # Add ancillary background data @@ -274,7 +273,7 @@ def run(conf): visit.DrawPlots() utils.saveImagesForViews(views,"tracking") - print " done. (%.2f seconds)" % (time.time()-start_time) + print (".*") % (time.time()-start_time) # clean up @@ -312,7 +311,7 @@ def run(conf): utils.createMoviesForViews(views,"tracking", movieFormats) # clean up - print "Cleaning up ..." + print (".*") subprocess.call("mkdir images", shell=True) subprocess.call("mv *tracking_*.png images", shell=True) subprocess.call("mv *source_*.png images", shell=True) diff --git a/python/meanie3D/visualisation/tracks.py b/python/meanie3D/visualisation/tracks.py index 054e9bd..c1b5ece 100644 --- a/python/meanie3D/visualisation/tracks.py +++ b/python/meanie3D/visualisation/tracks.py @@ -44,7 +44,7 @@ def run(conf): visitConf = utils.getValueForKeyPath(conf,'postprocessing.tracks.visit') if not visitConf: - print "No configuration for visuals. Nothing to do." + print (".*") return 0 # Set up background gradient, axis labels etc. @@ -73,8 +73,8 @@ def run(conf): track_pattern = "*-track_*.vtk" list = sorted(glob.glob(track_pattern)) - print "Looking with pattern " + track_pattern - print "Found %d track files." % len(list) + print (".*") + track_pattern + print (".*") % len(list) count = 0; for trackFile in list: @@ -88,7 +88,7 @@ def run(conf): # Plot the actual track data file = conf['tracks_dir'] + os.path.sep + trackFile - print "Adding plot for " + file + print (".*") + file utils.addPseudocolorPlot(file,trackPlotConf) count = count + 1 @@ -101,10 +101,10 @@ def run(conf): trackPlotConf['PseudocolorAttributes']['legendFlag'] = legendFlag # pp.pprint(trackPlotConf) - print "Drawing plots" + print (".*") visit.DrawPlots() - print "Saving image to %s" % os.getcwd() + print (".*") % os.getcwd() utils.saveImage("tracks",0) os.chdir(currentDirectory) diff --git a/python/meanie3D/visualisation/utils.py b/python/meanie3D/visualisation/utils.py index 905916d..0f15ff2 100644 --- a/python/meanie3D/visualisation/utils.py +++ b/python/meanie3D/visualisation/utils.py @@ -35,22 +35,22 @@ ret_code, paths_string = meanie3D.app.external.execute_command('python','-c "import sys; print sys.path"', True) if ret_code == 0: - print "Attempting to locate python module netCDF4" + print (".*") result = meanie3D.app.utils.find_in_paths(["/usr/lib","/usr/local/lib"],"netCDF4","site-packages") if not result: result = meanie3D.app.utils.find_in_paths(["/usr/lib","/usr/local/lib"],"netCDF4","dist-packages") if not result: - print "Failed to locate python module netCDF4" + print (".*") exit(-1) else: - print "Found netCDF4 at %s" % result + print (".*") % result sys.path.append(os.path.split(result)[0]); - print "Python path after adding system search directories:" + print (".*") print(sys.path) else: - print "Failed to obtain system's python path" + print (".*") exit(-1) import netCDF4 @@ -371,7 +371,7 @@ def plotMapdata(configuration, path): if os.path.exists(mapFile): addPseudocolorPlots(mapFile, configuration, path + ".plots") else: - print "ERROR:could not find map file at " + mapFile + print (".*") + mapFile return @@ -537,11 +537,11 @@ def create_movie(basename, moviename): :param moviename: :return: ''' - print "Creating movie '" + moviename + "' from files '" + basename + "*.png ..." + print (".*") args = "-limit memory 4GB -delay 50 -quality 100 -dispose Background %s*.png %s" % (basename, moviename) - print "convert %s" % args + print (".*") % args meanie3D.app.external.execute_command('convert', args, False) - print "done." + print (".*") return @@ -671,9 +671,9 @@ def add_datetime(conf, netcdf_file,time_index): date = date.replace(microsecond=0) have_valid_time = True except ValueError as vi: - print "Error reading time information from file:" - print vi.message - print "Falling back on time index" + print (".*") + print (vi.message) + print (".*") date = time_index x = 0.725 @@ -714,7 +714,7 @@ def create_dual_panel(basename_left, basename_right, basename_combined): left_files = sorted(glob.glob(basename_left + "*.png")) right_files = sorted(glob.glob(basename_right + "*.png")) if len(left_files) != len(right_files): - print "ERROR:the two image series " + basename_left + "*.png and " + basename_right + "*.png have different lengths!" + print (".*") return # create a small image to blank the datestamp with From 2a46d8e243a7e7c7a81f9ff42848fba4708e8e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sat, 20 Jun 2020 17:19:56 +0100 Subject: [PATCH 13/65] Added release notes file. --- RELEASE_NOTES.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 RELEASE_NOTES.md diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 0000000..18ee0a5 --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,9 @@ +# meanie3D Versions + +## 1.6.1 +* Removed all git submodules. +* Revised the cmake to make all but the core functions optional. +* Upgrade to newest gcc and clang versions. +* Upgrade to Python 3.x +* Project has a working Dockerfile now. +* Updated documentation. \ No newline at end of file From 78256a67e6d012199c0010ea13e2f15288ff3b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sat, 20 Jun 2020 23:09:20 +0100 Subject: [PATCH 14/65] Various cmake fixes. --- CMakeLists.txt | 87 ++++++++++++++++---------------------------------- 1 file changed, 27 insertions(+), 60 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c1b45d5..de43f36 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,7 +55,6 @@ INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/include) # Options # ------------------------------------- OPTION(FOR_DOCKER "Build for dockerized version" OFF) -OPTION(WITH_OPENMP "Enable/Disable OpenMP" ON) OPTION(WITH_TESTS "Enable/Disable building of tests" OFF) OPTION(WITH_VTK "Enable/Disable visualisation code (requires VTK)" OFF) OPTION(WITH_RADOLAN_UTILS "Compile with radolan adaptor (requires libradolan)" OFF) @@ -63,16 +62,6 @@ OPTION(WITH_SATELLITE_UTILS "Compile with tools for satellite data" OFF) OPTION(WITH_KONRAD_UTILS "Compile with binaries for KONRAD tracks" OFF) OPTION(WITH_DOCS "Compile source code documentation" OFF) -# ------------------------------------- -# Submodules -# ------------------------------------- - -ADD_SUBDIRECTORY(python) -IF (WITH_RADOLAN_UTILS) - ADD_SUBDIRECTORY(radolan) - INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/radolan/include) -ENDIF() - # ------------------------------------- # PRESETS # ------------------------------------- @@ -125,12 +114,14 @@ ELSE() ENDIF () # ------------------------------------- - # Radolan Adaptor + # Radolan Utils # ------------------------------------- SET(RADOLAN_ENABLED ${WITH_RADOLAN_UTILS}) IF (RADOLAN_ENABLED) - MESSAGE(STATUS "Compiling with radolan adaptor.") + MESSAGE(STATUS "RADOLAN utilities enabled") ADD_DEFINITIONS(-DWITH_RADOLAN=1) + ELSE() + MESSAGE(STATUS "RADOLAN utilities disabled") ENDIF () # ------------------------------------- @@ -138,8 +129,10 @@ ELSE() # ------------------------------------- SET(KONRAD_ENABLED ${WITH_KONRAD_UTILS}) IF (KONRAD_ENABLED) - MESSAGE(STATUS "Adding binaries for working with KONRAD data") + MESSAGE(STATUS "KONRAD utilities enabled") ADD_DEFINITIONS(-DWITH_KONRAD_UTILS=1) + ELSE() + MESSAGE(STATUS "KONRAD utilities disabled") ENDIF () # ------------------------------------- @@ -147,19 +140,12 @@ ELSE() # ------------------------------------- SET(SATELLITE_ENABLED ${WITH_SATELLITE_UTILS}) IF (SATELLITE_ENABLED) - MESSAGE(STATUS "Adding binaries for working with EUMETSAT satellite data") + MESSAGE(STATUS "Satellite data utilities enabled") ADD_DEFINITIONS(-DWITH_SATELLITE=1) + ELSE() + MESSAGE(STATUS "Satellite data utilities disabled") ENDIF () - # ------------------------------------- - # OpenCV (legacy) - # ------------------------------------- - - SET(OPENCV_ENABLED ${WITH_OPENCV}) - IF (OPENCV_ENABLED) - MESSAGE(STATUS "OpenCV is enabled.") - ADD_DEFINITIONS(-DWITH_OPENCV=1) - ENDIF () ENDIF() # ------------------------------------- @@ -254,20 +240,9 @@ ELSE () INCLUDE_DIRECTORIES(${SHP_INCLUDE_DIR}) ENDIF () -# OpenMP -IF (OPENMP_DISABLED) - IF (CMAKE_COMPILER_IS_GNUCXX) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") - ELSE(CMAKE_COMPILER_IS_GNUCXX) - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") - ENDIF (CMAKE_COMPILER_IS_GNUCXX) -ENDIF (OPENMP_DISABLED) - # VTK IF (VTK_ENABLED) - FIND_PACKAGE(VTK 6.0 REQUIRED NO_MODULE) + FIND_PACKAGE(VTK 8.2 REQUIRED NO_MODULE) IF (VTK_FOUND) MESSAGE(STATUS "VTK found") INCLUDE(${VTK_USE_FILE}) @@ -279,24 +254,12 @@ ENDIF () FIND_PACKAGE(libradolan REQUIRED) IF (libradolan_FOUND) - MESSAGE(STATUS "libradolan enabled") + MESSAGE(STATUS "libradolan found") INCLUDE_DIRECTORIES(${libradolan_INCLUDE_DIR}) ELSE() MESSAGE(FATAL_ERROR "libradolan not found") ENDIF() -IF (KONRAD_ENABLED) - MESSAGE(STATUS "KONRAD utilities enabled") -ELSE() - MESSAGE(STATUS "KONRAD utilities disabled") -ENDIF() - -IF (SATELLITE_ENABLED) - MESSAGE(STATUS "OASE satellite data utilities enabled") -ELSE() - MESSAGE(STATUS "OASE satellite data utilities disabled") -ENDIF() - # ------------------------------------- # Binaries # ------------------------------------- @@ -418,6 +381,7 @@ ADD_LIBRARY(meanie3D SHARED include/meanie3D/weights/weight_function_factory_impl.h include/meanie3D/weights.h src/exceptions/CFFileConversionException.cpp + src/adaptors/Radolan.cpp src/kdtree/kdtree.c src/numericalrecipes/betai.c src/numericalrecipes/betacf.c @@ -561,8 +525,7 @@ TARGET_LINK_LIBRARIES(meanie3D ${Boost_LIBRARIES} ${VTK_LIBRARIES} ${Blitz_LIBRARY} - ${OpenCV_LIBRARIES} - ${OpenMP_RT_LIBRARIES}) + ${NETCDF_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D PROPERTIES LINKER_LANGUAGE CXX) # Detection tool @@ -576,7 +539,7 @@ TARGET_LINK_LIBRARIES(meanie3D-detect ${VTK_LIBRARIES} ${HDF5_LIBRARIES} ${NETCDF_LIBRARIES}) -SET_TARGET_PROPERTIES(meanie3D-detect PROPERTIES LINKER_LANGUAGE CXX) +#SET_TARGET_PROPERTIES(meanie3D-detect PROPERTIES LINKER_LANGUAGE CXX) # tracking tool @@ -702,7 +665,7 @@ IF (VTK_ENABLED) ${HDF5_LIBRARIES} ${OpenMP_RT_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-cfm2vtk PROPERTIES LINKER_LANGUAGE CXX) -ENDIF () +ENDIF (VTK_ENABLED) # ------------------------------------------------------------- # # Tests @@ -815,19 +778,23 @@ INSTALL(TARGETS meanie3D-track RUNTIME DESTINATION "/usr/local/bin") INSTALL(TARGETS meanie3D-trackstats RUNTIME DESTINATION "/usr/local/bin") INSTALL(TARGETS meanie3D-timestamp RUNTIME DESTINATION "/usr/local/bin") INSTALL(TARGETS meanie3D-minmax RUNTIME DESTINATION "/usr/local/bin") -IF (WITH_SATELLITE_UTILS) + +IF (SATELLITE_ENABLED) INSTALL(TARGETS meanie3D-satconv RUNTIME DESTINATION "/usr/local/bin") INSTALL(TARGETS meanie3D-parallax_correction RUNTIME DESTINATION "/usr/local/bin") ENDIF() -IF (WITH_KONRAD_UTILS) + +IF (KONRAD_ENABLED) INSTALL(TARGETS meanie3D-trackstats-conrad RUNTIME DESTINATION "/usr/local/bin") -ENDIF() -IF (WITH_RADOLAN_UTILS) +ENDIF (KONRAD_ENABLED) + +IF (RADOLAN_ENABLED) INSTALL(TARGETS meanie3D-radolan2cfm RUNTIME DESTINATION "/usr/local/bin") -ENDIF() -IF (WITH_VTK) +ENDIF (RADOLAN_ENABLED) + +IF (VTK_ENABLED) INSTALL(TARGETS meanie3D-cfm2vtk RUNTIME DESTINATION "/usr/local/bin") -ENDIF (WITH_VTK) +ENDIF (VTK_ENABLED) # ------------------------------------------------------------- # # Documentation From 5d9ab5cbb757c2ae487f56cf2ac702656dad757d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sat, 20 Jun 2020 23:35:18 +0100 Subject: [PATCH 15/65] Misc fixes. --- CMakeLists.txt | 3 ++- python/meanie3D/meanie3D_run.py | 2 +- src/executables/meanie3D-mapdata.cpp | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de43f36..993e3cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -591,7 +591,8 @@ TARGET_LINK_LIBRARIES(meanie3D-mapdata ${Boost_LIBRARIES} ${VTK_LIBRARIES} ${HDF5_LIBRARIES} - ${NETCDF_LIBRARIES}) + ${NETCDF_LIBRARIES} + ${SHP_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-mapdata PROPERTIES LINKER_LANGUAGE CXX) # timestamp update diff --git a/python/meanie3D/meanie3D_run.py b/python/meanie3D/meanie3D_run.py index 6dc431a..447abdb 100755 --- a/python/meanie3D/meanie3D_run.py +++ b/python/meanie3D/meanie3D_run.py @@ -196,7 +196,7 @@ def main(): configuration = meanie3D.app.utils.load_configuration(config_file); print("-------------------------------------------------------------------------------------") - print("Configuration: ") + configuration['description'] + print(f"Configuration: {configuration['description']}") print("-------------------------------------------------------------------------------------") # Enrich the configuration with env/command line stuff diff --git a/src/executables/meanie3D-mapdata.cpp b/src/executables/meanie3D-mapdata.cpp index 1fcfb68..5401eff 100644 --- a/src/executables/meanie3D-mapdata.cpp +++ b/src/executables/meanie3D-mapdata.cpp @@ -159,7 +159,6 @@ void parse_commmandline(program_options::variables_map vm, /* ******************************************************** */ /* Some helper methods */ - /* ******************************************************** */ template From 66f7222f2972f8caf43463c9a62650ba1d90b13d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sun, 21 Jun 2020 00:16:59 +0100 Subject: [PATCH 16/65] Fixing cmake problem with python. Fix import paths. More work on documentation. --- CMakeLists.txt | 6 ++++ README.md | 19 ++++++++--- python/meanie3D/app/postprocessing.py | 4 +-- python/meanie3D/app/tracking.py | 3 +- python/meanie3D/app/utils.py | 2 +- src/HOWTO.md | 45 +++++++++++++++++++++++++++ 6 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 src/HOWTO.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 993e3cc..3826432 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,6 +148,12 @@ ELSE() ENDIF() +# ------------------------------------- +# Submodules +# ------------------------------------- + +ADD_SUBDIRECTORY(python) + # ------------------------------------- # Compiler Flags # ------------------------------------- diff --git a/README.md b/README.md index 85ea58d..fe80854 100644 --- a/README.md +++ b/README.md @@ -51,19 +51,18 @@ Optionally, the following libraries may be used, if they are switched on with th Meanie3D uses OpenMP by default (-DWITH_OPENMP=1) and requires an OpenMP enabled compiler, such as: -* GNU 4.3 or better (Linux, Mac) +* GNU 9.0 or better (Linux, Mac) * OpenMP/LLVM (Mac) - a clang implementation supporting OpenMP (http://clang-omp.github.io). Install this compiler for optimal results on OSX. You can disable the OpenMP implementation adding the flags -DWITH_OPENMP=0 to your cmake call. -### Obtain and compile Meanie3D +### Build instructions Meanie3D uses CMAKE to generate makefiles. You can use CMAKE's abilities to generate IDE files if you prefer. Start by cloning the master branch (for an up-to date but possibly unstable version) or one of the stable releases. git clone https://github.com/meteo-ubonn/meanie3D.git - TODO: revise handling of map data If you want to download the OASE topology and mapdata file for visualisation, you can obtain this file by adding: @@ -163,7 +162,19 @@ layout is not considered and you have to make adjustments to the search paths, p your Find.cmake file to juergen.simon@uni-bonn.de so I can include your changes in the next release and maybe save someone else the trouble. -## More information +## Python package +In addition to the C++ binaries, the installation also uses pip3 to put a python package in place. +This adds an executable simply called `meanie3D`, which is a front-end to the core functions. It allows +you to put your clustering and tracking parameters down in the form of a configuration file. The entire +pipeline is handled based on this configuration. This is the recommended way to run the software. For +details on the configuration file format, see HOWTO.md. +## Visualisation +The C++ binaries produce certain files when VTK is enabled. In order to generate imagery or tracking +movies, meanie3D relies on executing python scripts in VisIt (). You will have to install this software +to get access to the visualization. For details on what is possible by scripting VisIt with python, +see the following tutorial: https://www.visitusers.org/index.php?title=VisIt-tutorial-Python-scripting + +## More information Check out the Main Wiki Page at http://git.meteo.uni-bonn.de/projects/meanie3d/wiki for details on how the method works, file formats and what the individual binaries do. \ No newline at end of file diff --git a/python/meanie3D/app/postprocessing.py b/python/meanie3D/app/postprocessing.py index 0083205..14397c9 100644 --- a/python/meanie3D/app/postprocessing.py +++ b/python/meanie3D/app/postprocessing.py @@ -34,8 +34,8 @@ from . import __visitPath__ import meanie3D -from meanie3D.app import utils -from meanie3D.app import external +from . import utils +from . import external # ---------------------------------------------------------------------------- diff --git a/python/meanie3D/app/tracking.py b/python/meanie3D/app/tracking.py index 56e0595..3fdb541 100644 --- a/python/meanie3D/app/tracking.py +++ b/python/meanie3D/app/tracking.py @@ -29,8 +29,7 @@ import time from subprocess import call -import external -import utils +from meanie3D.app import external, utils ## Checks the configuration for mandatory items etc. # \param The configuration to check. diff --git a/python/meanie3D/app/utils.py b/python/meanie3D/app/utils.py index 7357141..7efb6da 100644 --- a/python/meanie3D/app/utils.py +++ b/python/meanie3D/app/utils.py @@ -29,7 +29,7 @@ import pdb import shutil import sys -import external +from . import external def load_configuration(filename): diff --git a/src/HOWTO.md b/src/HOWTO.md new file mode 100644 index 0000000..9ad8f5f --- /dev/null +++ b/src/HOWTO.md @@ -0,0 +1,45 @@ +# How to use this software + +Please make sure the software is compiled and available by following the +instructions in the README.md file. + +## General overview +The process of tracking objects over time is generally comprised of two steps: +### Cluster detection +Using the mean-shift algorithm, objects are detected in your data. The objects are +the result first running the mean-shift algorithm with vectors discretized to snap +onto the data set's own grid. The vectors are then linked into graphs. Each graph +has a mode, which is a point or area where the graph ends. A cluster is made up of +all points in the data set that end at the same mode. +### Tracking +When clustering a data set that varies over time (Example: satellite data), it is +interesting to study the development of clusters over time. For this to happen, the +clusters from time `t1` must be identified again in `t2` (if possible). The meanie3D +package provides a tool for doing this based on a weighed sum of three factors: +* Proximity + This is a measure of the spatial distance of the clusters modes between `t1` and `t2` + (the closer, the more likely it is a match) +* Cluster size + This is based on the assumption that the tracked phenomenon change slowly enough. + The more equal in size clusters from `t1` and `t2` are, the more likely a match. +* Histogram + A selected value is used and histogram of it's values is levvied for each cluster. + When comparing the `t1` and `t2` clusters, those histograms are compared using a + Kendall Rank Coefficient. The more highly correllated the histograms are, the more + likely a match + +All three factors are summed (with weights) to arrive at the final numbers. The clusters +are then paired off in the descending order of the match likelihood. The result is, that +clusters from `t2` are given identical ids of clusters in `t1`. The set of clusters with +the same ID over time constitute a track. + +## Python frontend + +## Docker version +The docker version does not support any of the visualisation features. This has +something to do with the fact, that the VisIt application used to generate imagery +can not run in headless mode. Therefore the only functions available in the docker +version are: +* Cluster detection +* Tracking +* Track statistics \ No newline at end of file From c3adce9ceb4cd303888388c9c75ad8880ba3d5ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sun, 21 Jun 2020 00:19:32 +0100 Subject: [PATCH 17/65] Docu location fixed. --- src/HOWTO.md => HOWTO.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/HOWTO.md => HOWTO.md (100%) diff --git a/src/HOWTO.md b/HOWTO.md similarity index 100% rename from src/HOWTO.md rename to HOWTO.md From 2fd0806f095675846948c8887ebca87b995f9052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sun, 21 Jun 2020 11:59:42 +0100 Subject: [PATCH 18/65] CMake outfitted with simple presets. Updated README. Python downgraded to visit's 2.7 restriction. --- CMakeLists.txt | 264 ++++++++++++------- HOWTO.md | 20 +- README.md | 44 ++++ include/meanie3D/clustering/conrad_cluster.h | 4 +- python/CMakeLists.txt | 14 +- python/meanie3D/app/__init__.py | 3 +- 6 files changed, 241 insertions(+), 108 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3826432..9dfbc97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,110 +54,185 @@ INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/include) # ------------------------------------- # Options # ------------------------------------- -OPTION(FOR_DOCKER "Build for dockerized version" OFF) OPTION(WITH_TESTS "Enable/Disable building of tests" OFF) -OPTION(WITH_VTK "Enable/Disable visualisation code (requires VTK)" OFF) +OPTION(WITH_VTK "Enable/Disable visualisation code (requires VisIt and VTK)" OFF) OPTION(WITH_RADOLAN_UTILS "Compile with radolan adaptor (requires libradolan)" OFF) OPTION(WITH_SATELLITE_UTILS "Compile with tools for satellite data" OFF) OPTION(WITH_KONRAD_UTILS "Compile with binaries for KONRAD tracks" OFF) OPTION(WITH_DOCS "Compile source code documentation" OFF) +OPTION(WITH_PYTHON "Add the python frontend" ON) # ------------------------------------- # PRESETS # ------------------------------------- -if (FOR_DOCKER) - MESSAGE("Setting preset options for building docker container") +IF(PRESET MATCHES "docker") + MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "Release") - ADD_DEFINITIONS(-DWITH_OPENMP=1) - ADD_DEFINITIONS(-DWITH_RADOLAN=0) - ADD_DEFINITIONS(-DWITH_SHAPELIB=0) - ADD_DEFINITIONS(-DWITH_VTK=0) - ADD_DEFINITIONS(-DWITH_TESTS=0) - ADD_DEFINITIONS(-DWITH_KONRAD_UTILS=0) - ADD_DEFINITIONS(-DWITH_SATELLITE=0) + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED OFF) + SET(TESTS_ENABLED OFF) + SET(RADOLAN_ENABLED OFF) + SET(SATELLITE_ENABLED OFF) + SET(KONRAD_ENABLED OFF) + SET(DOCS_ENABLED OFF) +ELSEIF (PRESET MATCHES "dev-all") + MESSAGE(STATUS "Selected preset ${PRESET}") + SET(CMAKE_BUILD_TYPE "Debug") + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED ON) + SET(TESTS_ENABLED ON) + SET(RADOLAN_ENABLED ON) + SET(SATELLITE_ENABLED ON) + SET(KONRAD_ENABLED ON) + SET(DOCS_ENABLED ON) +ELSEIF(PRESET MATCHES "dev-core") + MESSAGE(STATUS "Selected preset ${PRESET}") + SET(CMAKE_BUILD_TYPE "Debug") + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED OFF) + SET(TESTS_ENABLED OFF) + SET(RADOLAN_ENABLED OFF) + SET(SATELLITE_ENABLED OFF) + SET(KONRAD_ENABLED OFF) + SET(DOCS_ENABLED ON) +ELSEIF (PRESET MATCHES "dev-vtk") + MESSAGE(STATUS "Selected preset ${PRESET}") + SET(CMAKE_BUILD_TYPE "Debug") + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED ON) + SET(TESTS_ENABLED OFF) + SET(RADOLAN_ENABLED OFF) + SET(SATELLITE_ENABLED OFF) + SET(KONRAD_ENABLED OFF) + SET(DOCS_ENABLED ON) +ELSEIF (PRESET MATCHES "dev-all") + MESSAGE(STATUS "Selected preset ${PRESET}") + SET(CMAKE_BUILD_TYPE "Debug") + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED ON) + SET(TESTS_ENABLED ON) + SET(RADOLAN_ENABLED ON) + SET(SATELLITE_ENABLED ON) + SET(KONRAD_ENABLED ON) + SET(DOCS_ENABLED ON) +ELSEIF (PRESET MATCHES "prod-core") + MESSAGE(STATUS "Selected preset ${PRESET}") + SET(CMAKE_BUILD_TYPE "Release") + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED OFF) + SET(TESTS_ENABLED OFF) + SET(RADOLAN_ENABLED OFF) + SET(SATELLITE_ENABLED OFF) + SET(KONRAD_ENABLED OFF) + SET(DOCS_ENABLED OFF) +ELSEIF (PRESET MATCHES "prod-vtk") + MESSAGE(STATUS "Selected preset ${PRESET}") + SET(CMAKE_BUILD_TYPE "Release") + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED ON) + SET(TESTS_ENABLED OFF) + SET(RADOLAN_ENABLED OFF) + SET(SATELLITE_ENABLED OFF) + SET(KONRAD_ENABLED OFF) + SET(DOCS_ENABLED OFF) ELSE() - # ------------------------------------- - # Multithreading / Parallelism Choices - # ------------------------------------- + IF (NOT ${CMAKE_BUILD_TYPE}) + SET(CMAKE_BUILD_TYPE "Release") + ENDIF() SET(OPENMP_ENABLED ${WITH_OPENMP}) - IF (OPENMP_ENABLED) - MESSAGE(STATUS "OpenMP is enabled") - ADD_DEFINITIONS(-DWITH_OPENMP=1) - ELSE () - MESSAGE(STATUS "OpenMP is disabled") - ENDIF () - - # ------------------------------------- - # Visualization on/off - # ------------------------------------- + SET(PYTHON_ENABLED ${WITH_PYTHON}) + SET(TESTS_ENABLED ${WITH_TESTS}) SET(VTK_ENABLED ${WITH_VTK}) - IF (VTK_ENABLED) - MESSAGE(STATUS "VTK is enabled") - ADD_DEFINITIONS(-DWITH_VTK=1) - ELSE () - MESSAGE(STATUS "VTK is disabled") - ENDIF () + SET(RADOLAN_ENABLED ${WITH_RADOLAN_UTILS}) + SET(KONRAD_ENABLED ${WITH_KONRAD_UTILS}) + SET(SATELLITE_ENABLED ${WITH_SATELLITE_UTILS}) + SET(DOCS_ENABLED ${WITH_DOCS}) +ENDIF() - # ------------------------------------- - # Automated tests yes/no - # ------------------------------------- - SET(TESTS_ENABLED ${WITH_TESTS}) - IF (TESTS_ENABLED) - MESSAGE(STATUS "Tests are enabled.") - ENABLE_TESTING() - ADD_SUBDIRECTORY(gtest-1.7.0) - INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0) - INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0/include) - ELSE () - MESSAGE(STATUS "Tests are disabled. Use -DWITH_TESTS=ON/OFF to switch tests on or off.") - ENDIF () +# ------------------------------------- +# Multithreading / Parallelism Choices +# ------------------------------------- +IF (OPENMP_ENABLED) + MESSAGE(STATUS "OpenMP is enabled") + ADD_DEFINITIONS(-DWITH_OPENMP=1) +ELSE () + MESSAGE(STATUS "OpenMP is disabled") +ENDIF () - # ------------------------------------- - # Radolan Utils - # ------------------------------------- - SET(RADOLAN_ENABLED ${WITH_RADOLAN_UTILS}) - IF (RADOLAN_ENABLED) - MESSAGE(STATUS "RADOLAN utilities enabled") - ADD_DEFINITIONS(-DWITH_RADOLAN=1) - ELSE() - MESSAGE(STATUS "RADOLAN utilities disabled") - ENDIF () +# ------------------------------------- +# Visualization on/off +# ------------------------------------- +IF (VTK_ENABLED) + MESSAGE(STATUS "Visualization is enabled") + ADD_DEFINITIONS(-DWITH_VTK=1) +ELSE () + MESSAGE(STATUS "Visualization is disabled") +ENDIF () - # ------------------------------------- - # KONRAD - # ------------------------------------- - SET(KONRAD_ENABLED ${WITH_KONRAD_UTILS}) - IF (KONRAD_ENABLED) - MESSAGE(STATUS "KONRAD utilities enabled") - ADD_DEFINITIONS(-DWITH_KONRAD_UTILS=1) - ELSE() - MESSAGE(STATUS "KONRAD utilities disabled") - ENDIF () +# ------------------------------------- +# Automated tests yes/no +# ------------------------------------- +IF (TESTS_ENABLED) + MESSAGE(STATUS "Tests are enabled.") + ENABLE_TESTING() + ADD_DEFINITIONS(-DWITH_TESTS=1) + ADD_SUBDIRECTORY(gtest-1.7.0) + INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0) + INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0/include) +ELSE () + MESSAGE(STATUS "Tests are disabled. Use -DWITH_TESTS=ON/OFF to switch tests on or off.") +ENDIF () - # ------------------------------------- - # SATELLITE - # ------------------------------------- - SET(SATELLITE_ENABLED ${WITH_SATELLITE_UTILS}) - IF (SATELLITE_ENABLED) - MESSAGE(STATUS "Satellite data utilities enabled") - ADD_DEFINITIONS(-DWITH_SATELLITE=1) - ELSE() - MESSAGE(STATUS "Satellite data utilities disabled") - ENDIF () +# ------------------------------------- +# Radolan Utils +# ------------------------------------- +IF (RADOLAN_ENABLED) + MESSAGE(STATUS "RADOLAN utilities enabled") + ADD_DEFINITIONS(-DWITH_RADOLAN_UTILS=1) +ELSE() + MESSAGE(STATUS "RADOLAN utilities disabled") +ENDIF () -ENDIF() +# ------------------------------------- +# KONRAD +# ------------------------------------- +IF (KONRAD_ENABLED) + MESSAGE(STATUS "KONRAD utilities enabled") + ADD_DEFINITIONS(-DWITH_KONRAD_UTILS=1) +ELSE() + MESSAGE(STATUS "KONRAD utilities disabled") +ENDIF () # ------------------------------------- -# Submodules +# SATELLITE # ------------------------------------- +IF (SATELLITE_ENABLED) + MESSAGE(STATUS "Satellite data utilities enabled") + ADD_DEFINITIONS(-DWITH_SATELLITE=1) +ELSE() + MESSAGE(STATUS "Satellite data utilities disabled") +ENDIF () -ADD_SUBDIRECTORY(python) +# ------------------------------------- +# Python frontend +# ------------------------------------- +IF (PYTHON_ENABLED) + MESSAGE(STATUS "Installing python frontend") + ADD_DEFINITIONS(-DWITH_PYTHON=1) + ADD_SUBDIRECTORY(python) +ENDIF(PYTHON_ENABLED) # ------------------------------------- # Compiler Flags # ------------------------------------- - MESSAGE(STATUS "CMAKE_C_COMPILER = ${CMAKE_C_COMPILER}") MESSAGE(STATUS "CMAKE_CXX_COMPILER = ${CMAKE_CXX_COMPILER}") MESSAGE(STATUS "Compiler ID = ${CMAKE_CXX_COMPILER_ID}") @@ -254,7 +329,7 @@ IF (VTK_ENABLED) INCLUDE(${VTK_USE_FILE}) INCLUDE_DIRECTORIES(${VTK_INCLUDE_DIRS}) ELSE () - MESSAGE(FATAL_ERROR "VTK not found (http://www.vtk.org/VTK/resources/software.html)") + MESSAGE(FATAL_ERROR "VTK not found (http://www.vtk.org/vtk/resources/software.html)") ENDIF () ENDIF () @@ -626,21 +701,20 @@ ENDIF(RADOLAN_ENABLED) # trackstats KONRAD -IF (WITH_KONRAD_UTILS) +IF (KONRAD_ENABLED) ADD_EXECUTABLE(meanie3D-trackstats-conrad meanie3D src/executables/meanie3D-trackstats-conrad.cpp) TARGET_LINK_LIBRARIES(meanie3D-trackstats-conrad meanie3D ${VTK_LIBRARIES} ${Boost_LIBRARIES} ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES} - ${OpenMP_RT_LIBRARIES}) + ${HDF5_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-trackstats-conrad PROPERTIES LINKER_LANGUAGE CXX) ENDIF() # Satellite data binaries -IF (WITH_SATELLITE_UTILS) +IF (SATELLITE_ENABLED) # Parallax correction ADD_EXECUTABLE(meanie3D-parallax_correction src/executables/meanie3D-parallax_correction.cpp) TARGET_LINK_LIBRARIES(meanie3D-parallax_correction @@ -650,14 +724,14 @@ IF (WITH_SATELLITE_UTILS) ${HDF5_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-parallax_correction PROPERTIES LINKER_LANGUAGE CXX) -# satconv tool -ADD_EXECUTABLE(meanie3D-satconv src/executables/meanie3D-satconv.cpp) -TARGET_LINK_LIBRARIES(meanie3D-satconv - meanie3D - ${Boost_LIBRARIES} - ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES}) -SET_TARGET_PROPERTIES(meanie3D-satconv PROPERTIES LINKER_LANGUAGE CXX) + # satconv tool + ADD_EXECUTABLE(meanie3D-satconv src/executables/meanie3D-satconv.cpp) + TARGET_LINK_LIBRARIES(meanie3D-satconv + meanie3D + ${Boost_LIBRARIES} + ${NETCDF_LIBRARIES} + ${HDF5_LIBRARIES}) + SET_TARGET_PROPERTIES(meanie3D-satconv PROPERTIES LINKER_LANGUAGE CXX) ENDIF() # VTK-converter @@ -669,8 +743,7 @@ IF (VTK_ENABLED) ${Boost_LIBRARIES} ${VTK_LIBRARIES} ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES} - ${OpenMP_RT_LIBRARIES}) + ${HDF5_LIBRARIES}) SET_TARGET_PROPERTIES(meanie3D-cfm2vtk PROPERTIES LINKER_LANGUAGE CXX) ENDIF (VTK_ENABLED) @@ -723,7 +796,7 @@ IF (TESTS_ENABLED) ${Boost_LIBRARIES} ${NETCDF_LIBRARIES} ${FLANN_LIBRARIES} - ${OpenMP_RT_LIBRARIES} + ${LZ4} ${HDF5_LIBRARIES} ${VTK_LIBRARIES} ${Blitz_LIBRARY}) @@ -751,10 +824,11 @@ IF (TESTS_ENABLED) TARGET_LINK_LIBRARIES(m3D-test-detection gtest meanie3D + ${FLANN_LIBRARIES} + ${LZ4} ${Boost_LIBRARIES} ${NETCDF_LIBRARIES} ${HDF5_LIBRARIES} - ${OpenMP_RT_LIBRARIES} ${VTK_LIBRARIES}) SET_TARGET_PROPERTIES(m3D-test-detection PROPERTIES LINKER_LANGUAGE CXX) @@ -806,7 +880,7 @@ ENDIF (VTK_ENABLED) # ------------------------------------------------------------- # # Documentation # ------------------------------------------------------------- # -IF (WITH_DOCS) +IF (DOCS_ENABLED) SET(FULL_VERSION_FOR_DOXYGEN 0.1.0) INCLUDE(FindDoxygen) IF (DOXYGEN_EXECUTABLE) diff --git a/HOWTO.md b/HOWTO.md index 9ad8f5f..db4a6b9 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -10,7 +10,9 @@ Using the mean-shift algorithm, objects are detected in your data. The objects a the result first running the mean-shift algorithm with vectors discretized to snap onto the data set's own grid. The vectors are then linked into graphs. Each graph has a mode, which is a point or area where the graph ends. A cluster is made up of -all points in the data set that end at the same mode. +all points in the data set that end at the same mode. Each cluster is given a unique +ID (for the given time). + ### Tracking When clustering a data set that varies over time (Example: satellite data), it is interesting to study the development of clusters over time. For this to happen, the @@ -35,6 +37,20 @@ the same ID over time constitute a track. ## Python frontend +## Visualization + +### Clusters and tracks +In order to see a visual representation of the algorithms workings and it's results, +you will have to install the VisIt package. + +### Track Graph +A track graph describes the development of clusters over time, but not in terms of spatial +location, but rather in terms of it's development history, including merges and splits. +It is a useful tool to get a good overview over the provided tracks in terms of tracking +quality and for picking suitable tracks for subsequent analysis. + +TODO: instructions how to generate and view tracking graph. + ## Docker version The docker version does not support any of the visualisation features. This has something to do with the fact, that the VisIt application used to generate imagery @@ -42,4 +58,4 @@ can not run in headless mode. Therefore the only functions available in the dock version are: * Cluster detection * Tracking -* Track statistics \ No newline at end of file +* Track statistics and -Graph \ No newline at end of file diff --git a/README.md b/README.md index fe80854..1a3841c 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,50 @@ if this flag is set: The package comes with a tool `meanie3D-trackstats-conrad` which analyses KONRAD tracks in a way that makes the data comparable to meanie3D data. This is a specialized tool developed in the context of the OASE project. +## Build presets +A number of presets are provided to make the process easier. Those are selected via the -DPRESET= +flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available presets are: +* `docker` + * Code optimizations for build type 'Release' + * Core functions and python frontend +* `dev-core` + * Code optimizations for build type 'Debug' + * Core functions and python frontend. +* `dev-vtk` + * Code optimizations for build type 'Debug' + * Core functions and python frontend + * Visualization code + * Tests + * Documentation +* `dev-all` + * Code optimizations for build type 'Debug' + * Core functions and python frontend + * Visualization code + * Tests + * Documentation + * RADOLAN, Satellite and KONRAR utilities +* `prod-core` + * Code optimizations for build type 'Release' + * Core functions and python frontend. +* `prod-vtk` + * Code optimizations for build type 'Release' + * Core functions and python frontend + * Visualization code + * Tests + * Documentation +* `prod-all` + * Code optimizations for build type 'Release' + * Core functions and python frontend + * Visualization code + * Tests + * Documentation + * RADOLAN, Satellite and KONRAR utilities + +The term "core functions" refers to the detection, tracking and track evaluation code. For local +development in most cases, the preset `dev-vtk` is sufficient. For more aggressive optimisations, +use `prod-vtk`. If you just need the core functions and none of the visuals, choose the `core` sets +(`dev-core` or `prod-core`). + ## Frequently Asked Quesions ### I'm having trouble compiling the code diff --git a/include/meanie3D/clustering/conrad_cluster.h b/include/meanie3D/clustering/conrad_cluster.h index b8485cf..18b69e1 100644 --- a/include/meanie3D/clustering/conrad_cluster.h +++ b/include/meanie3D/clustering/conrad_cluster.h @@ -21,7 +21,6 @@ * SOFTWARE. */ -#ifdef WITH_KONRAD_UTILS #ifndef M3D_CONRAD_CLUSTER_H #define M3D_CONRAD_CLUSTER_H @@ -240,5 +239,4 @@ namespace m3D { }; } -#endif -#endif // KONRAD \ No newline at end of file +#endif \ No newline at end of file diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index e9ece72..2bd7dbe 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -18,9 +18,9 @@ GET_DIRECTORY_PROPERTY(PROJECT_URL DIRECTORY .. DEFINITION PROJECT_URL) GET_DIRECTORY_PROPERTY(PROJECT_LICENSE DIRECTORY .. DEFINITION PROJECT_LICENSE) # Python -FIND_PACKAGE (Python3 COMPONENTS Interpreter) -IF (Python3_FOUND) - MESSAGE(STATUS "Found Python interpreter (${Python3_VERSION}) ${Python3_EXECUTABLE}") +FIND_PACKAGE (Python COMPONENTS Interpreter) +IF (Python_FOUND) + MESSAGE(STATUS "Found Python interpreter (${Python_VERSION}) ${Python_EXECUTABLE}") ELSE () MESSAGE(FATAL_ERROR "python not found") ENDIF () @@ -28,11 +28,11 @@ ENDIF () # ------------------------------------------------------------- # # Python packaging # ------------------------------------------------------------- # -IF (Python3_FOUND) +IF (Python_FOUND) # Use the system's python path to append it to visit's python # path to enable importing modules to visit's python. - EXECUTE_PROCESS(COMMAND ${Python3_EXECUTABLE} -c "import sys; sys.stdout.write(':'.join(sys.path))" OUTPUT_VARIABLE SYSTEM_PYTHON_PATH) + EXECUTE_PROCESS(COMMAND ${Python_EXECUTABLE} -c "import sys; sys.stdout.write(':'.join(sys.path))" OUTPUT_VARIABLE SYSTEM_PYTHON_PATH) # Configure package file FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/meanie3D DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) @@ -48,8 +48,8 @@ IF (Python3_FOUND) ADD_CUSTOM_TARGET(copy-python-files ALL COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/meanie3D ${CMAKE_CURRENT_BINARY_DIR} DEPENDS meanie3D) - ADD_CUSTOM_TARGET(${PROJECT_NAME} ALL COMMAND ${Python3_EXECUTABLE} setup.py build) + ADD_CUSTOM_TARGET(${PROJECT_NAME} ALL COMMAND ${Python_EXECUTABLE} setup.py build) ADD_CUSTOM_TARGET(${PROJECT_NAME}-clean COMMAND rm -rf build dist *.egg-info setup.py meanie3D/__init__.py README.txt) - INSTALL(CODE "EXECUTE_PROCESS(COMMAND \"${Python3_EXECUTABLE}\" setup.py install WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})") + INSTALL(CODE "EXECUTE_PROCESS(COMMAND \"${Python_EXECUTABLE}\" setup.py install WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})") ENDIF () diff --git a/python/meanie3D/app/__init__.py b/python/meanie3D/app/__init__.py index 70c04df..e75109a 100644 --- a/python/meanie3D/app/__init__.py +++ b/python/meanie3D/app/__init__.py @@ -3,7 +3,7 @@ __visitPath__ = None __visitImportPath__ = None -import sys +import os, sys from meanie3D.app.utils import findVisitPaths # Import visit package @@ -11,6 +11,7 @@ if __visitImportPath__ and __visitPath__: __have_visit__ = True sys.path.append(__visitImportPath__) + sys.path.append(__visitImportPath__ + os.path.sep + 'visit') else: __have_visit__ = False sys.stderr.write("\nERROR:could not find visit import path. Visualisation is switched off.\n") From d7e46e2f1414dd4062616d3069b248312c98f560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sun, 21 Jun 2020 12:04:18 +0100 Subject: [PATCH 19/65] Update README --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 1a3841c..8fb6afd 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,13 @@ method of making sure you haven't broken anything critical. The unit tests can t make test +### -DWITH_PYTHON=YES +In addition to the C++ binaries, the installation also uses pip3 to put a python package in place. +This adds an executable simply called `meanie3D`, which is a front-end to the core functions. It allows +you to put your clustering and tracking parameters down in the form of a configuration file. The entire +pipeline is handled based on this configuration file. *This is the recommended way to run the software*. +For details on the configuration file format, see HOWTO.md. + ### -DWITH_DOCS=YES In order to start developing your own Meanie3D code, it might be useful to have API documentation of the various classes in the project. If you have doxygen installed, you can call the following make command to @@ -149,7 +156,7 @@ flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available pr * Core functions and python frontend * `dev-core` * Code optimizations for build type 'Debug' - * Core functions and python frontend. + * Core functions and python frontend * `dev-vtk` * Code optimizations for build type 'Debug' * Core functions and python frontend @@ -162,7 +169,7 @@ flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available pr * Visualization code * Tests * Documentation - * RADOLAN, Satellite and KONRAR utilities + * RADOLAN, Satellite and KONRAD utilities * `prod-core` * Code optimizations for build type 'Release' * Core functions and python frontend. @@ -178,7 +185,7 @@ flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available pr * Visualization code * Tests * Documentation - * RADOLAN, Satellite and KONRAR utilities + * RADOLAN, Satellite and KONRAD utilities The term "core functions" refers to the detection, tracking and track evaluation code. For local development in most cases, the preset `dev-vtk` is sufficient. For more aggressive optimisations, @@ -206,13 +213,6 @@ layout is not considered and you have to make adjustments to the search paths, p your Find.cmake file to juergen.simon@uni-bonn.de so I can include your changes in the next release and maybe save someone else the trouble. -## Python package -In addition to the C++ binaries, the installation also uses pip3 to put a python package in place. -This adds an executable simply called `meanie3D`, which is a front-end to the core functions. It allows -you to put your clustering and tracking parameters down in the form of a configuration file. The entire -pipeline is handled based on this configuration. This is the recommended way to run the software. For -details on the configuration file format, see HOWTO.md. - ## Visualisation The C++ binaries produce certain files when VTK is enabled. In order to generate imagery or tracking movies, meanie3D relies on executing python scripts in VisIt (). You will have to install this software From 4d08321508eda4cf1c1a5550b8a2c24f5886c733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sun, 21 Jun 2020 12:36:18 +0100 Subject: [PATCH 20/65] Downgrading to python 2.7 again, thanks to Visit dragging it's feet. --- python/CMakeLists.txt | 14 +++++++------- python/meanie2tic/meanie2tic.py | 8 ++++---- python/meanie3D/app/analysis.py | 2 +- python/meanie3D/app/postprocessing.py | 4 ++-- python/meanie3D/app/utils.py | 2 +- python/meanie3D/meanie3D_run.py | 4 ++-- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 2bd7dbe..1fbd107 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -18,9 +18,9 @@ GET_DIRECTORY_PROPERTY(PROJECT_URL DIRECTORY .. DEFINITION PROJECT_URL) GET_DIRECTORY_PROPERTY(PROJECT_LICENSE DIRECTORY .. DEFINITION PROJECT_LICENSE) # Python -FIND_PACKAGE (Python COMPONENTS Interpreter) -IF (Python_FOUND) - MESSAGE(STATUS "Found Python interpreter (${Python_VERSION}) ${Python_EXECUTABLE}") +FIND_PACKAGE (Python2 COMPONENTS Interpreter) +IF (Python2_FOUND) + MESSAGE(STATUS "Found Python interpreter (${Python2_VERSION}) ${Python2_EXECUTABLE}") ELSE () MESSAGE(FATAL_ERROR "python not found") ENDIF () @@ -28,11 +28,11 @@ ENDIF () # ------------------------------------------------------------- # # Python packaging # ------------------------------------------------------------- # -IF (Python_FOUND) +IF (Python2_FOUND) # Use the system's python path to append it to visit's python # path to enable importing modules to visit's python. - EXECUTE_PROCESS(COMMAND ${Python_EXECUTABLE} -c "import sys; sys.stdout.write(':'.join(sys.path))" OUTPUT_VARIABLE SYSTEM_PYTHON_PATH) + EXECUTE_PROCESS(COMMAND ${Python2_EXECUTABLE} -c "import sys; sys.stdout.write(':'.join(sys.path))" OUTPUT_VARIABLE SYSTEM_PYTHON_PATH) # Configure package file FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/meanie3D DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) @@ -48,8 +48,8 @@ IF (Python_FOUND) ADD_CUSTOM_TARGET(copy-python-files ALL COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR}/meanie3D ${CMAKE_CURRENT_BINARY_DIR} DEPENDS meanie3D) - ADD_CUSTOM_TARGET(${PROJECT_NAME} ALL COMMAND ${Python_EXECUTABLE} setup.py build) + ADD_CUSTOM_TARGET(${PROJECT_NAME} ALL COMMAND ${Python2_EXECUTABLE} setup.py build) ADD_CUSTOM_TARGET(${PROJECT_NAME}-clean COMMAND rm -rf build dist *.egg-info setup.py meanie3D/__init__.py README.txt) - INSTALL(CODE "EXECUTE_PROCESS(COMMAND \"${Python_EXECUTABLE}\" setup.py install WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})") + INSTALL(CODE "EXECUTE_PROCESS(COMMAND \"${Python2_EXECUTABLE}\" setup.py install WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})") ENDIF () diff --git a/python/meanie2tic/meanie2tic.py b/python/meanie2tic/meanie2tic.py index d0d9c32..5016747 100755 --- a/python/meanie2tic/meanie2tic.py +++ b/python/meanie2tic/meanie2tic.py @@ -111,7 +111,7 @@ def write_csv(json): # (12) Max LWP of the object (=maximal LWP value included) row.append(cluster['max'][0]) - print(f"{row}") + print(row) objects.writerow(row) @@ -143,7 +143,7 @@ def write_csv(json): # if the first row, write a header if li == 0: header = ['timestep','idstep1','idstep2','weight'] - print(f"{header}") + print(header) tracks.writerow(header) row = [] @@ -159,7 +159,7 @@ def write_csv(json): # (4) Weight factor to decide object relevance after split/merge event for life row.append(0) - print(f"{row}") + print(row) tracks.writerow(row) # ---------------------------------------------------------------------------- @@ -182,7 +182,7 @@ def main(): argv = sys.argv[1:] opts, args = getopt.getopt(argv, "f:h", ["file","help"]) except getopt.GetoptError as detail: - print(f"{detail}") + print(detail) sys.exit(2) if len(sys.argv) < 2: diff --git a/python/meanie3D/app/analysis.py b/python/meanie3D/app/analysis.py index 7aa93db..5702486 100644 --- a/python/meanie3D/app/analysis.py +++ b/python/meanie3D/app/analysis.py @@ -76,7 +76,7 @@ def run(): argv = sys.argv[1:] opts, args = getopt.getopt(argv, "c:f:s:o:r:h", ["resume","help","version","start=","end="]) except getopt.GetoptError as detail: - print(f"{detail}") + print(detail) sys.exit(2) num_params = 0 diff --git a/python/meanie3D/app/postprocessing.py b/python/meanie3D/app/postprocessing.py index 14397c9..287f602 100644 --- a/python/meanie3D/app/postprocessing.py +++ b/python/meanie3D/app/postprocessing.py @@ -167,10 +167,10 @@ def run_trackstats(configuration, directory): return_code = -1 try: - print(f'{".*".join(params)}') + print(".*".join(params)) return_code = external.execute_command("meanie3D-trackstats", " ".join(params), silent=True) except: - print(f'{(".*") % sys.exc_info()[0]}') + print((".*") % sys.exc_info()[0]) raise os.chdir("..") diff --git a/python/meanie3D/app/utils.py b/python/meanie3D/app/utils.py index 7efb6da..134aafb 100644 --- a/python/meanie3D/app/utils.py +++ b/python/meanie3D/app/utils.py @@ -271,7 +271,7 @@ def find(path,filename,requiredComponent=None): if result: return result except OSError as err: - print(f"{err}") + print(err) return None diff --git a/python/meanie3D/meanie3D_run.py b/python/meanie3D/meanie3D_run.py index 447abdb..d583bb5 100755 --- a/python/meanie3D/meanie3D_run.py +++ b/python/meanie3D/meanie3D_run.py @@ -105,7 +105,7 @@ def main(): "scales="] opts, args = getopt.getopt(argv, "c:f:s:o:r:h", long_args) except getopt.GetoptError as detail: - print(f"{detail}") + print(detail) sys.exit(2) scales = [] @@ -196,7 +196,7 @@ def main(): configuration = meanie3D.app.utils.load_configuration(config_file); print("-------------------------------------------------------------------------------------") - print(f"Configuration: {configuration['description']}") + print("Configuration: " + configuration['description']) print("-------------------------------------------------------------------------------------") # Enrich the configuration with env/command line stuff From e1141bc29e57903f2f83537c7f066c557c0c8464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sun, 21 Jun 2020 13:19:11 +0100 Subject: [PATCH 21/65] Downgrading to python 2.7 refac errors fixed (print statements). --- CMakeLists.txt | 1 + Dockerfile | 6 +- README.md | 2 +- python/meanie3D/app/analysis.py | 12 ++-- python/meanie3D/app/postprocessing.py | 70 +++++++++++------------ python/meanie3D/app/tracking.py | 26 ++++----- python/meanie3D/app/utils.py | 4 +- python/meanie3D/visualisation/clusters.py | 46 +++++++-------- python/meanie3D/visualisation/tracks.py | 12 ++-- python/meanie3D/visualisation/utils.py | 26 ++++----- 10 files changed, 103 insertions(+), 102 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9dfbc97..e6ab44f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -54,6 +54,7 @@ INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/include) # ------------------------------------- # Options # ------------------------------------- +OPTION(WITH_OPENMP "Enable/disable use of OpenMP for parallelization" ON) OPTION(WITH_TESTS "Enable/Disable building of tests" OFF) OPTION(WITH_VTK "Enable/Disable visualisation code (requires VisIt and VTK)" OFF) OPTION(WITH_RADOLAN_UTILS "Compile with radolan adaptor (requires libradolan)" OFF) diff --git a/Dockerfile b/Dockerfile index 6be24ec..f19839c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,13 @@ RUN apt-get -y install software-properties-common RUN apt-get -y update RUN apt-get -y install \ wget git cmake \ -gcc g++ python3 python3-pip \ +gcc g++ python python-pip \ libboost-all-dev libflann1.9 libflann-dev blitz++ \ shapelib libhdf5-dev netcdf-bin libnetcdf-dev libnetcdf-c++4 libnetcdf-c++4-dev zlib1g zlib1g-dev -RUN pip3 install setuptools netcdf4 external utils +RUN pip install setuptools netcdf4 external utils # Visualisation -# RUN pip3 install Cython h5py netcdf4 +# RUN pip install Cython h5py netcdf4 # RUN apt-get -y --fix-missing install gnuplot vtk6 libvtk6-dev # RUN wget --quiet http://portal.nersc.gov/project/visit/releases/2.10.0/visit2_10_0.linux-x86_64-rhel6-wmesa.tar.gz # RUN wget --quiet http://portal.nersc.gov/project/visit/releases/2.10.0/visit-install2_10_0 diff --git a/README.md b/README.md index 8fb6afd..d283e35 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ method of making sure you haven't broken anything critical. The unit tests can t make test ### -DWITH_PYTHON=YES -In addition to the C++ binaries, the installation also uses pip3 to put a python package in place. +In addition to the C++ binaries, the installation also uses pip to put a python package in place. This adds an executable simply called `meanie3D`, which is a front-end to the core functions. It allows you to put your clustering and tracking parameters down in the form of a configuration file. The entire pipeline is handled based on this configuration file. *This is the recommended way to run the software*. diff --git a/python/meanie3D/app/analysis.py b/python/meanie3D/app/analysis.py index 5702486..7a45d05 100644 --- a/python/meanie3D/app/analysis.py +++ b/python/meanie3D/app/analysis.py @@ -56,11 +56,11 @@ def usage(): Prints help and exits :return: ''' - print("meanie3D-trackgraph -f ") - print("Analyses a track dictionary and shows a track graph with splits, merges etc.") - print("-f : tracking dictionary file") - print("--help, -h : print this message and exits.") - print("--version : prints the version information and exits") + print "meanie3D-trackgraph -f " + print "Analyses a track dictionary and shows a track graph with splits, merges etc." + print "-f : tracking dictionary file" + print "--help, -h : print this message and exits." + print "--version : prints the version information and exits" sys.exit(1) return @@ -76,7 +76,7 @@ def run(): argv = sys.argv[1:] opts, args = getopt.getopt(argv, "c:f:s:o:r:h", ["resume","help","version","start=","end="]) except getopt.GetoptError as detail: - print(detail) + print detail sys.exit(2) num_params = 0 diff --git a/python/meanie3D/app/postprocessing.py b/python/meanie3D/app/postprocessing.py index 287f602..3a72b0b 100644 --- a/python/meanie3D/app/postprocessing.py +++ b/python/meanie3D/app/postprocessing.py @@ -56,10 +56,10 @@ def check_configuration(configuration): # Check the configuration dimensions = utils.getValueForKeyPath(configuration, 'data.dimensions') if (not dimensions): - print (".*") + print "ERROR:configuration must contain 'dimensions'" return -1 if (not len(dimensions) in [2, 3]): - print (".*") + print "ERROR:Can only process 2D or 3D" return -1 # Check that visualiseTracks has .vtk to work from @@ -69,7 +69,7 @@ def check_configuration(configuration): if tracks: if utils.getValueForKeyPath(tracks, 'visualiseTracks') and not utils.getValueForKeyPath(tracks, 'meanie3D-trackstats.vtk_tracks'): - print (".*") + print "WARNING: tracks.visualiseTracks = True but tracks.meanie3D-trackstats.vtk_tracks = False. Correcting." utils.setValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.vtk_tracks', True) # Complement vtk_dimensions to make our life a little easier down the road. @@ -83,7 +83,7 @@ def check_configuration(configuration): # Make sure that plotStats has gnuplot files to work with if utils.getValueForKeyPath(tracks, 'plotStats') and not utils.getValueForKeyPath(tracks, 'meanie3D-trackstats.gnuplot'): - print (".*") + print "WARNING: track.plot_stats = True but tracks.meanie3D-trackstats.gnuplot = False. Correcting." utils.setValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.gnuplot', True) @@ -98,19 +98,19 @@ def run_trackstats(configuration, directory): ''' pconf = configuration['postprocessing'] if not 'tracks' in pconf: - print (".*") + print "Skipping meanie3D-trackstats because 'tracks' configuration is missing" return False tracks = pconf['tracks'] if not 'meanie3D-trackstats' in tracks: - print (".*") + print "Skipping meanie3D-trackstats because 'tracks.meanie3D-trackstats' configuration is missing" return False if utils.getValueForKeyPath(configuration,'skip_trackstats'): - print (".*") + print "Skipping meanie3D-trackstats because --skip-trackstats is present" return False - print (f"{directory}") + print "Running meanie3D-trackstats for %s" % directory conf = tracks['meanie3D-trackstats'] os.chdir(directory) @@ -161,16 +161,16 @@ def run_trackstats(configuration, directory): params.append("-s netcdf") if configuration['resume'] and haveFiles: - print (".*") + print "Skipping meanie3D-trackstats" os.chdir("..") return True return_code = -1 try: - print(".*".join(params)) + print "meanie3D-trackstats %s" % (" ".join(params)) return_code = external.execute_command("meanie3D-trackstats", " ".join(params), silent=True) except: - print((".*") % sys.exc_info()[0]) + print "ERROR:%s" % sys.exc_info()[0] raise os.chdir("..") @@ -194,7 +194,7 @@ def plot_trackstats(configuration, directory): return False conf = tracks['meanie3D-trackstats'] - print (".*") % directory + print "Plotting .eps files for %s" % directory os.chdir(directory) @@ -249,7 +249,7 @@ def plot_trackstats(configuration, directory): try: return_code = external.execute_command("gnuplot", "plot_stats.gp") except: - print (".*") % sys.exc_info()[0] + print "ERROR:%s" % sys.exc_info()[0] os.chdir("..") return (return_code == 0) @@ -266,7 +266,7 @@ def plot_stats_comparison(configuration): ''' scales = configuration['scales'] if not scales: - print (".*") + print "No scales found, no comparison is done." return False pconf = configuration['postprocessing'] @@ -278,7 +278,7 @@ def plot_stats_comparison(configuration): conf = utils.getValueForKeyPath(pconf, 'tracks.meanie3D-trackstats') - print (".*") + print "Plotting .eps files for comparison" # create a tempfile for gnuplot f = open('plot_stats_comparison.gp', 'w') @@ -359,7 +359,7 @@ def plot_stats_comparison(configuration): try: return_code = external.execute_command("gnuplot", "plot_stats_comparison.gp") except: - print (".*") % sys.exc_info()[0] + print "ERROR:%s" % sys.exc_info()[0] os.chdir("..") return (return_code == 0) @@ -371,7 +371,7 @@ def run_comparison(configuration): :param configuration: :return: ''' - print (".*") + print "Running cross-scale comparison" return plot_stats_comparison(configuration) @@ -389,13 +389,13 @@ def compile_and_run_template(templatePath, configuration, replacements, director ''' scriptFilename = tempfile.mktemp() + ".py" # scriptFilename = os.path.abspath("generated.py") - # print (".*") + scriptFilename + # print "\tWriting python script for visualisation to: " + scriptFilename with open(templatePath) as infile, open(scriptFilename, 'w') as outfile: for line in infile: for src, target in replacements.iteritems(): line = line.replace(src, target) outfile.write(line) - # print (".*") + # print "\tDone." # Compile command line params for visualisation params = "-s %s" % scriptFilename @@ -408,10 +408,10 @@ def compile_and_run_template(templatePath, configuration, replacements, director # Run visualisation returnCode = -1 try: - print (".*") + params + print "Executing visit: " + params returnCode = external.execute_command('visit', params, silent=True) except: - print (".*") % sys.exc_info()[0] + print "ERROR:%s" % sys.exc_info()[0] # Change back out os.chdir('..') @@ -427,9 +427,9 @@ def visualise_tracks(configuration, directory): :param directory: :return: ''' - print (".*") % directory + print "Visualising tracks for %s" % directory if not __have_visit__: - print (".*") + print "Visit not found. Skipping." return home = os.path.abspath(os.path.dirname(meanie3D.__file__) + os.path.sep + os.path.pardir) templatePath = home + os.path.sep + os.path.sep.join(("meanie3D", "resources", "tracks_visit.py")) @@ -452,9 +452,9 @@ def visualise_clusters(configuration, directory): :param directory: :return: ''' - print (".*") % directory + print "Visualising clusters for %s" % directory if not __have_visit__: - print (".*") + print "Visit not found. Skipping." return home = os.path.abspath(os.path.dirname(meanie3D.__file__) + os.path.sep + os.path.pardir) templatePath = home + os.path.sep + os.path.sep.join(("meanie3D", "resources", "clusters_visit.py")) @@ -500,7 +500,7 @@ def cleanup(configuration, directory): :param directory: :return: ''' - print (".*") % directory + print "Cleaning temporary files for %s" % directory os.chdir(directory) if utils.getValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.vtk_tracks'): for filename in glob.glob('*.vtk'): @@ -541,23 +541,23 @@ def run(configuration): else: directories = ['clustering'] - print (".*") % str(directories) + print "Processing directories: %s" % str(directories) for directory in directories: if os.path.isdir(directory): - print (".*") % directory + print "Processing %s" % directory else: - print (".*") % directory + print "ERROR:%s is not a directory!" % directory continue # run the track statistics if configuration['time_operations']: - print (".*") + print "Running trackstats ..." start_time = time.time() if (run_trackstats(configuration, directory)): if configuration['time_operations']: - print (".*") % (time.time()-start_time) + print "Finished. (%.2f seconds)" % (time.time()-start_time) # Copy HTML files copy_html_files(configuration, directory) @@ -570,24 +570,24 @@ def run(configuration): if __have_visit__ and utils.getValueForKeyPath(configuration, 'postprocessing.tracks.visualiseTracks'): if configuration['time_operations']: - print (".*") + print "Visualising tracks ..." start_time = time.time() visualise_tracks(configuration, directory) if configuration['time_operations']: - print (".*") % (time.time()-start_time) + print "Finished. (%.2f seconds)" % (time.time()-start_time) if __have_visit__ and utils.getValueForKeyPath(configuration, 'postprocessing.clusters.visualiseClusters'): if configuration['time_operations']: - print (".*") + print "Visualising clusters ..." start_time = time.time() visualise_clusters(configuration, directory) if configuration['time_operations']: - print (".*") % (time.time()-start_time) + print "Finished. (%.2f seconds)" % (time.time()-start_time) cleanup(configuration, directory) diff --git a/python/meanie3D/app/tracking.py b/python/meanie3D/app/tracking.py index 3fdb541..32a0544 100644 --- a/python/meanie3D/app/tracking.py +++ b/python/meanie3D/app/tracking.py @@ -178,13 +178,13 @@ def run(config,time_index): # the first time step if time_index <= 0: # TODO: consider scenario where user opted not to remove previous results? - print (".*") + print "Removing results from previous runs" utils.create_ouput_directories(output_dir) else: resume_at_index = utils.number_of_netcdf_files(output_dir+"/netcdf") if time_index >= 0 and time_index < resume_at_index: return - print (".*") + str(resume_at_index) + print "Resuming at index " + str(resume_at_index) # Get a list of the files we need to process. source = utils.getValueForKeyPath(config,'source_directory') @@ -213,9 +213,9 @@ def run(config,time_index): cluster_file = cluster_file + "-" + str(time_index) cluster_file += "-clusters.nc" - print (".*") - print (".*") + netcdf_file - print (".*") + print "-------------------------------------------------------------------------------------" + print "Processing " + netcdf_file + print "-------------------------------------------------------------------------------------" # ---------------------------------------------- # Clustering @@ -230,7 +230,7 @@ def run(config,time_index): run_count = run_count + 1 continue - print (".*") + print "-- Clustering --" # build the clustering command params = detect_params @@ -279,11 +279,11 @@ def run(config,time_index): if config['time_operations']: start_time = time.time() - print (".*") + params + print "meanie3D-detect " + params external.execute_command("meanie3D-detect",params,True) if config['time_operations']: - print (".*") % (time.time()-start_time) + print "Finished. (%.2f seconds)" % (time.time()-start_time) # ---------------------------------------------- @@ -299,7 +299,7 @@ def run(config,time_index): else: logfile = output_dir+"/log/tracking_" + str(time_index)+".log" - print (".*") + print "-- Tracking --" params = tracking_params + " --previous %s --current %s " \ % (os.path.abspath(last_cluster_file),os.path.abspath(cluster_file)) params = params + " > " + os.path.abspath(logfile) @@ -308,11 +308,11 @@ def run(config,time_index): if config['time_operations']: start_time = time.time() - print (".*") + params + print "meanie3D-track" + params external.execute_command("meanie3D-track",params,True) if config['time_operations']: - print (".*") % (time.time()-start_time) + print "Finished. (%.2f seconds)" % (time.time()-start_time) # keep track last_cluster_file = cluster_file @@ -322,8 +322,8 @@ def run(config,time_index): # Clean out the trash if there is any if config['cleanup_vtk']: - print (".*") + print "Cleaning up *.vt*" call("rm -f *.vt*", shell=True) - print (".*") + print "Done." return \ No newline at end of file diff --git a/python/meanie3D/app/utils.py b/python/meanie3D/app/utils.py index 134aafb..8532f53 100644 --- a/python/meanie3D/app/utils.py +++ b/python/meanie3D/app/utils.py @@ -177,7 +177,7 @@ def setValueForKeyPath(object,keypath,value): # in visualisation objects. Try to set an enumerated value when hitting # this combination if type(value) is str and type(getattr(object,keypath)) is int: - print (".*") + print "Attempting to resolve constant" value = getattr(object,value) if type(object) is dict: @@ -271,7 +271,7 @@ def find(path,filename,requiredComponent=None): if result: return result except OSError as err: - print(err) + print err return None diff --git a/python/meanie3D/visualisation/clusters.py b/python/meanie3D/visualisation/clusters.py index 880a996..a3f3c87 100644 --- a/python/meanie3D/visualisation/clusters.py +++ b/python/meanie3D/visualisation/clusters.py @@ -56,7 +56,7 @@ def add_clusters(cluster_vtk_file,configuration): sys.stderr.write("ERROR:no value found for postprocessing.clusters.visit.cluster") return # plot the clusters - print (".*") % cluster_vtk_file + print "Adding clusters from file %s" % cluster_vtk_file utils.addPseudocolorPlot(cluster_vtk_file, clusterOptions) return @@ -76,12 +76,12 @@ def run(conf): clusterConf = utils.getValueForKeyPath(conf,'postprocessing.clusters') if not clusterConf: - print (".*") + print "No configuration for cluster postprocessing. Nothing to do." return 0 visitConf = utils.getValueForKeyPath(clusterConf,'visit') if not visitConf: - print (".*") + print "No configuration for visuals. Nothing to do." return 0 views = utils.getValueForKeyPath(visitConf,'views') @@ -93,10 +93,10 @@ def run(conf): utils.setAnnotations(conf,'postprocessing.clusters.visit.annotations') if not utils.getValueForKeyPath(conf,'resume'): - print (".*") + print "Removing results from previous runs" subprocess.call("rm -rf images movies *.vtk *.vtr *tracking*.png *source*.png", shell=True) else: - print (".*") + print "Removing intermediary files from previous runs" subprocess.call("rm -f *.vtk *.vtr *.vtu", shell=True) # Figure out the ending for the cluster vtk files @@ -109,12 +109,12 @@ def run(conf): # Glob the netcdf directory or find the single file uses_time = utils.getValueForKeyPath(conf,'uses_time') - print (".*") + os.path.abspath(os.getcwd()) + print "Current work directory: " + os.path.abspath(os.getcwd()) if uses_time: - print (".*") + conf['source_directory'] + print "Processing file " + conf['source_directory'] netcdf_file = conf['source_directory'] else: - print (".*") + conf['source_directory'] + print "Processing files in directory " + conf['source_directory'] netcdf_files = sorted(glob.glob(conf['source_directory']+"/*.nc")) # Keep track of number of images to allow @@ -153,12 +153,12 @@ def run(conf): label_vtk_file = basename + "-clusters-centers.vtk" displacement_vtk_file = basename + "-clusters-displacements.vtk" - print (".*") + netcdf_file - print (".*") + cluster_file + print "netcdf_file = " + netcdf_file + print "cluster_file = " + cluster_file # check if the files both exist if not os.path.exists(cluster_file): - print (".*") + print "Cluster file does not exist. Skipping." continue # predict the filenames for checking on resume @@ -170,10 +170,10 @@ def run(conf): if conf['resume'] == True: exists = utils.images_exist(visitConf['views'],"source",image_count) if exists == "all": - print (".*") + print "Source visualization " + number_postfix + " exists. Skipping." skip_source = True elif exists == "partial": - print (".*") + number_postfix + print "Deleting partial visualization " + number_postfix utils.delete_images(conf,"source",image_count) if skip_source == False: @@ -188,7 +188,7 @@ def run(conf): utils.add_datetime(clusterConf,netcdf_file,time_index) # Add source data and threshold it - print (".*") + print "Plotting source data ..." start_time = time.time() @@ -201,7 +201,7 @@ def run(conf): visit.DeleteAllPlots() visit.ClearWindow() - print (".*") % (time.time()-start_time) + print " done. (%.2f seconds)" % (time.time()-start_time) if utils.getValueForKeyPath(clusterConf,'visualiseClusters') and __have_vtk__: @@ -210,16 +210,16 @@ def run(conf): exists = utils.images_exist(visitConf['views'],"tracking",image_count) if exists == "all": - print (".*") + print "Cluster visualization "+number_postfix+" exists. Skipping." skip = True elif exists == "partial": - print (".*") + number_postfix + print "Deleting partial cluster visualization " + number_postfix utils.delete_images(conf,"tracking",image_count) if skip == False: # Run the conversion - print (".*") + print "-- Converting clusters to .vtr --" start_time = time.time() params = "-f %s %s" \ % (cluster_file,utils.getValueForKeyPath(conf,'postprocessing.clusters.meanie3D-cfm2vtk')) @@ -231,9 +231,9 @@ def run(conf): params += " --vtk-dimensions=%s" % vtkDimString # pdb.set_trace(); - print (".*") % params + print "meanie3D-cfm2vtk %s" % params meanie3D.app.external.execute_command('meanie3D-cfm2vtk', params) - print (".*") % (time.time()-start_time) + print " done. (%.2f seconds)" % (time.time()-start_time) # Move cluster output file to individual file if uses_time: @@ -241,7 +241,7 @@ def run(conf): os.rename(cluster_vtk_file,cluster_vtk_file_dst) cluster_vtk_file = cluster_vtk_file_dst - print (".*") + print "-- Rendering cluster scene --" start_time = time.time() # Add ancillary background data @@ -273,7 +273,7 @@ def run(conf): visit.DrawPlots() utils.saveImagesForViews(views,"tracking") - print (".*") % (time.time()-start_time) + print " done. (%.2f seconds)" % (time.time()-start_time) # clean up @@ -311,7 +311,7 @@ def run(conf): utils.createMoviesForViews(views,"tracking", movieFormats) # clean up - print (".*") + print "Cleaning up ..." subprocess.call("mkdir images", shell=True) subprocess.call("mv *tracking_*.png images", shell=True) subprocess.call("mv *source_*.png images", shell=True) diff --git a/python/meanie3D/visualisation/tracks.py b/python/meanie3D/visualisation/tracks.py index c1b5ece..054e9bd 100644 --- a/python/meanie3D/visualisation/tracks.py +++ b/python/meanie3D/visualisation/tracks.py @@ -44,7 +44,7 @@ def run(conf): visitConf = utils.getValueForKeyPath(conf,'postprocessing.tracks.visit') if not visitConf: - print (".*") + print "No configuration for visuals. Nothing to do." return 0 # Set up background gradient, axis labels etc. @@ -73,8 +73,8 @@ def run(conf): track_pattern = "*-track_*.vtk" list = sorted(glob.glob(track_pattern)) - print (".*") + track_pattern - print (".*") % len(list) + print "Looking with pattern " + track_pattern + print "Found %d track files." % len(list) count = 0; for trackFile in list: @@ -88,7 +88,7 @@ def run(conf): # Plot the actual track data file = conf['tracks_dir'] + os.path.sep + trackFile - print (".*") + file + print "Adding plot for " + file utils.addPseudocolorPlot(file,trackPlotConf) count = count + 1 @@ -101,10 +101,10 @@ def run(conf): trackPlotConf['PseudocolorAttributes']['legendFlag'] = legendFlag # pp.pprint(trackPlotConf) - print (".*") + print "Drawing plots" visit.DrawPlots() - print (".*") % os.getcwd() + print "Saving image to %s" % os.getcwd() utils.saveImage("tracks",0) os.chdir(currentDirectory) diff --git a/python/meanie3D/visualisation/utils.py b/python/meanie3D/visualisation/utils.py index 0f15ff2..905916d 100644 --- a/python/meanie3D/visualisation/utils.py +++ b/python/meanie3D/visualisation/utils.py @@ -35,22 +35,22 @@ ret_code, paths_string = meanie3D.app.external.execute_command('python','-c "import sys; print sys.path"', True) if ret_code == 0: - print (".*") + print "Attempting to locate python module netCDF4" result = meanie3D.app.utils.find_in_paths(["/usr/lib","/usr/local/lib"],"netCDF4","site-packages") if not result: result = meanie3D.app.utils.find_in_paths(["/usr/lib","/usr/local/lib"],"netCDF4","dist-packages") if not result: - print (".*") + print "Failed to locate python module netCDF4" exit(-1) else: - print (".*") % result + print "Found netCDF4 at %s" % result sys.path.append(os.path.split(result)[0]); - print (".*") + print "Python path after adding system search directories:" print(sys.path) else: - print (".*") + print "Failed to obtain system's python path" exit(-1) import netCDF4 @@ -371,7 +371,7 @@ def plotMapdata(configuration, path): if os.path.exists(mapFile): addPseudocolorPlots(mapFile, configuration, path + ".plots") else: - print (".*") + mapFile + print "ERROR:could not find map file at " + mapFile return @@ -537,11 +537,11 @@ def create_movie(basename, moviename): :param moviename: :return: ''' - print (".*") + print "Creating movie '" + moviename + "' from files '" + basename + "*.png ..." args = "-limit memory 4GB -delay 50 -quality 100 -dispose Background %s*.png %s" % (basename, moviename) - print (".*") % args + print "convert %s" % args meanie3D.app.external.execute_command('convert', args, False) - print (".*") + print "done." return @@ -671,9 +671,9 @@ def add_datetime(conf, netcdf_file,time_index): date = date.replace(microsecond=0) have_valid_time = True except ValueError as vi: - print (".*") - print (vi.message) - print (".*") + print "Error reading time information from file:" + print vi.message + print "Falling back on time index" date = time_index x = 0.725 @@ -714,7 +714,7 @@ def create_dual_panel(basename_left, basename_right, basename_combined): left_files = sorted(glob.glob(basename_left + "*.png")) right_files = sorted(glob.glob(basename_right + "*.png")) if len(left_files) != len(right_files): - print (".*") + print "ERROR:the two image series " + basename_left + "*.png and " + basename_right + "*.png have different lengths!" return # create a small image to blank the datestamp with From a760a6fffc5dfc06e7d9e77e11850f9f13799082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 00:32:10 +0100 Subject: [PATCH 22/65] Fixed OpenMP integration. --- CMakeLists.txt | 17 ++++++++++++----- include/meanie3D/parallel.h | 4 ++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e6ab44f..90c84a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ # cmake configuration # ------------------------------------- -CMAKE_MINIMUM_REQUIRED(VERSION 3.2.0) +CMAKE_MINIMUM_REQUIRED(VERSION 3.12.0) MARK_AS_ADVANCED(CMAKE_BACKWARDS_COMPATIBILITY) SET(CMAKE_ALLOW_LOOSE_LOOP_CONSTRUCTS TRUE) SET(CMAKE_VERBOSE_MAKEFILE ON) @@ -164,6 +164,9 @@ ENDIF() IF (OPENMP_ENABLED) MESSAGE(STATUS "OpenMP is enabled") ADD_DEFINITIONS(-DWITH_OPENMP=1) + FIND_PACKAGE(OpenMP REQUIRED) + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xpreprocessor -fopenmp") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") ELSE () MESSAGE(STATUS "OpenMP is disabled") ENDIF () @@ -620,7 +623,8 @@ TARGET_LINK_LIBRARIES(meanie3D-detect ${Boost_LIBRARIES} ${VTK_LIBRARIES} ${HDF5_LIBRARIES} - ${NETCDF_LIBRARIES}) + ${NETCDF_LIBRARIES} + OpenMP::OpenMP_CXX) #SET_TARGET_PROPERTIES(meanie3D-detect PROPERTIES LINKER_LANGUAGE CXX) # tracking tool @@ -631,7 +635,8 @@ TARGET_LINK_LIBRARIES(meanie3D-track ${Boost_LIBRARIES} ${VTK_LIBRARIES} ${HDF5_LIBRARIES} - ${NETCDF_LIBRARIES}) + ${NETCDF_LIBRARIES} + OpenMP::OpenMP_CXX) SET_TARGET_PROPERTIES(meanie3D-track PROPERTIES LINKER_LANGUAGE CXX) # trackstats @@ -642,7 +647,8 @@ TARGET_LINK_LIBRARIES(meanie3D-trackstats ${VTK_LIBRARIES} ${Boost_LIBRARIES} ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES}) + ${HDF5_LIBRARIES} + OpenMP::OpenMP_CXX) SET_TARGET_PROPERTIES(meanie3D-trackstats PROPERTIES LINKER_LANGUAGE CXX) # copydims @@ -744,7 +750,8 @@ IF (VTK_ENABLED) ${Boost_LIBRARIES} ${VTK_LIBRARIES} ${NETCDF_LIBRARIES} - ${HDF5_LIBRARIES}) + ${HDF5_LIBRARIES} + OpenMP::OpenMP_CXX) SET_TARGET_PROPERTIES(meanie3D-cfm2vtk PROPERTIES LINKER_LANGUAGE CXX) ENDIF (VTK_ENABLED) diff --git a/include/meanie3D/parallel.h b/include/meanie3D/parallel.h index 8072352..3dfe39d 100644 --- a/include/meanie3D/parallel.h +++ b/include/meanie3D/parallel.h @@ -24,6 +24,10 @@ #ifndef M3D_PARALLEL_H #define M3D_PARALLEL_H +#if WITH_OPENMP +#include +#endif + #define PROVIDE_MUTEX ! #define PROVIDE_THREADSAFETY ! From 893a09002eb4c84a7119646be8c29080cf095ad5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 09:50:30 +0100 Subject: [PATCH 23/65] C++11deprecated dynamic exception specification. --- include/meanie3D/adaptors/Radolan.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/include/meanie3D/adaptors/Radolan.h b/include/meanie3D/adaptors/Radolan.h index 1237fb7..9896865 100644 --- a/include/meanie3D/adaptors/Radolan.h +++ b/include/meanie3D/adaptors/Radolan.h @@ -53,9 +53,8 @@ namespace m3D { bool write_one_bytes_as_byte = false, const RDDataType *threshold = NULL, netCDF::NcFile::FileMode mode = netCDF::NcFile::replace, - bool omitOutside = true) - throw(CFFileConversionException); - + bool omitOutside = true); + /** Converts the radolan file at path into a CF-Metadata compliant NetCDF-File. * @param radolanPath full path to the radolan file * @param netcdfPath full path to the netcdf file to be created @@ -70,8 +69,7 @@ namespace m3D { const char *netcdfPath, bool write_one_bytes_as_byte, const RDDataType *threshold = NULL, - netCDF::NcFile::FileMode mode = netCDF::NcFile::write) - throw(CFFileConversionException); + netCDF::NcFile::FileMode mode = netCDF::NcFile::write); /** Simple function to get a visual rep of the file with ascii characters * on terminal. @@ -79,14 +77,15 @@ namespace m3D { * @param print values every latVertices points in y * @param print values every lonVertices points in x */ - void CFPrintConvertedRadolanScan(netCDF::NcFile *file, int latVertices = 20, int lonVertices = 20); + void CFPrintConvertedRadolanScan(netCDF::NcFile *file, + int latVertices = 20, + int lonVertices = 20); /** CF-Metadata 'standard_name' for the given scan type * @param scanType * @return standard_name */ const char *CFRadolanDataStandardName(RDScanType scanType); - } #endif From 85a5b47c3b58f454f0bd719e266bb210e8f7e010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 09:52:46 +0100 Subject: [PATCH 24/65] Breaking off if OpenMP is enabled but libomp could not be found. --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 90c84a2..8c1db92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -165,6 +165,9 @@ IF (OPENMP_ENABLED) MESSAGE(STATUS "OpenMP is enabled") ADD_DEFINITIONS(-DWITH_OPENMP=1) FIND_PACKAGE(OpenMP REQUIRED) + IF (NOT OpenMP_FOUND) + MESSAGE(FATAL_ERROR "Could not locate OpenMP") + ENDIF() SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xpreprocessor -fopenmp") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") ELSE () From 5d00d51ff243f2e26db21b246e96546412135290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 10:24:49 +0100 Subject: [PATCH 25/65] Exception specification. --- include/meanie3D/adaptors/Radolan.h | 9 ++++++--- src/adaptors/Radolan.cpp | 14 +++++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/include/meanie3D/adaptors/Radolan.h b/include/meanie3D/adaptors/Radolan.h index 9896865..e5db8a9 100644 --- a/include/meanie3D/adaptors/Radolan.h +++ b/include/meanie3D/adaptors/Radolan.h @@ -53,8 +53,9 @@ namespace m3D { bool write_one_bytes_as_byte = false, const RDDataType *threshold = NULL, netCDF::NcFile::FileMode mode = netCDF::NcFile::replace, - bool omitOutside = true); - + bool omitOutside = true) + throw(m3D::CFFileConversionException); + /** Converts the radolan file at path into a CF-Metadata compliant NetCDF-File. * @param radolanPath full path to the radolan file * @param netcdfPath full path to the netcdf file to be created @@ -69,7 +70,8 @@ namespace m3D { const char *netcdfPath, bool write_one_bytes_as_byte, const RDDataType *threshold = NULL, - netCDF::NcFile::FileMode mode = netCDF::NcFile::write); + netCDF::NcFile::FileMode mode = netCDF::NcFile::write) + throw(m3D::CFFileConversionException); /** Simple function to get a visual rep of the file with ascii characters * on terminal. @@ -86,6 +88,7 @@ namespace m3D { * @return standard_name */ const char *CFRadolanDataStandardName(RDScanType scanType); + } #endif diff --git a/src/adaptors/Radolan.cpp b/src/adaptors/Radolan.cpp index 9e5fbe2..2c8b5b6 100644 --- a/src/adaptors/Radolan.cpp +++ b/src/adaptors/Radolan.cpp @@ -26,7 +26,7 @@ namespace m3D { const RDDataType *threshold, netCDF::NcFile::FileMode mode, bool omitOutside) - throw (CFFileConversionException) + throw (m3D::CFFileConversionException) { if (mode == netCDF::NcFile::read) { @@ -73,13 +73,13 @@ namespace m3D { * @return NCFile* NetCDF-Filehandler * @throw CFFileConversionException */ - NcFile * CFConvertRadolanScan(RDScan *scan, - const char* netcdfPath, - bool write_one_bytes_as_byte, - const RDDataType *threshold, - NcFile::FileMode mode) + NcFile *CFConvertRadolanScan(RDScan *scan, + const char *netcdfPath, + bool write_one_bytes_as_byte, + const RDDataType *threshold, + NcFile::FileMode mode) - throw (CFFileConversionException) + throw(m3D::CFFileConversionException) { using namespace netCDF; using namespace std; From 4d83d38ccf9d0d2da5389906c13667d814963a8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 10:33:03 +0100 Subject: [PATCH 26/65] cmake fix around shapelib --- CMakeLists.txt | 2 +- cmake_modules/FindSHP.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c1db92..e475e57 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -314,7 +314,7 @@ FIND_PACKAGE(Blitz REQUIRED) IF (Blitz_FOUND) INCLUDE_DIRECTORIES(${Blitz_INCLUDE_DIR}) ELSE () - MESSAGE(FATAL_ERROR "blitz++ not found") + MESSAGE(FATAL_ERROR "blitz++ not found") ENDIF () # Shapefile diff --git a/cmake_modules/FindSHP.cmake b/cmake_modules/FindSHP.cmake index 428934a..dc5690b 100755 --- a/cmake_modules/FindSHP.cmake +++ b/cmake_modules/FindSHP.cmake @@ -4,7 +4,7 @@ FIND_LIBRARY(SHP NAMES shp PATHS /usr/lib /usr/local/lib /opt/local/lib ~/radola IF (SHP) SET(SHP_LIBRARIES ${SHP}) ELSE (SHP) - SET(SHP_LIBRARIES "NOTFOUND") + SET(SHP_LIBRARIES "") ENDIF(SHP) IF (SHP_INCLUDE_DIR AND SHP_LIBRARIES) From f11d89705a6dc10e9460d018651b0b0755a7d064 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 11:17:47 +0100 Subject: [PATCH 27/65] Cmake fix for docker --- CMakeLists.txt | 11 +++++++++++ cmake_modules/FindHDF5.cmake | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e475e57..0d9adbe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,6 +78,17 @@ IF(PRESET MATCHES "docker") SET(SATELLITE_ENABLED OFF) SET(KONRAD_ENABLED OFF) SET(DOCS_ENABLED OFF) +ELSEIF(PRESET MATCHES "docker-vtk") + MESSAGE(STATUS "Selected preset ${PRESET}") + SET(CMAKE_BUILD_TYPE "Release") + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED ON) + SET(TESTS_ENABLED OFF) + SET(RADOLAN_ENABLED OFF) + SET(SATELLITE_ENABLED OFF) + SET(KONRAD_ENABLED OFF) + SET(DOCS_ENABLED OFF) ELSEIF (PRESET MATCHES "dev-all") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "Debug") diff --git a/cmake_modules/FindHDF5.cmake b/cmake_modules/FindHDF5.cmake index 07c47cf..6ca8ef3 100644 --- a/cmake_modules/FindHDF5.cmake +++ b/cmake_modules/FindHDF5.cmake @@ -3,7 +3,7 @@ # Example: -DWITH_HDF5=/usr/local SET(HDF5_ROOT ${WITH_HDF5}) -IF (FOR_DOCKER) +IF (PRESET MATCHES "^docker.*") MESSAGE(STATUS "Looking for HDF5 in Debian locations") FIND_PATH(HDF5_INCLUDE_DIR hdf5.h PATHS /usr/include/hdf5/serial) FIND_LIBRARY(HDF5 NAMES hdf5 PATHS /usr/lib/x86_64-linux-gnu/hdf5/serial) From e450537d11debf18372df3cd71a0d5c51667020f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 11:37:19 +0100 Subject: [PATCH 28/65] Dropping 8.2 requirement on VTK as it's not available on Debian. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d9adbe..85068db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,7 +341,7 @@ ENDIF () # VTK IF (VTK_ENABLED) - FIND_PACKAGE(VTK 8.2 REQUIRED NO_MODULE) + FIND_PACKAGE(VTK REQUIRED NO_MODULE) IF (VTK_FOUND) MESSAGE(STATUS "VTK found") INCLUDE(${VTK_USE_FILE}) From bcb88c923d6cf6ccd64e4e0073e9470de3a2ca99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 13:03:22 +0100 Subject: [PATCH 29/65] Documentation updated. Dockerfiles for core- and vtk version patched. Docker makefile revised. --- Dockerfile | 16 +++---------- Dockerfile.vtk | 50 +++++++++++++++++++++++++++++++++++++++ Makefile.docker | 62 ++++++++++++++++++++++++++++++++++++++----------- README.md | 10 +++++--- 4 files changed, 109 insertions(+), 29 deletions(-) create mode 100644 Dockerfile.vtk diff --git a/Dockerfile b/Dockerfile index f19839c..a3a49e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,5 @@ FROM debian:stable -ENV PACKAGES= RUN apt-get -y update --fix-missing RUN apt-get -y upgrade RUN apt-get -y dist-upgrade @@ -8,21 +7,12 @@ RUN apt-get -y install software-properties-common RUN apt-get -y update RUN apt-get -y install \ wget git cmake \ -gcc g++ python python-pip \ +gcc g++ libomp5 \ +python python-pip \ libboost-all-dev libflann1.9 libflann-dev blitz++ \ shapelib libhdf5-dev netcdf-bin libnetcdf-dev libnetcdf-c++4 libnetcdf-c++4-dev zlib1g zlib1g-dev RUN pip install setuptools netcdf4 external utils -# Visualisation -# RUN pip install Cython h5py netcdf4 -# RUN apt-get -y --fix-missing install gnuplot vtk6 libvtk6-dev -# RUN wget --quiet http://portal.nersc.gov/project/visit/releases/2.10.0/visit2_10_0.linux-x86_64-rhel6-wmesa.tar.gz -# RUN wget --quiet http://portal.nersc.gov/project/visit/releases/2.10.0/visit-install2_10_0 -# RUN chmod a+x visit-install2_10_0 -# RUN echo "1" | ./visit-install2_10_0 2.10.0 linux-x86_64-rhel6-wmesa /usr/local/visit -# ENV VISIT_EXECUTABLE=/usr/local/visit/bin/visit -# RUN rm -rf visit* - # libradolan RUN git clone https://github.com/JuergenSimon/radolan.git RUN cd radolan && cmake . && make install && cd .. && rm -rf radolan @@ -32,7 +22,7 @@ RUN git clone --recurse-submodules --depth=1 https://github.com/JuergenSimon/mea WORKDIR /meanie3D RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch --all RUN git checkout --track remotes/origin/dockerize && git pull -RUN git pull && cmake -DFOR_DOCKER=YES . && make install +RUN git pull && cmake -DPRESET=docker . && make install # Cleanup WORKDIR / diff --git a/Dockerfile.vtk b/Dockerfile.vtk new file mode 100644 index 0000000..ed7adf8 --- /dev/null +++ b/Dockerfile.vtk @@ -0,0 +1,50 @@ +FROM debian:stable + +ARG VISIT_VERSION="3.1.0" +ARG VISIT_PKG_URL="https://portal.nersc.gov/project/visit/releases/3.1.0/visit3_1_0.linux-x86_64-debian9.tar.gz" +ARG VISIT_INSTALLER_URL=https://portal.nersc.gov/project/visit/releases/3.1.0/${VISIT_INSTALLER} +ARG VISIT_INSTALLER=visit-install3_1_0 + +RUN apt-get -y update --fix-missing +RUN apt-get -y upgrade +RUN apt-get -y dist-upgrade +RUN apt-get -y install software-properties-common +RUN apt-get -y update +RUN apt-get -y install \ +wget git cmake \ +gcc g++ libomp5 \ +python python-pip \ +libboost-all-dev libflann1.9 libflann-dev blitz++ \ +shapelib libhdf5-dev netcdf-bin libnetcdf-dev libnetcdf-c++4 libnetcdf-c++4-dev zlib1g zlib1g-dev +RUN pip install setuptools netcdf4 external utils + +# Visualisation +RUN pip install Cython h5py netcdf4 +RUN apt-get -y --fix-missing install gnuplot vtk7 libvtk7-dev +RUN wget --quiet ${VISIT_INSTALLER_URL} +RUN wget --quiet ${VISIT_PKG_URL} +RUN chmod u+x ./visit-install3_1_0 +RUN echo "1" | ./visit-install3_1_0 3.1.0 linux-x86_64 /usr/local/visit +ENV VISIT_EXECUTABLE=/usr/local/visit/bin/visit + +# libradolan +RUN git clone https://github.com/JuergenSimon/radolan.git +RUN cd radolan && cmake . && make install && cd .. + +# Meanie3D +RUN git clone --recurse-submodules --depth=1 https://github.com/JuergenSimon/meanie3D +WORKDIR /meanie3D +RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch --all +RUN git checkout --track remotes/origin/dockerize && git pull +RUN git pull && cmake -DPRESET=docker-vtk . && make install + +# Cleanup +WORKDIR / +RUN rm -rf meanie3D radolan visit* +RUN apt-get remove -y wget git cmake +RUN apt autoremove -y + +# Prepare for runtime +RUN mkdir /data +ENV LD_LIBRARY_PATH=/usr/local/lib +ENTRYPOINT ["/usr/local/bin/meanie3D"] \ No newline at end of file diff --git a/Makefile.docker b/Makefile.docker index 07d0f7f..cae0bdd 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -1,21 +1,57 @@ # Docker repository repository=iontichy -image=meanie3d -version=0.1.0 work_dir=. -build_args= -all: image push -.PHONY : all +# Core +image=meanie3d +version=0.1.0 +tag="${repository}/${image}:${version}" +latest="${repository}/${image}:latest" -image: - docker build $(build_args) --tag ${repository}/${image}:${version} -f Dockerfile ${work_dir} - docker tag ${repository}/${image}:${version} ${repository}/${image}:latest +docker: + docker build --tag ${tag} -f Dockerfile ${work_dir} + docker tag ${tag} ${latest} +.PHONY: image -image-nocache: - docker build --no-cache $(build_args) --tag ${repository}/${image}:${version} -f Dockerfile ${work_dir} - docker tag ${repository}/${image}:${version} ${repository}/${image}:latest +docker-nocache: + docker build --no-cache --tag ${tag} -f Dockerfile ${work_dir} + docker tag ${tag} ${latest} +.PHONY: image-nocache push: - docker push ${repository}/${image}:${version} - docker push ${repository}/${image}:latest \ No newline at end of file + docker push ${tag} + docker push ${latest} +.PHONY: push + +# With VTK +imageVtk=meanie3d-vtk +versionVtk=0.1.0 +tagVtk="${repository}/${imageVtk}:${versionVtk}" +latestVtk="${repository}/${imageVtk}:latest" + +docker-vtk: + docker build --tag ${tagVtk} -f Dockerfile.vtk ${work_dir} + docker tag ${tagVtk} ${latestVtk} +.PHONY: image-vtk + +docker-nocache-vtk: + docker build --no-cache --tag ${tagVtk} -f Dockerfile.vtk ${work_dir} + docker tag ${tagVtk} ${latestVtk} +.PHONY: image-nocache-vtk + +push-vtk: + docker push ${tagVtk} + docker push ${latestVtk} +.PHONY: push-vtk + +# General targets + +clean: + docker images | grep meanie3d | awk '{print $3}' | xargs docker rmi -f &&\ + docker images | grep none | awk '{print $3}' | xargs docker rmi -f &&\ + docker system prune -f +.PHONY: clean + +all: docker push docker-vtk push-vtk +.PHONY: all + diff --git a/README.md b/README.md index d283e35..0568fee 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,13 @@ Meanie3D comes with a number of dependencies that need to be installed prior to * NetCDF4-python (try running @pip install netCDF4@ or download and install from http://unidata.github.io/netcdf4-python) * libradolan (http://meteo-ubonn.github.io/radolan/) -Optionally, the following libraries may be used, if they are switched on with the appropriate flags: -* VTK 6.0 or better (http://www.vtk.org/VTK/resources/software.html) -* doxygen (https://www.doxygen.nl/download.html) +The following libraries may be used, if they are switched on with the appropriate flags +* Visualization: + * Imagemagick (https://imagemagick.org) + * VTK 7.0 or better (http://www.vtk.org) + * Visit 3.0.0 or better (https://wci.llnl.gov/simulation/computer-codes/visit) +* Generated source code documentation: + * doxygen (https://www.doxygen.nl/download.html) ### Compiler Prerequisites From d5f61aa195d8be6cd85571a89432bc540a966fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 13:06:20 +0100 Subject: [PATCH 30/65] Documentation updated. --- README.md | 115 +++++++++++++++++++++++++++--------------------------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 0568fee..d9a6554 100644 --- a/README.md +++ b/README.md @@ -64,33 +64,78 @@ flags -DWITH_OPENMP=0 to your cmake call. Meanie3D uses CMAKE to generate makefiles. You can use CMAKE's abilities to generate IDE files if you prefer. Start by cloning the master branch (for an up-to date but possibly unstable version) or one of the stable releases. - +``` git clone https://github.com/meteo-ubonn/meanie3D.git - +``` TODO: revise handling of map data If you want to download the OASE topology and mapdata file for visualisation, you can obtain this file by adding: - +``` git clone http://git.meteo.uni-bonn.de/git/oase-mapdata - +``` Create a build directory: - +``` mkdir meanie3D-make cd meanie3D-make cmake ../meanie3D +``` + +### Build presets +A number of presets are provided to make the process easier. Those are selected via the -DPRESET= +flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available presets are: +* `docker` + * Code optimizations for build type 'Release' + * Core functions and python frontend +* `dev-core` + * Code optimizations for build type 'Debug' + * Core functions and python frontend +* `dev-vtk` + * Code optimizations for build type 'Debug' + * Core functions and python frontend + * Visualization code + * Tests + * Documentation +* `dev-all` + * Code optimizations for build type 'Debug' + * Core functions and python frontend + * Visualization code + * Tests + * Documentation + * RADOLAN, Satellite and KONRAD utilities +* `prod-core` + * Code optimizations for build type 'Release' + * Core functions and python frontend. +* `prod-vtk` + * Code optimizations for build type 'Release' + * Core functions and python frontend + * Visualization code + * Tests + * Documentation +* `prod-all` + * Code optimizations for build type 'Release' + * Core functions and python frontend + * Visualization code + * Tests + * Documentation + * RADOLAN, Satellite and KONRAD utilities + +The term "core functions" refers to the detection, tracking and track evaluation code. For local +development in most cases, the preset `dev-vtk` is sufficient. For more aggressive optimisations, +use `prod-vtk`. If you just need the core functions and none of the visuals, choose the `core` sets +(`dev-core` or `prod-core`). ### Available build types In order to switch optimizations on, tell cmake to use the release build type: - +``` cmake -DCMAKE_BUILD_TYPE=Release ../meanie3D - +``` Note that there have been some problems on Linux with aggressive optimization and NetCDF. Your mileage may vary. You should try the release build in any event, since it speeds up performance a lot. If you observe unexpected problems in reading/writing NetCDF files, you may have fallen victim to the problem and revert to standard build (leave the -DCMAKE_BUILD_TYPE=Release). Once all dependencies are successfully resolved, install the product by calling the following: - +``` make install - +``` There are a number of options to customize your installation: ### -DWITH_VTK=YES @@ -120,9 +165,9 @@ is switched on by default. Meanie3D has a number of regression tests, that cover the core algorithms and collection classes. This will become important to you if you should decide to work on the core algorithms yourself. The tests are a good method of making sure you haven't broken anything critical. The unit tests can then be run by calling - +``` make test - +``` ### -DWITH_PYTHON=YES In addition to the C++ binaries, the installation also uses pip to put a python package in place. This adds an executable simply called `meanie3D`, which is a front-end to the core functions. It allows @@ -134,9 +179,9 @@ For details on the configuration file format, see HOWTO.md. In order to start developing your own Meanie3D code, it might be useful to have API documentation of the various classes in the project. If you have doxygen installed, you can call the following make command to create a browsable HTML documentation in doc/html (open the file index.html). - +``` make docs - +``` ### -DWITH_RADOLAN_UTILS=YES This will result in compilation of the `meanie3D-radolan2cfm` utility, which converts files in RADOLAN format to a cf-metadata compliant netCDF file, which then can be used to run the tracking. @@ -152,50 +197,6 @@ if this flag is set: The package comes with a tool `meanie3D-trackstats-conrad` which analyses KONRAD tracks in a way that makes the data comparable to meanie3D data. This is a specialized tool developed in the context of the OASE project. -## Build presets -A number of presets are provided to make the process easier. Those are selected via the -DPRESET= -flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available presets are: -* `docker` - * Code optimizations for build type 'Release' - * Core functions and python frontend -* `dev-core` - * Code optimizations for build type 'Debug' - * Core functions and python frontend -* `dev-vtk` - * Code optimizations for build type 'Debug' - * Core functions and python frontend - * Visualization code - * Tests - * Documentation -* `dev-all` - * Code optimizations for build type 'Debug' - * Core functions and python frontend - * Visualization code - * Tests - * Documentation - * RADOLAN, Satellite and KONRAD utilities -* `prod-core` - * Code optimizations for build type 'Release' - * Core functions and python frontend. -* `prod-vtk` - * Code optimizations for build type 'Release' - * Core functions and python frontend - * Visualization code - * Tests - * Documentation -* `prod-all` - * Code optimizations for build type 'Release' - * Core functions and python frontend - * Visualization code - * Tests - * Documentation - * RADOLAN, Satellite and KONRAD utilities - -The term "core functions" refers to the detection, tracking and track evaluation code. For local -development in most cases, the preset `dev-vtk` is sufficient. For more aggressive optimisations, -use `prod-vtk`. If you just need the core functions and none of the visuals, choose the `core` sets -(`dev-core` or `prod-core`). - ## Frequently Asked Quesions ### I'm having trouble compiling the code From 7fedcfb391e06f83cf2ca40c51a57404e949dc02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 13:08:56 +0100 Subject: [PATCH 31/65] Documentation updated. --- README.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d9a6554..850a355 100644 --- a/README.md +++ b/README.md @@ -136,9 +136,12 @@ calling the following: ``` make install ``` -There are a number of options to customize your installation: -### -DWITH_VTK=YES +### Options +If you would like to select your own options, you can leave the PRESET parameter and switch things +on and off using the following options: + +#### -DWITH_VTK=YES Because of the large footprint of the VTK package, the visualization code is disabled by default. While visualization is not necessary to run the algorithm, it can be useful to develop your parameters to have visual queues as to what is happening. Setting this flag will result in the following changes: @@ -157,43 +160,45 @@ have visual queues as to what is happening. Setting this flag will result in the *Important Notes*: Visualization is switched off in the Docker version. The visualiation code uses libradolan. If you do switch this on, you will be required to install libradolan as well. -### -DWITH_OPENMP=YES +#### -DWITH_OPENMP=YES In order to speed the process up, meanie3D uses OpenMP to parallelize it's computation. This option is switched on by default. -### -D WITH_TESTS=YES +#### -D WITH_TESTS=YES Meanie3D has a number of regression tests, that cover the core algorithms and collection classes. This will become important to you if you should decide to work on the core algorithms yourself. The tests are a good method of making sure you haven't broken anything critical. The unit tests can then be run by calling ``` make test ``` -### -DWITH_PYTHON=YES + +#### -DWITH_PYTHON=YES In addition to the C++ binaries, the installation also uses pip to put a python package in place. This adds an executable simply called `meanie3D`, which is a front-end to the core functions. It allows you to put your clustering and tracking parameters down in the form of a configuration file. The entire pipeline is handled based on this configuration file. *This is the recommended way to run the software*. For details on the configuration file format, see HOWTO.md. -### -DWITH_DOCS=YES +#### -DWITH_DOCS=YES In order to start developing your own Meanie3D code, it might be useful to have API documentation of the various classes in the project. If you have doxygen installed, you can call the following make command to create a browsable HTML documentation in doc/html (open the file index.html). ``` make docs ``` -### -DWITH_RADOLAN_UTILS=YES + +#### -DWITH_RADOLAN_UTILS=YES This will result in compilation of the `meanie3D-radolan2cfm` utility, which converts files in RADOLAN format to a cf-metadata compliant netCDF file, which then can be used to run the tracking. -### -DWITH_SATELLITE_UTILS=YES +#### -DWITH_SATELLITE_UTILS=YES The package comes with binaries to perform some conversion on satellite data. Those binaries were provided in the context of research work for the OASE project. The following binaries will be provided if this flag is set: * `meanie3D-satconv` - Converts spectral radiance to equivalent brightness temperature or vice versa * `meanie3D-parallax_correction` - Applies parallax correction to mseviri satellite data in OASE composite files. -### -DWITH_KONRAD_UTILS=YES +#### -DWITH_KONRAD_UTILS=YES The package comes with a tool `meanie3D-trackstats-conrad` which analyses KONRAD tracks in a way that makes the data comparable to meanie3D data. This is a specialized tool developed in the context of the OASE project. From 6f8df19fa35657f9e60bd652b528896591fdc65e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 13:09:57 +0100 Subject: [PATCH 32/65] Documentation updated. --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 850a355..58abf49 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ calling the following: If you would like to select your own options, you can leave the PRESET parameter and switch things on and off using the following options: -#### -DWITH_VTK=YES +#### -DWITH_VTK=ON/OFF Because of the large footprint of the VTK package, the visualization code is disabled by default. While visualization is not necessary to run the algorithm, it can be useful to develop your parameters to have visual queues as to what is happening. Setting this flag will result in the following changes: @@ -160,11 +160,11 @@ have visual queues as to what is happening. Setting this flag will result in the *Important Notes*: Visualization is switched off in the Docker version. The visualiation code uses libradolan. If you do switch this on, you will be required to install libradolan as well. -#### -DWITH_OPENMP=YES +#### -DWITH_OPENMP=ON/OFF In order to speed the process up, meanie3D uses OpenMP to parallelize it's computation. This option is switched on by default. -#### -D WITH_TESTS=YES +#### -D WITH_TESTS=ON/OFF Meanie3D has a number of regression tests, that cover the core algorithms and collection classes. This will become important to you if you should decide to work on the core algorithms yourself. The tests are a good method of making sure you haven't broken anything critical. The unit tests can then be run by calling @@ -172,14 +172,14 @@ method of making sure you haven't broken anything critical. The unit tests can t make test ``` -#### -DWITH_PYTHON=YES +#### -DWITH_PYTHON=ON/OFF In addition to the C++ binaries, the installation also uses pip to put a python package in place. This adds an executable simply called `meanie3D`, which is a front-end to the core functions. It allows you to put your clustering and tracking parameters down in the form of a configuration file. The entire pipeline is handled based on this configuration file. *This is the recommended way to run the software*. For details on the configuration file format, see HOWTO.md. -#### -DWITH_DOCS=YES +#### -DWITH_DOCS=ON/OFF In order to start developing your own Meanie3D code, it might be useful to have API documentation of the various classes in the project. If you have doxygen installed, you can call the following make command to create a browsable HTML documentation in doc/html (open the file index.html). @@ -187,18 +187,18 @@ create a browsable HTML documentation in doc/html (open the file index.html). make docs ``` -#### -DWITH_RADOLAN_UTILS=YES +#### -DWITH_RADOLAN_UTILS=ON/OFF This will result in compilation of the `meanie3D-radolan2cfm` utility, which converts files in RADOLAN format to a cf-metadata compliant netCDF file, which then can be used to run the tracking. -#### -DWITH_SATELLITE_UTILS=YES +#### -DWITH_SATELLITE_UTILS=ON/OFF The package comes with binaries to perform some conversion on satellite data. Those binaries were provided in the context of research work for the OASE project. The following binaries will be provided if this flag is set: * `meanie3D-satconv` - Converts spectral radiance to equivalent brightness temperature or vice versa * `meanie3D-parallax_correction` - Applies parallax correction to mseviri satellite data in OASE composite files. -#### -DWITH_KONRAD_UTILS=YES +#### -DWITH_KONRAD_UTILS=ON/OFF The package comes with a tool `meanie3D-trackstats-conrad` which analyses KONRAD tracks in a way that makes the data comparable to meanie3D data. This is a specialized tool developed in the context of the OASE project. From f3a87720d9850863a88a466052a8e6b4352db576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 22 Jun 2020 15:43:34 +0100 Subject: [PATCH 33/65] Added an 'uninstall' target. More documentation updated. Fixed some kinks here and there. Version update. Added presets for minsizerel build types (fast and fast-vtk) --- CMakeLists.txt | 58 +++++++++++++++++++------- README.md | 37 +++++++++++----- include/meanie3D/version.h | 2 +- python/CMakeLists.txt | 2 +- python/README.txt | 2 +- python/meanie3D/__init__.py | 2 +- python/meanie3D/visualisation/utils.py | 8 ++-- python/setup.py | 6 +-- 8 files changed, 82 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 85068db..80efd69 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,12 +21,12 @@ ENABLE_LANGUAGE(CXX) SET(${PROJECT_NAME}_MAJOR_VERSION 1) SET(${PROJECT_NAME}_MINOR_VERSION 6) -SET(${PROJECT_NAME}_PATCH_LEVEL 0) -SET(PACKAGE_VERSION "1.6.0") +SET(${PROJECT_NAME}_PATCH_LEVEL 1) +SET(PACKAGE_VERSION "1.6.1") SET(PROJECT_URL "http://git.meteo.uni-bonn.de/projects/meanie3d") SET(PROJECT_LICENSE "MIT License") SET(PROJECT_AUTHOR "Juergen Simon") -SET(PROJECT_AUTHOR_EMAIL "tachyonimpulse@gmail.com") +SET(PROJECT_AUTHOR_EMAIL "simon@webtecc.com") # ------------------------------------- # Paths and build configuration @@ -155,6 +155,28 @@ ELSEIF (PRESET MATCHES "prod-vtk") SET(SATELLITE_ENABLED OFF) SET(KONRAD_ENABLED OFF) SET(DOCS_ENABLED OFF) +ELSEIF (PRESET MATCHES "fast-core") + MESSAGE(STATUS "Selected preset ${PRESET}") + SET(CMAKE_BUILD_TYPE "MinSizeRel") + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED OFF) + SET(TESTS_ENABLED OFF) + SET(RADOLAN_ENABLED OFF) + SET(SATELLITE_ENABLED OFF) + SET(KONRAD_ENABLED OFF) + SET(DOCS_ENABLED OFF) +ELSEIF (PRESET MATCHES "fast-vtk") + MESSAGE(STATUS "Selected preset ${PRESET}") + SET(CMAKE_BUILD_TYPE "MinSizeRel") + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED ON) + SET(TESTS_ENABLED OFF) + SET(RADOLAN_ENABLED OFF) + SET(SATELLITE_ENABLED OFF) + SET(KONRAD_ENABLED OFF) + SET(DOCS_ENABLED OFF) ELSE() IF (NOT ${CMAKE_BUILD_TYPE}) SET(CMAKE_BUILD_TYPE "Release") @@ -169,6 +191,11 @@ ELSE() SET(DOCS_ENABLED ${WITH_DOCS}) ENDIF() +# ------------------------------------- +# Compilation type +# ------------------------------------- +MESSAGE(STATUS "Build type: ${CMAKE_BUILD_TYPE}") + # ------------------------------------- # Multithreading / Parallelism Choices # ------------------------------------- @@ -776,10 +803,7 @@ ENDIF (VTK_ENABLED) IF (TESTS_ENABLED) # Test Suite ADD_EXECUTABLE(m3D-test-kernel test/kernels/testcases.h test/kernels/test.cpp) - TARGET_LINK_LIBRARIES(m3D-test-kernel - meanie3D - gtest - ${Boost_LIBRARIES}) + TARGET_LINK_LIBRARIES(m3D-test-kernel meanie3D gtest ${Boost_LIBRARIES} OpenMP::OpenMP_CXX) SET_TARGET_PROPERTIES(m3D-test-kernel PROPERTIES LINKER_LANGUAGE CXX) # Unit tests for vector utils @@ -790,10 +814,7 @@ IF (TESTS_ENABLED) test/collections/tests_set.h test/collections/tests_vector.h test/collections/test.cpp) - TARGET_LINK_LIBRARIES(m3D-test-collections - gtest - meanie3D - ${Boost_LIBRARIES}) + TARGET_LINK_LIBRARIES(m3D-test-collections gtest meanie3D ${Boost_LIBRARIES} OpenMP::OpenMP_CXX) SET_TARGET_PROPERTIES(m3D-test-collections PROPERTIES LINKER_LANGUAGE CXX) # Unit tests for class FeatureSpace @@ -821,7 +842,8 @@ IF (TESTS_ENABLED) ${LZ4} ${HDF5_LIBRARIES} ${VTK_LIBRARIES} - ${Blitz_LIBRARY}) + ${Blitz_LIBRARY} + OpenMP::OpenMP_CXX) SET_TARGET_PROPERTIES(m3D-test-featurespace PROPERTIES LINKER_LANGUAGE CXX) # KDTree @@ -851,7 +873,8 @@ IF (TESTS_ENABLED) ${Boost_LIBRARIES} ${NETCDF_LIBRARIES} ${HDF5_LIBRARIES} - ${VTK_LIBRARIES}) + ${VTK_LIBRARIES} + OpenMP::OpenMP_CXX) SET_TARGET_PROPERTIES(m3D-test-detection PROPERTIES LINKER_LANGUAGE CXX) ADD_TEST(KernelTest m3D-test-kernel) @@ -899,6 +922,13 @@ IF (VTK_ENABLED) INSTALL(TARGETS meanie3D-cfm2vtk RUNTIME DESTINATION "/usr/local/bin") ENDIF (VTK_ENABLED) +ADD_CUSTOM_TARGET(${PROJECT_NAME}-uninstall COMMAND cat install_manifest.txt | xargs rm -rvf) +ADD_CUSTOM_TARGET(uninstall) +ADD_DEPENDENCIES(uninstall ${PROJECT_NAME}-uninstall) +IF (PYTHON_ENABLED) + ADD_DEPENDENCIES(uninstall ${PROJECT_NAME}-python-uninstall) +ENDIF() + # ------------------------------------------------------------- # # Documentation # ------------------------------------------------------------- # @@ -913,7 +943,7 @@ IF (DOCS_ENABLED) ENDIF (DOXYGEN_DOT_EXECUTABLE) CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/doxy.in ${PROJECT_BINARY_DIR}/Doxyfile) FILE(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/html/radolan/doxygen) - ADD_CUSTOM_TARGET(meanie3D-docs COMMAND ${DOXYGEN_EXECUTABLE} ${PROJECT_BINARY_DIR}/Doxyfile) + ADD_CUSTOM_TARGET(docs COMMAND ${DOXYGEN_EXECUTABLE} ${PROJECT_BINARY_DIR}/Doxyfile) MESSAGE(STATUS "Doxygen source code documentation will be available") ENDIF (DOXYGEN_EXECUTABLE) ENDIF() \ No newline at end of file diff --git a/README.md b/README.md index 58abf49..43b491f 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Meanie3D comes with a number of dependencies that need to be installed prior to * Boost 1.56 or better (http://www.boost.org) * FLANN 1.8.0 or better (http://www.cs.ubc.ca/research/flann/) * Blitz++ (http://sourceforge.net/projects/blitz/) -* shapelib v1.3+ (http://shapelib.maptools.org) +* OpenMP (libomp) (https://www.openmp.org) * NetCDF 4.2 or better (http://www.unidata.ucar.edu/software/netcdf/) *including* the netcdf4-C++ API * HDF5 (http://www.hdfgroup.org/HDF5/) * Python 2.5 or better (https://www.python.org) @@ -43,9 +43,15 @@ Meanie3D comes with a number of dependencies that need to be installed prior to * NetCDF4-python (try running @pip install netCDF4@ or download and install from http://unidata.github.io/netcdf4-python) * libradolan (http://meteo-ubonn.github.io/radolan/) +Optional: + The following libraries may be used, if they are switched on with the appropriate flags +* Data format + * shapelib v1.3+ (http://shapelib.maptools.org) - When present, certain utilities have additional options for outputting + data in ESRI shapefile format. * Visualization: * Imagemagick (https://imagemagick.org) + * gnuplot (http://gnuplot.sourceforge.net) * VTK 7.0 or better (http://www.vtk.org) * Visit 3.0.0 or better (https://wci.llnl.gov/simulation/computer-codes/visit) * Generated source code documentation: @@ -54,11 +60,10 @@ The following libraries may be used, if they are switched on with the appropriat ### Compiler Prerequisites Meanie3D uses OpenMP by default (-DWITH_OPENMP=1) and requires an OpenMP enabled compiler, such as: - * GNU 9.0 or better (Linux, Mac) -* OpenMP/LLVM (Mac) - a clang implementation supporting OpenMP (http://clang-omp.github.io). Install this -compiler for optimal results on OSX. You can disable the OpenMP implementation adding the -flags -DWITH_OPENMP=0 to your cmake call. +* OpenMP/LLVM (Mac) - a clang implementation supporting OpenMP (http://clang-omp.github.io). Note that as of clang 10.0.0 OpenMP is supported, but you will still have to install the omp library (`brew install libomp`) + +You can disable the OpenMP implementation adding the flags -DWITH_OPENMP=0 to your cmake call. ### Build instructions @@ -80,7 +85,7 @@ Create a build directory: ``` ### Build presets -A number of presets are provided to make the process easier. Those are selected via the -DPRESET= +A number of presets are provided to make the process easier. Those are selected via the -DPRESET=\ flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available presets are: * `docker` * Code optimizations for build type 'Release' @@ -124,22 +129,32 @@ use `prod-vtk`. If you just need the core functions and none of the visuals, cho (`dev-core` or `prod-core`). ### Available build types -In order to switch optimizations on, tell cmake to use the release build type: +The following build types are available: +* Debug (debug symbols, no optimizations) +* Release (some debug symbols, optimized) +* RelWithDebInfo (debug symbols, optimized) +* MinSizeRel (minimal footprint, aggressive optimizations, no debug symbols) + +To select a build type, use the flag `CMAKE_BUILD_TYPE` like so: ``` cmake -DCMAKE_BUILD_TYPE=Release ../meanie3D ``` -Note that there have been some problems on Linux with aggressive optimization and NetCDF. Your mileage may vary. + +*Note:* there have been some problems on Linux with aggressive optimization and NetCDF. Your mileage may vary. You should try the release build in any event, since it speeds up performance a lot. If you observe unexpected problems in reading/writing NetCDF files, you may have fallen victim to the problem and revert to standard build -(leave the -DCMAKE_BUILD_TYPE=Release). Once all dependencies are successfully resolved, install the product by +(leave the -DCMAKE_BUILD_TYPE=Debug). Once all dependencies are successfully resolved, install the product by calling the following: ``` make install ``` +*Note:* When using presets, the preset defines the build type. Setting this flag together with a preset has +no effect on the preset's choice. ### Options If you would like to select your own options, you can leave the PRESET parameter and switch things -on and off using the following options: +on and off. When using presets, the preset defines the options. Setting any of the following options +together with a -DPRESET=\ option does not alter the preset's settings. #### -DWITH_VTK=ON/OFF Because of the large footprint of the VTK package, the visualization code is disabled by default. While @@ -220,7 +235,7 @@ one installed (http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2. If cmake has trouble locating packages you installed, check the files in the cmake_modules directory and adjust the search paths there instead of hardwiring paths into the CMakeLists.txt. If your system layout is not considered and you have to make adjustments to the search paths, please send a copy of -your Find.cmake file to juergen.simon@uni-bonn.de so I can include your changes in the next release +your Find\.cmake file to simon@webtecc.com so I can include your changes in the next release and maybe save someone else the trouble. ## Visualisation diff --git a/include/meanie3D/version.h b/include/meanie3D/version.h index 6655aa3..420aa09 100644 --- a/include/meanie3D/version.h +++ b/include/meanie3D/version.h @@ -29,7 +29,7 @@ namespace m3D { - static const std::string VERSION = "1.6.0"; + static const std::string VERSION = "1.6.1"; void print_version() { diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 1fbd107..1f2fb2d 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -50,6 +50,6 @@ IF (Python2_FOUND) DEPENDS meanie3D) ADD_CUSTOM_TARGET(${PROJECT_NAME} ALL COMMAND ${Python2_EXECUTABLE} setup.py build) ADD_CUSTOM_TARGET(${PROJECT_NAME}-clean COMMAND rm -rf build dist *.egg-info setup.py meanie3D/__init__.py README.txt) - + ADD_CUSTOM_TARGET(${PROJECT_NAME}-uninstall COMMAND ${Python2_EXECUTABLE} -m pip uninstall meanie3D) INSTALL(CODE "EXECUTE_PROCESS(COMMAND \"${Python2_EXECUTABLE}\" setup.py install WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})") ENDIF () diff --git a/python/README.txt b/python/README.txt index 9f38a00..97a5828 100644 --- a/python/README.txt +++ b/python/README.txt @@ -1,6 +1,6 @@ Project Name: meanie3D-python -Version: 1.6.0 +Version: 1.6.1 Url: http://git.meteo.uni-bonn.de/projects/meanie3d Author: Juergen Simon Email: tachyonimpulse@gmail.com diff --git a/python/meanie3D/__init__.py b/python/meanie3D/__init__.py index ee2c0c4..a8a5d56 100644 --- a/python/meanie3D/__init__.py +++ b/python/meanie3D/__init__.py @@ -1,6 +1,6 @@ __author__ = 'Juergen Simon' __email__ = 'tachyonimpulse@gmail.com' -__version__ = '1.6.0' +__version__ = '1.6.1' __url__ = 'http://git.meteo.uni-bonn.de/projects/meanie3d' __all__ = ['app', 'visualisation', 'resources'] diff --git a/python/meanie3D/visualisation/utils.py b/python/meanie3D/visualisation/utils.py index 905916d..f743df2 100644 --- a/python/meanie3D/visualisation/utils.py +++ b/python/meanie3D/visualisation/utils.py @@ -36,10 +36,12 @@ ret_code, paths_string = meanie3D.app.external.execute_command('python','-c "import sys; print sys.path"', True) if ret_code == 0: print "Attempting to locate python module netCDF4" - result = meanie3D.app.utils.find_in_paths(["/usr/lib","/usr/local/lib"],"netCDF4","site-packages") + paths = paths_string.split(",") + paths.append('/usr/lib') + paths.append('/usr/local/lib') + result = meanie3D.app.utils.find_in_paths(paths,"netCDF4","site-packages") if not result: - result = meanie3D.app.utils.find_in_paths(["/usr/lib","/usr/local/lib"],"netCDF4","dist-packages") - + result = meanie3D.app.utils.find_in_paths(paths,"netCDF4","dist-packages") if not result: print "Failed to locate python module netCDF4" exit(-1) diff --git a/python/setup.py b/python/setup.py index a91344e..ae7e4fc 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,15 +1,15 @@ from setuptools import find_packages, setup setup( name='meanie3D', - version='1.6.0', + version='1.6.1', packages=["meanie3D","meanie3D.app", "meanie3D.visualisation", "meanie3D.resources"], include_package_data=True, url='http://git.meteo.uni-bonn.de/projects/meanie3d', license='MIT License', author='Juergen Simon', - author_email='tachyonimpulse@gmail.com', + author_email='simon@webtecc.com', zip_safe = False, - description='A python script for running the meanie3D clustering and tracking algorithms as well as processing and visualising the results.', + description='Python scripts for running the meanie3D clustering and tracking algorithms and visualising the results.', entry_points = { 'console_scripts' : [ 'meanie3D = meanie3D.meanie3D_run:main' From 1a963ce0fdc0d229a8ffc2edb44af48e6570cb70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 23 Jun 2020 11:26:20 +0100 Subject: [PATCH 34/65] Fix configuration for labels. --- python/configurations/meanie3D-2D.json | 6 +++--- python/configurations/meanie3D-3D.json | 6 +++--- python/configurations/oase-3d-xband_oase_zh.json | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/configurations/meanie3D-2D.json b/python/configurations/meanie3D-2D.json index 1bb9780..3470b7f 100644 --- a/python/configurations/meanie3D-2D.json +++ b/python/configurations/meanie3D-2D.json @@ -436,14 +436,14 @@ "LabelAttributes": { "legendFlag": 0, "restrictNumberOfLabels": 0, - "specifyTextColor1": 0, - "textColor1": [ + + "textFont1.color": [ 255, 0, 0, 0 ], - "textHeight1": 0.02, + "textFont1.scale": 0.02, "formatTemplate": "%g", "horizontalJustification": "Left", "verticalJustification": "Bottom" diff --git a/python/configurations/meanie3D-3D.json b/python/configurations/meanie3D-3D.json index 8695f52..05749b1 100644 --- a/python/configurations/meanie3D-3D.json +++ b/python/configurations/meanie3D-3D.json @@ -500,14 +500,14 @@ "LabelAttributes": { "legendFlag": 0, "restrictNumberOfLabels": 0, - "specifyTextColor1": 0, - "textColor1": [ + + "textFont1.color": [ 255, 0, 0, 0 ], - "textHeight1": 0.02, + "textFont1.scale": 0.02, "formatTemplate": "%g", "horizontalJustification": "Left", "verticalJustification": "Bottom" diff --git a/python/configurations/oase-3d-xband_oase_zh.json b/python/configurations/oase-3d-xband_oase_zh.json index 05d4598..73534b2 100644 --- a/python/configurations/oase-3d-xband_oase_zh.json +++ b/python/configurations/oase-3d-xband_oase_zh.json @@ -563,14 +563,14 @@ "LabelAttributes": { "legendFlag": 0, "restrictNumberOfLabels": 0, - "specifyTextColor1": 0, - "textColor1": [ + + "textFont1.color": [ 255, 0, 0, 0 ], - "textHeight1": 0.02, + "textFont1.scale": 0.02, "formatTemplate": "%g", "horizontalJustification": "Left", "verticalJustification": "Bottom" From b2b7f2e59845e59e62405a56e750f91ac655208c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 23 Jun 2020 11:26:45 +0100 Subject: [PATCH 35/65] Python cleanup and linting. --- python/meanie2tic/meanie2tic.py | 355 +++++++++--------- python/meanie3D/__init__.py | 15 +- python/meanie3D/__init__.py.in | 12 +- python/meanie3D/app/__init__.py | 6 +- python/meanie3D/app/analysis.py | 63 ++-- python/meanie3D/app/external.py | 163 +++++---- python/meanie3D/app/postprocessing.py | 279 +++++++------- python/meanie3D/app/tracking.py | 195 +++++----- python/meanie3D/app/utils.py | 162 ++++----- python/meanie3D/meanie3D_run.py | 46 ++- python/meanie3D/resources/clusters_visit.py | 23 +- python/meanie3D/resources/tracks_visit.py | 18 +- python/meanie3D/visualisation/__init__.py | 8 +- python/meanie3D/visualisation/clusters.py | 230 ++++++------ python/meanie3D/visualisation/tracks.py | 59 +-- python/meanie3D/visualisation/utils.py | 379 ++++++++++---------- python/setup.py | 33 +- 17 files changed, 1028 insertions(+), 1018 deletions(-) diff --git a/python/meanie2tic/meanie2tic.py b/python/meanie2tic/meanie2tic.py index 5016747..7015f7a 100755 --- a/python/meanie2tic/meanie2tic.py +++ b/python/meanie2tic/meanie2tic.py @@ -3,205 +3,208 @@ import csv import getopt import json -import os import sys + def load_json(filename): - ''' Load a json file - :param filename: - :return: - ''' - json_data=open(filename) - data = json.load(json_data) - json_data.close() - return data; + """ Load a json file + :param filename: + :return: + """ + json_data = open(filename) + data = json.load(json_data) + json_data.close() + return data; def write_csv(json): - ''' - Converts a meanie3D track-dictionary.json into the output files - required by the tracking intercomparison project. The output - consists of two CSV files, that are constructed as follows: - - Table 1 characterizes the objects identified and should provide for all time steps the following information in 12 columns - (1) Time step (e.g. ranging from 1 to 1819) [uint] - (2) Object ID (should be unique for each object per time step) [uint] - (3) X coordinate of object center [float] - (4) Y coordinate of object center [float] - (5) Size of object (=number of grid boxes included) [uint] - (6) X_min (minimal X location of the object) [float] - (7) X_max (maximal X location of the object) [float] - (8) Y_min (minimal Y location of the object) [float] - (9) Y_max (maximal Y location of the object) [float] - (10) Mean LWP of the object (=average value of LWP for all grid boxes included) [float] - (11) Min. LWP of the object (=smallest LWP value included) [float] - (12) Max LWP of the object (=maximal LWP value included) [float] - - Table 2 should contain the tracking information for all time steps (rows) in 4 columns: - (1) Basis time step t_i (e.g. ranging from 1 to 1818) [uint] - (2) Object ID identified in time step t_i (from Table 1) [uint] - (3) Object ID of identified successor object in time step t_i+1 (from Table 1) [uint] - (4) Weight factor to decide object relevance after split/merge event for life - time statistics (optional), e.g. transition probability or overlap ratio - - :param json: - :return: - ''' - - # - # Write the objects - # - - objects_file = open('objects.csv', 'wb') - if (not objects_file): - sys.stderr.write("Could not open 'objects.csv' for writing.") - sys.exit(-1) - - objects = csv.writer(objects_file, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL) - - print("Writing object information:\n") - tracks = json['tracks']; - for ti in range(0,len(tracks)): - track = tracks[ti] - clusters = track['clusters'] - - # if the first row, write a header - if ti == 0: - header = ['timestep','id','centerx','centery','volume', - 'boxxmin','boxxmax','boxymin','boxymax', - 'lwpmean','lwpmin','lwpmax'] - print(header) - objects.writerow(header) - - for ci in range(0,len(clusters)): - cluster = clusters[ci] - - row = [] - - # (1) Time step (e.g. ranging from 1 to 1819) - row.append(cluster['step'] + 1), - - # (2) Object ID (should be unique for each object per time step) - row.append(cluster['uuid']) - - # (3) X coordinate of object center - # (4) Y coordinate of object center - mode = cluster['mode'] - for mi in range(0,len(mode)): - row.append(mode[mi]) - - # (5) Size of object (=number of grid boxes included) - row.append(cluster['size']) - - # (6) X_min (minimal X location of the object) - # (7) X_max (maximal X location of the object) - # (8) Y_min (minimal Y location of the object) - # (9) Y_max (maximal Y location of the object) - row.append(cluster['bounding_box_min'][1]); - row.append(cluster['bounding_box_max'][1]); - row.append(cluster['bounding_box_min'][0]); - row.append(cluster['bounding_box_max'][0]); - - # (10) Mean LWP of the object (=average value of LWP for all grid boxes included) - row.append(cluster['median'][0]) - - # (11) Min. LWP of the object (=smallest LWP value included) - row.append(cluster['min'][0]) - - # (12) Max LWP of the object (=maximal LWP value included) - row.append(cluster['max'][0]) - - print(row) - objects.writerow(row) - - - # - # Write the tracks - # - print("Writing tracking information:\n") - - tracks_file = open('tracks.csv', 'wb') - if (not tracks_file): - sys.stderr.write("Could not open 'tracks.csv' for writing.") - sys.exit(-1) - - tracks = csv.writer(tracks_file, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL) - - tree = json['tree'] - nodes = tree['nodes'] - links = tree['links'] + """ + Converts a meanie3D track-dictionary.json into the output files + required by the tracking intercomparison project. The output + consists of two CSV files, that are constructed as follows: + + Table 1 characterizes the objects identified and should provide for all time steps the following information in 12 columns + (1) Time step (e.g. ranging from 1 to 1819) [uint] + (2) Object ID (should be unique for each object per time step) [uint] + (3) X coordinate of object center [float] + (4) Y coordinate of object center [float] + (5) Size of object (=number of grid boxes included) [uint] + (6) X_min (minimal X location of the object) [float] + (7) X_max (maximal X location of the object) [float] + (8) Y_min (minimal Y location of the object) [float] + (9) Y_max (maximal Y location of the object) [float] + (10) Mean LWP of the object (=average value of LWP for all grid boxes included) [float] + (11) Min. LWP of the object (=smallest LWP value included) [float] + (12) Max LWP of the object (=maximal LWP value included) [float] + + Table 2 should contain the tracking information for all time steps (rows) in 4 columns: + (1) Basis time step t_i (e.g. ranging from 1 to 1818) [uint] + (2) Object ID identified in time step t_i (from Table 1) [uint] + (3) Object ID of identified successor object in time step t_i+1 (from Table 1) [uint] + (4) Weight factor to decide object relevance after split/merge event for life + time statistics (optional), e.g. transition probability or overlap ratio + + :param json: + :return: + """ + + # + # Write the objects + # + + objects_file = open('objects.csv', 'wb') + if (not objects_file): + sys.stderr.write("Could not open 'objects.csv' for writing.") + sys.exit(-1) + + objects = csv.writer(objects_file, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL) + + print("Writing object information:\n") + tracks = json['tracks']; + for ti in range(0, len(tracks)): + track = tracks[ti] + clusters = track['clusters'] + + # if the first row, write a header + if ti == 0: + header = ['timestep', 'id', 'centerx', 'centery', 'volume', + 'boxxmin', 'boxxmax', 'boxymin', 'boxymax', + 'lwpmean', 'lwpmin', 'lwpmax'] + print(header) + objects.writerow(header) + + for ci in range(0, len(clusters)): + cluster = clusters[ci] + + row = [] + + # (1) Time step (e.g. ranging from 1 to 1819) + row.append(cluster['step'] + 1), + + # (2) Object ID (should be unique for each object per time step) + row.append(cluster['uuid']) + + # (3) X coordinate of object center + # (4) Y coordinate of object center + mode = cluster['mode'] + for mi in range(0, len(mode)): + row.append(mode[mi]) + + # (5) Size of object (=number of grid boxes included) + row.append(cluster['size']) + + # (6) X_min (minimal X location of the object) + # (7) X_max (maximal X location of the object) + # (8) Y_min (minimal Y location of the object) + # (9) Y_max (maximal Y location of the object) + row.append(cluster['bounding_box_min'][1]); + row.append(cluster['bounding_box_max'][1]); + row.append(cluster['bounding_box_min'][0]); + row.append(cluster['bounding_box_max'][0]); + + # (10) Mean LWP of the object (=average value of LWP for all grid boxes included) + row.append(cluster['median'][0]) + + # (11) Min. LWP of the object (=smallest LWP value included) + row.append(cluster['min'][0]) + + # (12) Max LWP of the object (=maximal LWP value included) + row.append(cluster['max'][0]) + + print(row) + objects.writerow(row) + + # + # Write the tracks + # + print("Writing tracking information:\n") + + tracks_file = open('tracks.csv', 'wb') + if (not tracks_file): + sys.stderr.write("Could not open 'tracks.csv' for writing.") + sys.exit(-1) + + tracks = csv.writer(tracks_file, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL) + + tree = json['tree'] + nodes = tree['nodes'] + links = tree['links'] + + for li in range(0, len(links)): + link = links[li] - for li in range(0,len(links)): - link = links[li] + source_index = link['source'] - 1 + source = nodes[source_index] - source_index = link['source'] -1 - source = nodes[source_index] + target_index = link['target'] - 1 + target = nodes[target_index] - target_index = link['target'] - 1 - target = nodes[target_index] + # if the first row, write a header + if li == 0: + header = ['timestep', 'idstep1', 'idstep2', 'weight'] + print(header) + tracks.writerow(header) - # if the first row, write a header - if li == 0: - header = ['timestep','idstep1','idstep2','weight'] - print(header) - tracks.writerow(header) + row = [] + # (1) Basis time step t_i (e.g. ranging from 1 to 1818) + row.append(source['step'] + 1) - row = [] - # (1) Basis time step t_i (e.g. ranging from 1 to 1818) - row.append(source['step']+1) + # (2) Object ID identified in time step t_i (from Table 1) + row.append(source['uuid']) - # (2) Object ID identified in time step t_i (from Table 1) - row.append(source['uuid']) + # (3) Object ID of identified successor object in time step t_i+1 (from Table 1) + row.append(target['uuid']) - # (3) Object ID of identified successor object in time step t_i+1 (from Table 1) - row.append(target['uuid']) + # (4) Weight factor to decide object relevance after split/merge event for life + row.append(0) - # (4) Weight factor to decide object relevance after split/merge event for life - row.append(0) + print(row) + tracks.writerow(row) - print(row) - tracks.writerow(row) # ---------------------------------------------------------------------------- ## Prints usage and exits # def usage(): - print("meanie2tic.py [--file,-f] [--help,-h]") - print("Converts meanie3D track-dictionary.json into CSV files for Tracking Intercomparison project") - print("--file,-f : track dictionary file (defaults to 'track-dictionary.json' if omitted)") - print("--help, -h : print this message and exit.") - sys.exit(1) - return + print("meanie2tic.py [--file,-f] [--help,-h]") + print("Converts meanie3D track-dictionary.json into CSV files for Tracking Intercomparison project") + print("--file,-f : track dictionary file (defaults to 'track-dictionary.json' if omitted)") + print("--help, -h : print this message and exit.") + sys.exit(1) + return + + # ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------- ## Main function def main(): - # Parse command line - try: - argv = sys.argv[1:] - opts, args = getopt.getopt(argv, "f:h", ["file","help"]) - except getopt.GetoptError as detail: - print(detail) - sys.exit(2) - - if len(sys.argv) < 2: - usage() - - filename = "track-dictionary.json" - for o, a in opts: - if o in ["--help","-h"]: - usage() - elif o in ["--file","-f"]: - filename = a - - json = load_json(filename) - if not json: - sys.stderr.write("ERROR:could not read file %s" % filename) - sys.exit(-1) - - write_csv(json) - return + # Parse command line + try: + argv = sys.argv[1:] + opts, args = getopt.getopt(argv, "f:h", ["file", "help"]) + except getopt.GetoptError as detail: + print(detail) + sys.exit(2) + + if len(sys.argv) < 2: + usage() + + filename = "track-dictionary.json" + for o, a in opts: + if o in ["--help", "-h"]: + usage() + elif o in ["--file", "-f"]: + filename = a + + json = load_json(filename) + if not json: + sys.stderr.write("ERROR:could not read file %s" % filename) + sys.exit(-1) + + write_csv(json) + return + # ---------------------------------------------------------------------------- @@ -210,5 +213,5 @@ def main(): # ---------------------------------------------------------------------------- if __name__ == "__main__": - main() - sys.exit(0) + main() + sys.exit(0) diff --git a/python/meanie3D/__init__.py b/python/meanie3D/__init__.py index a8a5d56..c386870 100644 --- a/python/meanie3D/__init__.py +++ b/python/meanie3D/__init__.py @@ -7,24 +7,27 @@ import os.path import sys + def getVersion(): - ''' + """ :return:meanie3D package version - ''' + """ from . import __version__ return __version__ + def getHome(): - ''' + """ meanie3D package location :return: - ''' + """ return os.path.abspath(os.path.dirname(__file__)) + def appendSystemPythonPath(): - ''' + """ Returns the system's python path to import outside modules. - ''' + """ system_python_path = ":/Library/Frameworks/SQLite3.framework/Versions/C/Python/2.7:/Library/Frameworks/GEOS.framework/Versions/3/Python/2.7:/Library/Python/2.7/site-packages/numpy-override:/Library/Python/2.7/site-packages/matplotlib-override:/Library/Frameworks/GDAL.framework/Versions/1.11/Python/2.7/site-packages:/Library/Python/2.7/site-packages/pip-1.5.2-py2.7.egg:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python27.zip:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-darwin:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-mac:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/plat-mac/lib-scriptpackages:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-tk:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-old:/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/lib-dynload:/Library/Python/2.7/site-packages:/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python:/System/Library/Frameworks/Python.framework/Versions/2.7/Extras/lib/python/PyObjC".strip() paths = system_python_path.split(':') for path in paths: diff --git a/python/meanie3D/__init__.py.in b/python/meanie3D/__init__.py.in index 0a58148..97ea321 100644 --- a/python/meanie3D/__init__.py.in +++ b/python/meanie3D/__init__.py.in @@ -8,23 +8,23 @@ import os.path import sys def getVersion(): - ''' + """ :return:meanie3D package version - ''' + """ from . import __version__ return __version__ def getHome(): - ''' + """ meanie3D package location :return: - ''' + """ return os.path.abspath(os.path.dirname(__file__)) def appendSystemPythonPath(): - ''' + """ Returns the system's python path to import outside modules. - ''' + """ system_python_path = "${SYSTEM_PYTHON_PATH}".strip() paths = system_python_path.split(':') for path in paths: diff --git a/python/meanie3D/app/__init__.py b/python/meanie3D/app/__init__.py index e75109a..1b45304 100644 --- a/python/meanie3D/app/__init__.py +++ b/python/meanie3D/app/__init__.py @@ -1,9 +1,11 @@ -__all__ = ["analysis","external", "postprocessing","utils","tracking"] +__all__ = ["analysis", "external", "postprocessing", "utils", "tracking"] __have_visit__ = None __visitPath__ = None __visitImportPath__ = None -import os, sys +import os +import sys + from meanie3D.app.utils import findVisitPaths # Import visit package diff --git a/python/meanie3D/app/analysis.py b/python/meanie3D/app/analysis.py index 7a45d05..fe3a7fa 100644 --- a/python/meanie3D/app/analysis.py +++ b/python/meanie3D/app/analysis.py @@ -1,4 +1,4 @@ -''' +""" The MIT License (MIT) (c) Juergen Simon 2014 (juergen.simon@uni-bonn.de) @@ -20,85 +20,82 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' +""" import getopt import sys + import meanie3D + def loadTrackingDictionary(path): - ''' + """ Loads a tracking dictionary as output by meanie3D-trackstats -t + TODO: implement me :param path: :return: parsed JSON dictionary - ''' + """ return + def constructTreeJson(trackingDictionary): - ''' + """ Parses a tracking dictionary and creates data for visualising the tracks in d3.js + TODO: implement me :param trackingDictionary: :return: - ''' + """ + def showTrackTree(tree): - ''' + """ Displays the given tree JSON in d3.js + TODO: implement me :param tree: :return: - ''' + """ return def usage(): - ''' + """ Prints help and exits :return: - ''' - print "meanie3D-trackgraph -f " - print "Analyses a track dictionary and shows a track graph with splits, merges etc." - print "-f : tracking dictionary file" - print "--help, -h : print this message and exits." - print "--version : prints the version information and exits" - sys.exit(1) - return + """ + print("meanie3D-trackgraph -f ") + print("Analyses a track dictionary and shows a track graph with splits, merges etc.") + print("-f : tracking dictionary file") + print("--help, -h : print this message and exits.") + print("--version : prints the version information and exits") def run(): - ''' + """ Parses the command line and performs the analysis of the track dictionary. :return: - ''' + """ # Parse command line try: argv = sys.argv[1:] - opts, args = getopt.getopt(argv, "c:f:s:o:r:h", ["resume","help","version","start=","end="]) + opts, args = getopt.getopt(argv, "c:f:s:o:r:h", ["resume", "help", "version", "start=", "end="]) except getopt.GetoptError as detail: - print detail + print(detail) sys.exit(2) num_params = 0 - dictionaryPath = None - for o, a in opts: - - if o in ['--file','-f']: - dictionaryPath = a + if o in ['--file', '-f']: num_params = num_params + 1 - elif o in ["--help"]: usage() - sys.exit() - + sys.exit(0) elif o in ["--version"]: meanie3D.getVersion() - + sys.exit(0) else: usage() - + sys.exit(1) if num_params < 2: usage() - - uses_time = False \ No newline at end of file diff --git a/python/meanie3D/app/external.py b/python/meanie3D/app/external.py index 349e3e0..c499a61 100644 --- a/python/meanie3D/app/external.py +++ b/python/meanie3D/app/external.py @@ -1,4 +1,4 @@ -''' +""" The MIT License (MIT) (c) Juergen Simon 2014 (juergen.simon@uni-bonn.de) @@ -20,62 +20,70 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' +""" import os -import subprocess import platform -from subprocess import Popen +import subprocess +import sys +from subprocess import Popen, CalledProcessError # --------------------------------------------- # Locating commands # --------------------------------------------- # Constant denoting failure to locate a command -COMMAND_NOT_FOUND="NOT_FOUND" -COMMAND_MAP={} - -## Get DYLD_LIBRARY_PATH depending on operating system. OSX needs -# ------------------------------------------------------------------- -# special work because of the homebrew gfx libraries, which get -# in the way of the system libraries. -# \return DYLD_LIBRARY_PATH +COMMAND_NOT_FOUND = "NOT_FOUND" +COMMAND_MAP = {} + + def __get_dyld_library_path(): + """ + Get DYLD_LIBRARY_PATH depending on operating system. OSX needs special work because of the + homebrew gfx libraries, which get in the way of the system libraries. + :return: DYLD_LIBRARY_PATH + """ path = "/usr/local/lib" if platform.system() == 'Darwin': - path = "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/:"+path + path = "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/:" + path return path -# Returns the standard search path for the binaries -# @return search path + def __command_search_paths(): - paths=['/usr/local/bin', - '/usr/bin/', - '/bin', - '/sbin', - '/usr/local/visit/bin', - '/Applications/VisIt.app/Contents/Resources/bin'] + """ + Returns the standard search path for the binaries + :return: search path + """ + paths = ['/usr/local/bin', + '/usr/bin/', + '/bin', + '/sbin', + '/usr/local/visit/bin', + '/Applications/VisIt.app/Contents/Resources/bin'] return paths -# ------------------------------------------------------------------- -## Get the complete shell command with added DYLD_LIBRARY_PATH etc. -# for the given executable. -# \param executable name (like meanie3D-detect, meanie3D-track etc.) -# \return shell command to run the binary + def getCommand(executable): - bin_prefix = "export DYLD_LIBRARY_PATH="+__get_dyld_library_path()+";" + """ + Get the complete shell command with added DYLD_LIBRARY_PATH etc. for the given executable. + :param executable: executable name (like meanie3D-detect, meanie3D-track etc.) + :return: shell command to run the binary + """ + bin_prefix = "export DYLD_LIBRARY_PATH=" + __get_dyld_library_path() + ";" command = bin_prefix + executable return command -# Checks if the given command can be found in the given path. -# If the recurse flag is set, all subdirectories are searched recursively. -# -# @param command -# @param path -# @param recurse -# @return fully qualified command path or "NOT_FOUND" -# @throws IOError if a command is not executable + def locateCommandInPath(command, path, recurse): + """ + Checks if the given command can be found in the given path. + If the recurse flag is set, all subdirectories are searched recursively. + :param command: + :param path: + :param recurse: + :return: fully qualified command path or "NOT_FOUND" + :except: IOError if a command is not executable + """ result = COMMAND_NOT_FOUND if os.path.exists(path) and os.path.isdir(path): files = os.listdir(path) @@ -104,23 +112,24 @@ def locateCommandInPath(command, path, recurse): def hasCommand(command): """ Tests if the command is available - @param command name - @returns True or False + :param command: command name + :returns: True or False """ try: locateCommands([command]) + return True except IOError: return False - + def locateCommandsInPaths(command_list, path_list, recurse): """ Attempts to locate the given executables in the given filesystem paths. - @param command_list - list of strings containing the commands - @param path_list - list of strings containing the search paths - @param recurse - boolean. If the method recurses into each path. - @returns a dictionary containing the command names mapping to the command paths - @throws IOError if a command can not be located or is not executable + :param command_list: list of strings containing the commands + :param path_list: list of strings containing the search paths + :param recurse: boolean. If True the method recurses into each path. + :returns: a dictionary containing the command names mapping to the command paths + :except: IOError if a command can not be located or is not executable """ for command in command_list: result = COMMAND_NOT_FOUND @@ -130,37 +139,43 @@ def locateCommandsInPaths(command_list, path_list, recurse): COMMAND_MAP[command] = result break if result == COMMAND_NOT_FOUND: - raise IOError('Could not locate command '+command) + raise IOError('Could not locate command ' + command) return COMMAND_MAP -# Attempts to locate the given executables in the standard filesystem -# paths (/usr/local/bin /var/opt/bin /usr/bin) -# -# @param list of strings containing the commands -# @return a dictionary containing the command names mapping to the command paths -# @throws IOError if a command can not be located or is not executable def locateCommands(command_list, recurse=True): + """ + Attempts to locate the given executables in the standard filesystem + paths (/usr/local/bin /var/opt/bin /usr/bin) + :param command_list: list of strings containing the commands + :param recurse: boolean + :return: a dictionary containing the command names mapping to the command paths + """ return locateCommandsInPaths(command_list, __command_search_paths(), recurse) -## Executes the given command and returns the code -# -# \param command -# \param parameters -# \return process returncode -# \throws IOError if a command is not executable -# \throws Exception if a command was not searched for before attempting to run -def execute_command(command, parameters, withStdOut=False, silent=True): + +def execute_command(command, parameters, return_output=False, silent=True): + """ + Executes the given command and returns the code + :param command: Command to run + :param parameters: Command arguments + :param return_output: If True, capture and return content of stdout and stderr + :param silent: If True, pipe everything to /dev/null. Mutually exclusive with return_output + :exception ValueError: command can not be located + :exception IOError: command failed to execute + :return: (return_code, stdout, stderr) + """ # Make sure we searched for the command first - if not COMMAND_MAP.has_key(command): - raise Exception( - "Requested command: %s was not found. All commands must be located before they can be used") + if command not in COMMAND_MAP: + print("WARN: should locate command %s before using it", command) + if not hasCommand(command): + raise ValueError("Requested command: %s was not found") cmd = getCommand(COMMAND_MAP.get(command)) + ' ' + parameters - if withStdOut: + if return_output: p = Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() - return p.returncode, out + return p.returncode, out, err else: if silent: p = Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -169,3 +184,25 @@ def execute_command(command, parameters, withStdOut=False, silent=True): p = Popen(cmd, shell=True, stdout=subprocess.STDOUT, stderr=subprocess.STDOUT) p.communicate() return p.returncode + + +def run(command, args, return_output=False): + """ + Convenience wrapper around executing external commands + :param command: Command to execute + :param args: Arguments to the program (as string) + :param return_output: Should return output of stderr? + :return: ({boolean}, {stdout}, {stderr}) + """ + try: + return_code, outstream, errstream = execute_command(command, args, return_output) + return (return_code == 0), outstream, errstream + except ValueError as ex: + print("ERROR: %s", ex.__str__()) + except CalledProcessError as ex: + print("ERROR: %s", ex.__str__()) + except IOError as ex: + print("ERROR: %s", ex.__str__()) + except: + print("ERROR:%s" % sys.exc_info()[0]) + return False diff --git a/python/meanie3D/app/postprocessing.py b/python/meanie3D/app/postprocessing.py index 3a72b0b..45e67da 100644 --- a/python/meanie3D/app/postprocessing.py +++ b/python/meanie3D/app/postprocessing.py @@ -1,4 +1,4 @@ -''' +""" The MIT License (MIT) (c) Juergen Simon 2014 (juergen.simon@uni-bonn.de) @@ -20,46 +20,41 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' +""" import glob import os import shutil -import sys import tempfile import time # Own packages -from . import __have_visit__ -from . import __visitPath__ - import meanie3D -from . import utils +from . import __have_visit__ from . import external +from . import utils # ---------------------------------------------------------------------------- - if __have_visit__: - import visit external.locateCommands(['visit']) # ---------------------------------------------------------------------------- def check_configuration(configuration): - ''' + """ Checks the consistency of the given configuration and hotfixes :param configuration: :return: - ''' + """ # Check the configuration dimensions = utils.getValueForKeyPath(configuration, 'data.dimensions') - if (not dimensions): - print "ERROR:configuration must contain 'dimensions'" + if not dimensions: + print("ERROR:configuration must contain 'dimensions'") return -1 - if (not len(dimensions) in [2, 3]): - print "ERROR:Can only process 2D or 3D" + if not len(dimensions) in [2, 3]: + print("ERROR:Can only process 2D or 3D") return -1 # Check that visualiseTracks has .vtk to work from @@ -67,9 +62,10 @@ def check_configuration(configuration): if postprocessing: tracks = utils.getValueForKeyPath(postprocessing, 'tracks') if tracks: - if utils.getValueForKeyPath(tracks, 'visualiseTracks') and not utils.getValueForKeyPath(tracks, - 'meanie3D-trackstats.vtk_tracks'): - print "WARNING: tracks.visualiseTracks = True but tracks.meanie3D-trackstats.vtk_tracks = False. Correcting." + if utils.getValueForKeyPath(tracks, 'visualiseTracks') and not \ + utils.getValueForKeyPath(tracks, 'meanie3D-trackstats.vtk_tracks'): + print("WARNING: tracks.visualiseTracks = True but tracks.meanie3D-trackstats.vtk_tracks " + "= False. Correcting.") utils.setValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.vtk_tracks', True) # Complement vtk_dimensions to make our life a little easier down the road. @@ -77,40 +73,42 @@ def check_configuration(configuration): vtkDimensions = utils.getValueForKeyPath(configuration, 'data.vtkDimensions') if vtkDimensions: vtkDimString = ",".join(vtkDimensions) - utils.setValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.vtkDimensions', + utils.setValueForKeyPath(configuration, + 'postprocessing.tracks.meanie3D-trackstats.vtkDimensions', vtkDimString) # Make sure that plotStats has gnuplot files to work with - if utils.getValueForKeyPath(tracks, 'plotStats') and not utils.getValueForKeyPath(tracks, - 'meanie3D-trackstats.gnuplot'): - print "WARNING: track.plot_stats = True but tracks.meanie3D-trackstats.gnuplot = False. Correcting." + if utils.getValueForKeyPath(tracks, 'plotStats') and not \ + utils.getValueForKeyPath(tracks, 'meanie3D-trackstats.gnuplot'): + print("WARNING: track.plot_stats = True but " + "tracks.meanie3D-trackstats.gnuplot = False. Correcting.") utils.setValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.gnuplot', True) # ---------------------------------------------------------------------------- def run_trackstats(configuration, directory): - ''' + """ Runs the meanie3D-trackstats command. :param configuration: :param directory: :return: - ''' + """ pconf = configuration['postprocessing'] - if not 'tracks' in pconf: - print "Skipping meanie3D-trackstats because 'tracks' configuration is missing" + if 'tracks' not in pconf: + print("Skipping meanie3D-trackstats because 'tracks' configuration is missing") return False tracks = pconf['tracks'] - if not 'meanie3D-trackstats' in tracks: - print "Skipping meanie3D-trackstats because 'tracks.meanie3D-trackstats' configuration is missing" + if 'meanie3D-trackstats' not in tracks: + print("Skipping meanie3D-trackstats because 'tracks.meanie3D-trackstats' configuration is missing") return False - if utils.getValueForKeyPath(configuration,'skip_trackstats'): - print "Skipping meanie3D-trackstats because --skip-trackstats is present" + if utils.getValueForKeyPath(configuration, 'skip_trackstats'): + print("Skipping meanie3D-trackstats because --skip-trackstats is present") return False - print "Running meanie3D-trackstats for %s" % directory + print("Running meanie3D-trackstats for %s" % directory) conf = tracks['meanie3D-trackstats'] os.chdir(directory) @@ -118,83 +116,76 @@ def run_trackstats(configuration, directory): # Assemble command line params haveFiles = True params = [] - if (conf['dictionary'] == True): + if conf['dictionary']: params.append("-t") haveFiles = haveFiles and os.path.exists('track-dictionary.json') - if (conf['gnuplot'] == True): + if conf['gnuplot']: params.append("-g") haveFiles = haveFiles and os.path.exists('plot_stats.gp') - if (conf['length'] == True): + if conf['length']: params.append("--create-length-statistics") haveFiles = haveFiles and os.path.exists("lengths.txt") haveFiles = haveFiles and os.path.exists("lengths-hist.txt") - if (conf['length_classes']): + if conf['length_classes']: params.append("--length-histogram-classes=%s" % conf['length_classes']) - if (conf['speed'] == True): + if conf['speed']: params.append("--create-speed-statistics") haveFiles = haveFiles and os.path.exists("speeds.txt") haveFiles = haveFiles and os.path.exists("speeds-hist.txt") - if (conf['speed_classes']): + if conf['speed_classes']: params.append("--speed-histogram-classes=%s" % conf['speed_classes']) - if (conf['direction'] == True): + if conf['direction']: params.append("--create-direction-statistics") haveFiles = haveFiles and os.path.exists("directions.txt") haveFiles = haveFiles and os.path.exists("directions-hist.txt") - if (conf['direction_classes']): + if conf['direction_classes']: params.append("--direction-histogram-classes=%s" % conf['direction_classes']) - if (conf['size'] == True): + if conf['size']: params.append("--create-cluster-statistics") haveFiles = haveFiles and os.path.exists("sizes.txt") haveFiles = haveFiles and os.path.exists("sizes-hist.txt") - if (conf['size_classes']): + if conf['size_classes']: params.append("--cluster-histogram-classes=%s" % conf['size_classes']) - if (conf['cumulated'] == True): + if conf['cumulated']: params.append("--create-cumulated-size-statistics") - if (conf['cumulated_classes']): + if conf['cumulated_classes']: params.append("--size-histogram-classes=%s" % conf['cumulated_classes']) - if (conf['vtk_tracks'] == True): + if conf['vtk_tracks']: params.append("--write-center-tracks-as-vtk ") # Can' be sure in this case, better safe than sorry haveFiles = False - if (__have_visit__ and conf['vtkDimensions']): + if __have_visit__ and conf['vtkDimensions']: params.append("--vtk-dimensions=%s" % conf['vtkDimensions']) params.append("-s netcdf") if configuration['resume'] and haveFiles: - print "Skipping meanie3D-trackstats" + print("Skipping meanie3D-trackstats") os.chdir("..") return True - return_code = -1 - try: - print "meanie3D-trackstats %s" % (" ".join(params)) - return_code = external.execute_command("meanie3D-trackstats", " ".join(params), silent=True) - except: - print "ERROR:%s" % sys.exc_info()[0] - raise - + success = external.run("meanie3D-trackstats", " ".join(params)) os.chdir("..") - return (return_code == 0) + return success # ---------------------------------------------------------------------------- def plot_trackstats(configuration, directory): - ''' + """ Runs gnuplot to produce .eps files :param configuration: :param directory: :return: - ''' + """ pconf = configuration['postprocessing'] - if not 'tracks' in pconf: + if 'tracks' not in pconf: return False tracks = pconf['tracks'] - if not 'meanie3D-trackstats' in tracks: + if 'meanie3D-trackstats' not in tracks: return False conf = tracks['meanie3D-trackstats'] - print "Plotting .eps files for %s" % directory + print("Plotting .eps files for %s" % directory) os.chdir(directory) @@ -202,7 +193,7 @@ def plot_trackstats(configuration, directory): f = open('plot_stats.gp', 'w') # track length - if (conf['length'] == True): + if conf['length']: f.write('set term postscript\n') f.write('set output "lengths-hist.eps"\n') f.write('set title "Distribution of track length"\n') @@ -212,7 +203,7 @@ def plot_trackstats(configuration, directory): f.write('plot "lengths-hist.txt" with boxes\n') # sizes - if (conf['size']): + if conf['size']: f.write('# cluster size\n') f.write('set output "sizes-hist.eps"\n') f.write('set title "Distribution of cluster size"\n') @@ -226,7 +217,7 @@ def plot_trackstats(configuration, directory): f.write('unset logscale x\n') # speed - if (conf['speed']): + if conf['speed']: f.write('set output "speeds-hist.eps"\n') f.write('set title "Distribution of cluster speeds"\n') f.write('set xlabel "Cluster speed in [m/s]"\n') @@ -235,50 +226,43 @@ def plot_trackstats(configuration, directory): f.write('plot "speeds-hist.txt" with boxes\n') # directions - if (conf['direction']): + if conf['direction']: f.write('set output "directions-hist.eps"\n') f.write('set title "Distribution of tracking direction"\n') f.write('set xlabel "Cluster direction in [deg]"\n') f.write('set ylabel "Number of clusters"\n') f.write('set xtics 15\n') f.write('plot "directions-hist.txt" with boxes\n') - f.close() - - return_code = -1 - try: - return_code = external.execute_command("gnuplot", "plot_stats.gp") - except: - print "ERROR:%s" % sys.exc_info()[0] - + success = external.run("gnuplot", "plot_stats.gp") os.chdir("..") - return (return_code == 0) + return success # ---------------------------------------------------------------------------- def plot_stats_comparison(configuration): - ''' + """ Plots comparison charts for each of the parameters to meanie3D-trackstats, namely--length, --directions, --speed and --direction :param configuration: :return: - ''' + """ scales = configuration['scales'] if not scales: - print "No scales found, no comparison is done." + print("No scales found, no comparison is done.") return False pconf = configuration['postprocessing'] if not pconf: return False - if not utils.getValueForKeyPath(pconf,'tracks.runScaleComparison'): + if not utils.getValueForKeyPath(pconf, 'tracks.runScaleComparison'): return False conf = utils.getValueForKeyPath(pconf, 'tracks.meanie3D-trackstats') - print "Plotting .eps files for comparison" + print("Plotting .eps files for comparison") # create a tempfile for gnuplot f = open('plot_stats_comparison.gp', 'w') @@ -287,7 +271,7 @@ def plot_stats_comparison(configuration): f.write('set style fill transparent pattern 4 border 2\n') # track length - if utils.getValueForKeyPath(conf,'length'): + if utils.getValueForKeyPath(conf, 'length'): f.write('set title "Distribution of track length"\n') f.write('set xlabel "Track length in [#steps]"\n') f.write('set ylabel "log(#tracks)"\n') @@ -298,14 +282,14 @@ def plot_stats_comparison(configuration): f.write('plot ') for i in range(0, len(scales)): scale = scales[i] - f.write('"scale%s/lengths-hist.txt" with linespoints title "t=%s"' % (scale,scale)) - if (i < len(scales)-1): + f.write('"scale%s/lengths-hist.txt" with linespoints title "t=%s"' % (scale, scale)) + if i < len(scales) - 1: f.write(",") f.write('\nunset logscale y\n') f.write('set autoscale x\n') # cluster size - if utils.getValueForKeyPath(conf,'size'): + if utils.getValueForKeyPath(conf, 'size'): f.write('set title "Distribution of cluster size"\n') f.write('set xlabel "log(cluster size in [#gridpoints])"\n') f.write('set ylabel "number of clusters"\n') @@ -316,14 +300,14 @@ def plot_stats_comparison(configuration): f.write('plot ') for i in range(0, len(scales)): scale = scales[i] - f.write('"scale%s/sizes-hist.txt" with linespoints title "t=%s"' % (scale,scale)) - if (i < len(scales)-1): + f.write('"scale%s/sizes-hist.txt" with linespoints title "t=%s"' % (scale, scale)) + if i < len(scales) - 1: f.write(",") f.write('\nunset logscale x\n') f.write('set autoscale x\n') # speed - if utils.getValueForKeyPath(conf,'speed'): + if utils.getValueForKeyPath(conf, 'speed'): f.write('set title "Distribution of cluster speeds"\n') f.write('set xlabel "Cluster speed in [m/s]"\n') f.write('set ylabel "Number of clusters"\n') @@ -332,13 +316,13 @@ def plot_stats_comparison(configuration): f.write('plot ') for i in range(0, len(scales)): scale = scales[i] - f.write('"scale%s/speeds-hist.txt" with linespoints title "t=%s"' % (scale,scale)) - if (i < len(scales)-1): + f.write('"scale%s/speeds-hist.txt" with linespoints title "t=%s"' % (scale, scale)) + if i < len(scales) - 1: f.write(",") f.write("\n") # directions - if utils.getValueForKeyPath(conf,'direction'): + if utils.getValueForKeyPath(conf, 'direction'): f.write('set title "Distribution of tracking direction"\n') f.write('set xlabel "Cluster direction in [deg]"\n') f.write('set ylabel "Number of clusters"\n') @@ -348,54 +332,48 @@ def plot_stats_comparison(configuration): f.write('plot ') for i in range(0, len(scales)): scale = scales[i] - f.write('"scale%s/directions-hist.txt" with linespoints title "t=%s"' % (scale,scale)) - if (i < len(scales)-1): + f.write('"scale%s/directions-hist.txt" with linespoints title "t=%s"' % (scale, scale)) + if i < len(scales) - 1: f.write(",") f.write("\n") - f.close() - return_code = -1 - try: - return_code = external.execute_command("gnuplot", "plot_stats_comparison.gp") - except: - print "ERROR:%s" % sys.exc_info()[0] - + success = external.run("gnuplot", "plot_stats_comparison.gp") os.chdir("..") - return (return_code == 0) + return success def run_comparison(configuration): - ''' + """ Runs cross-scale analysis. :param configuration: :return: - ''' - print "Running cross-scale comparison" + """ + print("Running cross-scale comparison") return plot_stats_comparison(configuration) # ---------------------------------------------------------------------------- -def compile_and_run_template(templatePath, configuration, replacements, directory): - ''' +def compile_and_run_template(template_path, configuration, replacements, directory): + """ Compiles the gievn template into an executable python script and runs it through visit. - :param templatePath: + :param template_path: :param configuration: :param replacements: :param directory: :return:True if the command was successful, False else. - ''' + """ scriptFilename = tempfile.mktemp() + ".py" # scriptFilename = os.path.abspath("generated.py") - # print "\tWriting python script for visualisation to: " + scriptFilename - with open(templatePath) as infile, open(scriptFilename, 'w') as outfile: + # print("\tWriting python script for visualisation to: " + scriptFilename) + with open(template_path) as infile, open(scriptFilename, 'w') as outfile: for line in infile: for src, target in replacements.iteritems(): line = line.replace(src, target) outfile.write(line) - # print "\tDone." + # print("\tDone.") # Compile command line params for visualisation params = "-s %s" % scriptFilename @@ -405,31 +383,28 @@ def compile_and_run_template(templatePath, configuration, replacements, director # change into directory os.chdir(directory) + # Run visualisation - returnCode = -1 - try: - print "Executing visit: " + params - returnCode = external.execute_command('visit', params, silent=True) - except: - print "ERROR:%s" % sys.exc_info()[0] + success = external.run('visit', params) # Change back out os.chdir('..') - return (returnCode == 0) + + return success # ---------------------------------------------------------------------------- def visualise_tracks(configuration, directory): - ''' + """ Visualises the tracks found. :param configuration: :param directory: :return: - ''' - print "Visualising tracks for %s" % directory + """ + print("Visualising tracks for %s" % directory) if not __have_visit__: - print "Visit not found. Skipping." + print("Visit not found. Skipping.") return home = os.path.abspath(os.path.dirname(meanie3D.__file__) + os.path.sep + os.path.pardir) templatePath = home + os.path.sep + os.path.sep.join(("meanie3D", "resources", "tracks_visit.py")) @@ -446,15 +421,15 @@ def visualise_tracks(configuration, directory): # ---------------------------------------------------------------------------- def visualise_clusters(configuration, directory): - ''' + """ Visualises the clusters. :param configuration: :param directory: :return: - ''' - print "Visualising clusters for %s" % directory + """ + print("Visualising clusters for %s" % directory) if not __have_visit__: - print "Visit not found. Skipping." + print("Visit not found. Skipping.") return home = os.path.abspath(os.path.dirname(meanie3D.__file__) + os.path.sep + os.path.pardir) templatePath = home + os.path.sep + os.path.sep.join(("meanie3D", "resources", "clusters_visit.py")) @@ -463,9 +438,9 @@ def visualise_clusters(configuration, directory): 'P_NETCDF_DIR': configuration['source_directory'], 'P_RESUME': str(configuration['resume']), 'P_M3D_HOME': home, - 'P_USES_TIME' : str(configuration['uses_time']), - 'P_START_TIME_INDEX' : str(configuration['start_time_index']), - 'P_END_TIME_INDEX' : str(configuration['end_time_index']), + 'P_USES_TIME': str(configuration['uses_time']), + 'P_START_TIME_INDEX': str(configuration['start_time_index']), + 'P_END_TIME_INDEX': str(configuration['end_time_index']), 'P_CONFIGURATION_FILE': os.path.abspath(configuration['config_file']) } return compile_and_run_template(templatePath, configuration, replacements, directory) @@ -474,12 +449,12 @@ def visualise_clusters(configuration, directory): # ---------------------------------------------------------------------------- def copy_html_files(configuration, directory): - ''' + """ Copies the files required for html presentation. :param configuration: :param directory: :return: - ''' + """ home = os.path.abspath(os.path.dirname(meanie3D.__file__) + os.path.sep + os.path.pardir) path = home + os.path.sep + os.path.sep.join(("meanie3D", "resources", "index.html")) shutil.copy(path, os.path.abspath(directory)) @@ -493,25 +468,25 @@ def copy_html_files(configuration, directory): # ---------------------------------------------------------------------------- def cleanup(configuration, directory): - ''' + """ Removes any files that are not part of the results and were produced in any of the previous steps. :param configuration: :param directory: :return: - ''' - print "Cleaning temporary files for %s" % directory + """ + print("Cleaning temporary files for %s" % directory) os.chdir(directory) - if utils.getValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.vtk_tracks'): - for filename in glob.glob('*.vtk'): - os.remove(filename) + # if utils.getValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.vtk_tracks'): + # for filename in glob.glob('*.vtk'): + # os.remove(filename) if utils.getValueForKeyPath(configuration, 'postprocessing.tracks.meanie3D-trackstats.gnuplot'): for filename in glob.glob('*.gp'): os.remove(filename) - if (os._exists("visitlog.py")): - os.remove("visitlog.py") + # if (os._exists("visitlog.py")): + # os.remove("visitlog.py") os.chdir("..") return @@ -520,44 +495,44 @@ def cleanup(configuration, directory): # ---------------------------------------------------------------------------- def run(configuration): - ''' + """ Runs postprocessing steps according to the section 'postprocessing' in the configuration. :param configuration: :return: - ''' + """ # Check configuration section 'postprocessing' check_configuration(configuration) - if not 'postprocessing' in configuration: + if 'postprocessing' not in configuration: return # In case scale parameters were given, the output dirs are scaleXYZ. # Otherwise it's 'clustering'. To be safe, iterate over both scales = configuration['scales'] if scales: - directories = ['scale%s' % (scale) for scale in scales] + directories = ['scale%s' % scale for scale in scales] else: directories = ['clustering'] - print "Processing directories: %s" % str(directories) + print("Processing directories: %s" % str(directories)) for directory in directories: if os.path.isdir(directory): - print "Processing %s" % directory + print("Processing %s" % directory) else: - print "ERROR:%s is not a directory!" % directory + print("ERROR:%s is not a directory!" % directory) continue # run the track statistics + start_time = 0 if configuration['time_operations']: - print "Running trackstats ..." + print("Running trackstats ...") start_time = time.time() - if (run_trackstats(configuration, directory)): - + if run_trackstats(configuration, directory): if configuration['time_operations']: - print "Finished. (%.2f seconds)" % (time.time()-start_time) + print("Finished. (%.2f seconds)" % (time.time() - start_time)) # Copy HTML files copy_html_files(configuration, directory) @@ -570,28 +545,28 @@ def run(configuration): if __have_visit__ and utils.getValueForKeyPath(configuration, 'postprocessing.tracks.visualiseTracks'): if configuration['time_operations']: - print "Visualising tracks ..." + print("Visualising tracks ...") start_time = time.time() visualise_tracks(configuration, directory) if configuration['time_operations']: - print "Finished. (%.2f seconds)" % (time.time()-start_time) + print("Finished. (%.2f seconds)" % (time.time() - start_time)) if __have_visit__ and utils.getValueForKeyPath(configuration, 'postprocessing.clusters.visualiseClusters'): if configuration['time_operations']: - print "Visualising clusters ..." + print("Visualising clusters ...") start_time = time.time() visualise_clusters(configuration, directory) if configuration['time_operations']: - print "Finished. (%.2f seconds)" % (time.time()-start_time) + print("Finished. (%.2f seconds)" % (time.time() - start_time)) cleanup(configuration, directory) - if (utils.getValueForKeyPath(configuration, 'postprocessing.tracks.runScaleComparison')): + if utils.getValueForKeyPath(configuration, 'postprocessing.tracks.runScaleComparison'): run_comparison(configuration) # ---------------------------------------------------------------------------- diff --git a/python/meanie3D/app/tracking.py b/python/meanie3D/app/tracking.py index 32a0544..1bb2e12 100644 --- a/python/meanie3D/app/tracking.py +++ b/python/meanie3D/app/tracking.py @@ -1,4 +1,4 @@ -''' +""" The MIT License (MIT) (c) Juergen Simon 2014 (juergen.simon@uni-bonn.de) @@ -20,60 +20,61 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' - +""" import glob import os import os.path +import sys import time from subprocess import call from meanie3D.app import external, utils -## Checks the configuration for mandatory items etc. -# \param The configuration to check. -# \throws RuntimeError in case something is amiss. + def checkConfiguration(conf): + """ + Checks the configuration for mandatory items etc. + :param conf: The configuration to check. + :except: RuntimeError in case something is amiss. + :return: + """ if not conf['data']: raise RuntimeError("Missing 'data' section") data = conf['data'] - if not 'variables' in data: + if "variables" not in data: raise RuntimeError("Missing 'data.variables'") - if not 'dimensions' in data: + if "dimensions" not in data: raise RuntimeError("Missing 'data.dimensions'") - # TODO: finish this return -## Runs a batch of files through the clustering and tracking. -# -# \param configuration with the following keys: -# -# description : a description of the configuration -# netcdf_dir : directory containing files to process -# output_dir : directory to write results to -# m3d_home : ${MEANIE3D_HOME} environment variable -# resume : if True then pick off where you left off, if False -# all previous results are deleted and processing -# starts from scratch -# meanie3D-detect : parameters for meanie3D-detect -# meanie3D-track: parameters for meanie3D-track -# scale : scale parameter (optional) -# use_previous: True: using previous results to enhance tracking -# Absent or False: no use of previous results. -# use_ci_score : Use the CI-Score algorithm -# -# \param time index the index in time dimension to read and process. -# -def run(config,time_index): +def run(config, time_index): + """ + Runs a batch of files through the clustering and tracking. + :param config: configuration with the following keys: + description : a description of the configuration + netcdf_dir : directory containing files to process + output_dir : directory to write results to + m3d_home : ${MEANIE3D_HOME} environment variable + resume : if True then pick off where you left off, if False all previous results + are deleted and processing starts from scratch + meanie3D-detect : parameters for meanie3D-detect + meanie3D-track : parameters for meanie3D-track + scale : scale parameter (optional) + use_previous : True: using previous results to enhance tracking. Absent or False: + no use of previous results. + use_ci_score : Use the CI-Score algorithm + :param time_index: Time index to start at. + :return: + """ checkConfiguration(config) - data = utils.getSafe(config,'data') - detection = utils.getSafe(config,'detection') - tracking = utils.getSafe(config,'tracking') + data = utils.getSafe(config, 'data') + detection = utils.getSafe(config, 'detection') + tracking = utils.getSafe(config, 'tracking') # Piece together the command line params for detection and tracking tracking_params = "" @@ -90,13 +91,13 @@ def run(config,time_index): for variable in data['variables']: variables.append(variable['name']) - if (utils.getSafe(variable,'lowerThreshold')): + if utils.getSafe(variable, 'lowerThreshold'): lowerThresholds.append("%s=%s" % (variable['name'], str(variable['lowerThreshold']))) - if (utils.getSafe(variable,'upperThreshold')): + if utils.getSafe(variable, 'upperThreshold'): upperThresholds.append("%s=%s" % (variable['name'], str(variable['upperThreshold']))) - if (utils.getSafe(variable,'replacementValue')): + if utils.getSafe(variable, 'replacementValue'): replacementValues.append("%s=%s" % (variable['name'], str(variable['replacementValue']))) detect_params += " -v %s" % ','.join(variables) @@ -110,84 +111,84 @@ def run(config,time_index): if replacementValues: detect_params += " --replacement-values %s" % ','.join(lowerThresholds) - if utils.getSafe(data,'vtkDimensions'): + if utils.getSafe(data, 'vtkDimensions'): detect_params += " --vtk-dimensions %s" % ','.join(data['vtkDimensions']) if detection: - minClusterSize = utils.getSafe(detection,'minClusterSize') + minClusterSize = utils.getSafe(detection, 'minClusterSize') if minClusterSize: detect_params = "%s -m %d" % (detect_params, minClusterSize) - additionalParams = utils.getSafe(detection,'meanie3D-detect') + additionalParams = utils.getSafe(detection, 'meanie3D-detect') if detect_params: detect_params = "%s %s" % (detect_params, additionalParams) if tracking: - - if utils.getSafe(tracking,'meanie3D-track'): + + if utils.getSafe(tracking, 'meanie3D-track'): tracking_params = "%s %s" % (tracking_params, tracking['meanie3D-track']) - - if utils.getSafe(tracking,'histogramVariable'): + + if utils.getSafe(tracking, 'histogramVariable'): tracking_params = "%s --tracking-variable %s" % (tracking_params, tracking['histogramVariable']) - - if utils.getSafe(tracking,'histogramWeight'): + + if utils.getSafe(tracking, 'histogramWeight'): tracking_params = "%s --wt %s" % (tracking_params, tracking['histogramWeight']) else: tracking_params = "%s --wt 0" % tracking_params - - if utils.getSafe(tracking,'positionWeight'): + + if utils.getSafe(tracking, 'positionWeight'): tracking_params = "%s --wr %s" % (tracking_params, tracking['positionWeight']) else: tracking_params = "%s --wr 0" % tracking_params - - if utils.getSafe(tracking,'sizeWeight'): + + if utils.getSafe(tracking, 'sizeWeight'): tracking_params = "%s --ws %s" % (tracking_params, tracking['sizeWeight']) else: tracking_params = "%s --ws 0" % tracking_params - - if utils.getSafe(tracking,'maxSpeed'): + + if utils.getSafe(tracking, 'maxSpeed'): tracking_params = "%s --max-speed %f" % (tracking_params, tracking['maxSpeed']) - - if utils.getSafe(tracking,'maxTime'): + + if utils.getSafe(tracking, 'maxTime'): tracking_params = "%s --max-time %d" % (tracking_params, tracking['maxTime']) - if utils.getSafe(tracking,'maxSizeDeviation'): + if utils.getSafe(tracking, 'maxSizeDeviation'): tracking_params = "%s --max-size-deviation %d" % (tracking_params, tracking['maxSizeDeviation']) - if utils.getSafe(tracking,'useDisplacementVectors'): - tracking_params = "%s --use-displacement-vectors" % tracking_params + if utils.getSafe(tracking, 'useDisplacementVectors'): + tracking_params = "%s --use-displacement-vectors" % tracking_params - resume_at_index = 0; + resume_at_index = 0 # In case a scale T parameter is given, the output dir # is scale. Otherwise it's 'clustering' output_dir = config['output_dir'] - scale = utils.getValueForKeyPath(config,'scale') + scale = utils.getValueForKeyPath(config, 'scale') if scale: output_dir = output_dir + "/scale" + scale else: output_dir = output_dir + "/clustering" # --range ? - bandwidth = utils.getValueForKeyPath(config,'ranges') + bandwidth = utils.getValueForKeyPath(config, 'ranges') # Resume? - if not utils.getValueForKeyPath(config,'resume'): + if not utils.getValueForKeyPath(config, 'resume'): # consider time index. Even if not resuming, the # output directories should only be created at # the first time step if time_index <= 0: # TODO: consider scenario where user opted not to remove previous results? - print "Removing results from previous runs" + print("Removing results from previous runs") utils.create_ouput_directories(output_dir) else: - resume_at_index = utils.number_of_netcdf_files(output_dir+"/netcdf") + resume_at_index = utils.number_of_netcdf_files(output_dir + "/netcdf") if time_index >= 0 and time_index < resume_at_index: return - print "Resuming at index " + str(resume_at_index) + print("Resuming at index %s" % str(resume_at_index)) # Get a list of the files we need to process. - source = utils.getValueForKeyPath(config,'source_directory') + source = utils.getValueForKeyPath(config, 'source_directory') if os.path.isdir(source): # Multiple files in a directory netcdf_list = sorted(glob.glob(source + os.path.sep + '*.nc')) @@ -202,20 +203,20 @@ def run(config,time_index): # Decide if the tracking is run in an individual step # or inside of meanie3D-detect - inline_tracking = utils.getValueForKeyPath(config,'inline_tracking') + inline_tracking = utils.getValueForKeyPath(config, 'inline_tracking') for netcdf_file in netcdf_list: basename = os.path.basename(netcdf_file) - cluster_file = output_dir + os.path.sep + "netcdf" + os.path.sep + os.path.splitext(basename)[0]; + cluster_file = output_dir + os.path.sep + "netcdf" + os.path.sep + os.path.splitext(basename)[0] if time_index >= 0: - last_cluster_file = cluster_file + "-" + str(time_index-1) + "-clusters.nc" + last_cluster_file = cluster_file + "-" + str(time_index - 1) + "-clusters.nc" cluster_file = cluster_file + "-" + str(time_index) cluster_file += "-clusters.nc" - print "-------------------------------------------------------------------------------------" - print "Processing " + netcdf_file - print "-------------------------------------------------------------------------------------" + print("-------------------------------------------------------------------------------------") + print("Processing " + netcdf_file) + print("-------------------------------------------------------------------------------------") # ---------------------------------------------- # Clustering @@ -230,17 +231,17 @@ def run(config,time_index): run_count = run_count + 1 continue - print "-- Clustering --" + print("-- Clustering --") # build the clustering command params = detect_params # amend for time index and make proper logfile name if time_index < 0: - logfile = output_dir+"/log/clustering_" + str(run_count)+".log" + logfile = output_dir + "/log/clustering_" + str(run_count) + ".log" else: params = params + " --time-index " + str(time_index) - logfile = output_dir+"/log/clustering_" + str(time_index)+".log" + logfile = output_dir + "/log/clustering_" + str(time_index) + ".log" # scale? if scale and scale is not 'None': @@ -260,31 +261,35 @@ def run(config,time_index): if inline_tracking and have_previous: params += (" --inline-tracking %s" % tracking_params) - if utils.getValueForKeyPath(detection,"usePrevious") and have_previous: + if utils.getValueForKeyPath(detection, "usePrevious") and have_previous: params += " --postprocess-with-previous-output" # add ci-comparison-file if applicable - if run_count >= 3 and utils.getSafe(detection,'useCIScore'): - params += " --ci-comparison-file " + os.path.abspath(netcdf_list[run_count-3]) - proto_file = "protoclusters-" + os.path.splitext(os.path.basename(netcdf_list[run_count-3]))[0] + ".nc" + if run_count >= 3 and utils.getSafe(detection, 'useCIScore'): + params += " --ci-comparison-file " + os.path.abspath(netcdf_list[run_count - 3]) + proto_file = "protoclusters-" + os.path.splitext(os.path.basename(netcdf_list[run_count - 3]))[ + 0] + ".nc" params += " --ci-comparison-protocluster-file " + os.path.abspath(proto_file) # Input file - params += " --file %s --output %s" % (os.path.abspath(netcdf_file),os.path.abspath(cluster_file)) + params += " --file %s --output %s" % (os.path.abspath(netcdf_file), os.path.abspath(cluster_file)) # complete command with directing output to logfile - params = "%s > %s" % (params,os.path.abspath(logfile)) + params = "%s > %s" % (params, os.path.abspath(logfile)) # execute + start_time = 0 if config['time_operations']: start_time = time.time() - print "meanie3D-detect " + params - external.execute_command("meanie3D-detect",params,True) + print("meanie3D-detect " + params) + success = external.run("meanie3D-detect", params, return_output=True) + if not success: + print("ERROR: could not perform detection") + sys.exit(1) if config['time_operations']: - print "Finished. (%.2f seconds)" % (time.time()-start_time) - + print("Finished. (%.2f seconds)" % (time.time() - start_time)) # ---------------------------------------------- # Tracking @@ -295,24 +300,28 @@ def run(config,time_index): if (time_index <= 0 and (run_count > 0)) or (time_index > 0): if time_index < 0: - logfile = output_dir+"/log/tracking_" + str(run_count)+".log" + logfile = output_dir + "/log/tracking_" + str(run_count) + ".log" else: - logfile = output_dir+"/log/tracking_" + str(time_index)+".log" + logfile = output_dir + "/log/tracking_" + str(time_index) + ".log" - print "-- Tracking --" + print("-- Tracking --") params = tracking_params + " --previous %s --current %s " \ - % (os.path.abspath(last_cluster_file),os.path.abspath(cluster_file)) + % (os.path.abspath(last_cluster_file), os.path.abspath(cluster_file)) params = params + " > " + os.path.abspath(logfile) # execute + start_time = 0 if config['time_operations']: start_time = time.time() - print "meanie3D-track" + params - external.execute_command("meanie3D-track",params,True) + print("meanie3D-track" + params) + success = external.run("meanie3D-track", params, return_output=True) + if not success: + print("ERROR: could not perform detection") + sys.exit(1) if config['time_operations']: - print "Finished. (%.2f seconds)" % (time.time()-start_time) + print("Finished. (%.2f seconds)" % (time.time() - start_time)) # keep track last_cluster_file = cluster_file @@ -322,8 +331,8 @@ def run(config,time_index): # Clean out the trash if there is any if config['cleanup_vtk']: - print "Cleaning up *.vt*" + print("Cleaning up *.vt*") call("rm -f *.vt*", shell=True) - print "Done." - return \ No newline at end of file + print("Done.") + return diff --git a/python/meanie3D/app/utils.py b/python/meanie3D/app/utils.py index 8532f53..4024a4b 100644 --- a/python/meanie3D/app/utils.py +++ b/python/meanie3D/app/utils.py @@ -26,9 +26,9 @@ import json import os import os.path -import pdb import shutil import sys + from . import external @@ -38,7 +38,7 @@ def load_configuration(filename): :param filename: :return: """ - json_data=open(filename) + json_data = open(filename) data = json.load(json_data) json_data.close() return data @@ -51,18 +51,18 @@ def number_of_netcdf_files(source_dir): :return: """ netcdf_pattern = source_dir + "/*.nc" - netcdf_list=sorted(glob.glob(netcdf_pattern)) + netcdf_list = sorted(glob.glob(netcdf_pattern)) return len(netcdf_list) -def numbered_filename(filename,index): +def numbered_filename(filename, index): """ :param filename: :param index: :return: """ basename = os.path.basename(filename) - return os.path.splitext(basename)[0]+"-"+str(index)+".nc" + return os.path.splitext(basename)[0] + "-" + str(index) + ".nc" def create_ouput_directories(base_path): @@ -75,13 +75,13 @@ def create_ouput_directories(base_path): os.makedirs(base_path) # logs - log_dir = base_path+"/log" + log_dir = base_path + "/log" if os.path.exists(log_dir): shutil.rmtree(log_dir) os.makedirs(log_dir) # netcdf results - netcdf_dir = base_path+"/netcdf" + netcdf_dir = base_path + "/netcdf" if os.path.exists(netcdf_dir): shutil.rmtree(netcdf_dir) os.makedirs(netcdf_dir) @@ -97,198 +97,191 @@ def askYesNo(prompt): """ result = None msg = "%s [y|n]:" % prompt - while not result in ('y','Y','n','N'): + while result not in ('y', 'Y', 'n', 'N'): result = raw_input(msg) - return result in ('y','Y') + return result in ('y', 'Y') def removeOutputDirectories(config): """ :param config: - :param scales: :return: """ dirsExist = False output_dir = config['output_dir'] if config['scales']: for scale in config['scales']: - dir = output_dir + "/scale"+str(scale) - dirsExist = (dirsExist or os.path.exists(dir)) + _dir = output_dir + "/scale" + str(scale) + dirsExist = (dirsExist or os.path.exists(_dir)) else: - dir = output_dir + "/clustering" - dirsExist = os.path.exists(dir) + _dir = output_dir + "/clustering" + dirsExist = os.path.exists(_dir) if dirsExist: if askYesNo("Results exist from previous runs. They will be removed. Do you wish to proceed?"): if config['scales']: for scale in config['scales']: - dir = output_dir + "/scale"+str(scale) - external.execute_command("rm","-rf %s" % os.path.abspath(dir)) + _dir = output_dir + "/scale" + str(scale) + external.execute_command("rm", "-rf %s" % os.path.abspath(_dir)) else: - dir = output_dir + "/clustering" - external.execute_command("rm","-rf %s" % os.path.abspath(dir)) + _dir = output_dir + "/clustering" + external.execute_command("rm", "-rf %s" % os.path.abspath(_dir)) else: return False return True -def getSafe(dict,key): +def getSafe(dictionary, key): """ Tests if the dictionary has the given key. If so, it retrieves the value and returns it. If not, returns None. - :param dict: + :param dictionary: :param key: :return: """ - if key in dict and dict[key]: - return dict[key] + if key in dict and dictionary[key]: + return dictionary[key] else: return None def capitalize(line): """ - Courtesy of http://stackoverflow.com/questions/1549641/how-to-capitalize-the-first-letter-of-each-word-in-a-string-python + Courtesy of + http://stackoverflow.com/questions/1549641/how-to-capitalize-the-first-letter-of-each-word-in-a-string-python :param line: :return: """ return ' '.join(s[0].upper() + s[1:] for s in line.split(' ')) -def setValueForKeyPath(object,keypath,value): +def setValueForKeyPath(obj, keypath, value): """ Sets the given object values along a keypath separated by periods. Example: axes2D.yAxis.title.units. - :param object: + :param obj: :param keypath: :param value: :return: """ keys = keypath.split(".") if len(keys) > 1: - if type(object) is dict: - nextObject = object[keys[0]] + if type(obj) is dict: + nextObject = obj[keys[0]] else: - nextObject = getattr(object,keys[0]) + nextObject = getattr(obj, keys[0]) keys.pop(0) remainingPath = ".".join(keys) - setValueForKeyPath(nextObject,remainingPath,value) + setValueForKeyPath(nextObject, remainingPath, value) else: # The specific enumerated values in Visit are configured # as strings in configuration dictionary, but are int values # in visualisation objects. Try to set an enumerated value when hitting # this combination - if type(value) is str and type(getattr(object,keypath)) is int: - print "Attempting to resolve constant" - value = getattr(object,value) + if type(value) is str and type(getattr(obj, keypath)) is int: + print("Attempting to resolve constant") + value = getattr(obj, value) - if type(object) is dict: - object[keypath] = value + if type(obj) is dict: + obj[keypath] = value else: try: - setattr(object,keypath,value) + setattr(obj, keypath, value) except: try: # try a setter - setter = getattr(object,'Set'+capitalize(keypath)) + setter = getattr(obj, 'Set' + capitalize(keypath)) if setter: setter(value) except: - sys.stderr.write("Can't set value %s for key '%s' on object" % (str(value),keypath)) + sys.stderr.write("Can't set value %s for key '%s' on object" % (str(value), keypath)) raise return -def getValueForKeyPath(object,keypath): +def getValueForKeyPath(obj, keypath): """ Gets the given object values along a keypath separated by periods. Example: axes2D.yAxis.title.units - :param object: + :param obj: :param keypath: :return: """ keys = keypath.split(".") if len(keys) > 1: - if object is None: + if obj is None: return None - elif type(object) is dict: - nextObject = getSafe(object,keys[0]) + elif type(obj) is dict: + nextObject = getSafe(obj, keys[0]) else: - nextObject = getattr(object,keys[0]) + nextObject = getattr(obj, keys[0]) keys.pop(0) remainingPath = ".".join(keys) - return getValueForKeyPath(nextObject,remainingPath) + return getValueForKeyPath(nextObject, remainingPath) else: # The specific enumerated values in Visit are configured # as strings in configuration dictionary, but are int values # in visualisation objects. Try to set an enumerated value when hitting # this combination - value = None - if object is None: + if obj is None: return None - elif type(object) is dict: - value = getSafe(object,keypath) + elif type(obj) is dict: + value = getSafe(obj, keypath) else: - value = getattr(object,keypath) + value = getattr(obj, keypath) return value -def setValuesFromDictionary(object,dictionary): +def setValuesFromDictionary(obj, dictionary): """ Sets the values from the dictionary on the given object - :param object: + :param obj: :param dictionary: :return: """ - for key,value in dictionary.items(): - setValueForKeyPath(object,key,value) + for key, value in dictionary.items(): + setValueForKeyPath(obj, key, value) return -def find(path,filename,requiredComponent=None): +def find(path, filename, required_component=None): """ Recursive directory search :param path: :param filename: - :param requiredComponent: must have one component in the path to the result that matches this + :param required_component: must have one component in the path to the result that matches this :return: fully qualified path to result or None """ - try: - if os.path.exists(path) and os.path.isdir(path): - files = os.listdir(path) - for file_ in files: - if file_ == filename: - if requiredComponent: - components = path.split(os.path.sep) - if not requiredComponent in components: - continue - return os.path.abspath(path + os.sep + filename) - else: - return os.path.abspath(path + os.sep + filename) - - for f in files: - full_path = os.path.abspath(path + os.sep + f) - if os.path.isdir(full_path): - result = find(full_path, filename, requiredComponent) - if result: - return result - except OSError as err: - print err - + for root, dirs, files in os.walk(path, topdown=True): + all_files = [] + for foo in files: + all_files.append(foo) + for bar in dirs: + all_files.append(bar) + for _file in all_files: + if _file == filename: + complete_path = os.path.abspath(root + os.path.sep + _file) + if required_component: + components = complete_path.split(os.path.sep) + if required_component not in components: + continue + return complete_path + else: + return complete_path return None -def find_in_paths(paths,filename,requiredComponent=None): +def find_in_paths(paths, filename, required_component=None): """ Recursive directory search :rtype : string :param paths: :param filename: - :param requiredComponent: + :param required_component: :return: """ result = None for path in paths: - result = find(path,filename,requiredComponent) - if result != None: + result = find(path, filename, required_component) + if result is not None: break return result @@ -301,15 +294,15 @@ def findVisitPaths(): visitPath = None visitImportPath = None try: - cmdMap = external.locateCommands(["visit"],True) + cmdMap = external.locateCommands(["visit"], True) if cmdMap['visit']: visitPath = os.path.abspath(os.path.join(cmdMap['visit'], os.pardir + os.sep + os.pardir)) if visitPath: - visitImportPath = find(visitPath,"site-packages") + visitImportPath = find(visitPath, "site-packages") except: return None, None - return visitPath,visitImportPath + return visitPath, visitImportPath def strToBool(s): @@ -331,4 +324,3 @@ def strToBool(s): raise ValueError else: return False - diff --git a/python/meanie3D/meanie3D_run.py b/python/meanie3D/meanie3D_run.py index d583bb5..827eb88 100755 --- a/python/meanie3D/meanie3D_run.py +++ b/python/meanie3D/meanie3D_run.py @@ -1,6 +1,6 @@ #!/usr/bin/python -''' +""" The MIT License (MIT) (c) Juergen Simon 2014 (juergen.simon@uni-bonn.de) @@ -22,27 +22,29 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' +""" +import getopt import os import sys -import getopt import meanie3D import meanie3D.app import meanie3D.app.external +import meanie3D.app.postprocessing import meanie3D.app.tracking import meanie3D.app.utils -import meanie3D.app.postprocessing # Make sure the C++ executables are installed -meanie3D.app.external.locateCommands(["meanie3D-detect","meanie3D-track","meanie3D-trackstats","rm"]) +meanie3D.app.external.locateCommands(["meanie3D-detect", "meanie3D-track", "meanie3D-trackstats", "rm"]) + # ---------------------------------------------------------------------------- ## Prints usage and exits # def usage(): - print("meanie3D -c= -f= [-s=] [--start=t1 --end=t2] [--resume,-r] [--help,-h] [--version] [--time-operations]") + print( + "meanie3D -c= -f= [-s=] [--start=t1 --end=t2] [--resume,-r] [--help,-h] [--version] [--time-operations]") print("runs a complete set of netcdf files through the clustering/tracking") print("-c : json config file specifying variables etc.") print("-f : directory containing the files. It is assumed that") @@ -61,6 +63,8 @@ def usage(): print("--skip-trackstats: skip meanie3D-trackstats, even if the configuration is present") sys.exit(1) return + + # ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------- @@ -75,6 +79,8 @@ def print_configuration_format(): print(' "postprocessing" : {...} /* call --help config.postprocessing for help on this section */') print('}') sys.exit(1) + + # ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------- @@ -84,6 +90,8 @@ def print_version(): print("meanie3D: " + meanie3D.getVersion() + "\n") sys.exit(1) return + + # ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------- @@ -132,10 +140,10 @@ def main(): netcdf_dir = a num_params = num_params + 1 - elif o in ["--scales","-s"]: + elif o in ["--scales", "-s"]: scales = str(a).split(',') - elif o in ["--ranges","-r"]: + elif o in ["--ranges", "-r"]: ranges = a elif o == "-o": @@ -179,7 +187,7 @@ def main(): # sanity checks on time range - if start_time_index != -1 or end_time_index != -1 : + if start_time_index != -1 or end_time_index != -1: if (start_time_index == -1 and end_time_index != -1) or (start_time_index != -1 and end_time_index == -1): print("start-time-index and end-time-index must both be set") @@ -219,32 +227,34 @@ def main(): return # Run the detection and tracking steps - detection = meanie3D.app.utils.getSafe(configuration,'detection') - tracking = meanie3D.app.utils.getSafe(configuration,'tracking') + detection = meanie3D.app.utils.getSafe(configuration, 'detection') + tracking = meanie3D.app.utils.getSafe(configuration, 'tracking') if (detection or tracking): # run the actual clustering/tracking script if not scales: if uses_time == False: - meanie3D.app.tracking.run(configuration,-1) + meanie3D.app.tracking.run(configuration, -1) else: - for time_index in range(start_time_index,end_time_index): - meanie3D.app.tracking.run(configuration,time_index) + for time_index in range(start_time_index, end_time_index): + meanie3D.app.tracking.run(configuration, time_index) else: for scale in scales: configuration["scale"] = scale if uses_time == False: - meanie3D.app.tracking.run(configuration,-1) + meanie3D.app.tracking.run(configuration, -1) else: - for time_index in range(start_time_index,end_time_index): - meanie3D.app.tracking.run(configuration,time_index) + for time_index in range(start_time_index, end_time_index): + meanie3D.app.tracking.run(configuration, time_index) # Run the postprocessing steps if (configuration['postprocessing']): meanie3D.app.postprocessing.run(configuration) return + + # ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------- @@ -252,4 +262,4 @@ def main(): # ---------------------------------------------------------------------------- if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/python/meanie3D/resources/clusters_visit.py b/python/meanie3D/resources/clusters_visit.py index f5d6d6d..c65cf00 100644 --- a/python/meanie3D/resources/clusters_visit.py +++ b/python/meanie3D/resources/clusters_visit.py @@ -1,6 +1,6 @@ #!/usr/bin/python -''' +""" The MIT License (MIT) (c) Juergen Simon 2014 (juergen.simon@uni-bonn.de) @@ -22,7 +22,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' +""" __author__ = "Juergen Simon" __email__ = "juergen.simon@uni-bonn.de" @@ -34,19 +34,18 @@ # produced by meanie3D-trackstats --write-center-tracks-as-vtk # Parameters -_NETCDF_DIR = "P_NETCDF_DIR" -_CLUSTER_DIR = "P_CLUSTER_DIR" -_M3D_HOME = "P_M3D_HOME" -_RESUME = "P_RESUME" -_CONFIG_FILE = "P_CONFIGURATION_FILE" -_USES_TIME = P_USES_TIME -_START_TIME = "P_START_TIME_INDEX" -_END_TIME = "P_END_TIME_INDEX" +_NETCDF_DIR = "P_NETCDF_DIR" +_CLUSTER_DIR = "P_CLUSTER_DIR" +_M3D_HOME = "P_M3D_HOME" +_RESUME = "P_RESUME" +_CONFIG_FILE = "P_CONFIGURATION_FILE" +_USES_TIME = "P_USES_TIME" +_START_TIME = "P_START_TIME_INDEX" +_END_TIME = "P_END_TIME_INDEX" # Import modules import sys sys.path.append(_M3D_HOME) -import meanie3D.app.utils import meanie3D.visualisation.clusters # Parse configuration data @@ -65,4 +64,4 @@ # run it # pdb.set_trace() meanie3D.visualisation.clusters.run(configuration) -quit() \ No newline at end of file +quit() diff --git a/python/meanie3D/resources/tracks_visit.py b/python/meanie3D/resources/tracks_visit.py index 415343b..f465bfd 100644 --- a/python/meanie3D/resources/tracks_visit.py +++ b/python/meanie3D/resources/tracks_visit.py @@ -1,6 +1,6 @@ #!/usr/bin/python -''' +""" The MIT License (MIT) (c) Juergen Simon 2014 (juergen.simon@uni-bonn.de) @@ -22,7 +22,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' +""" __author__ = "Juergen Simon" __email__ = "juergen.simon@uni-bonn.de" @@ -34,18 +34,14 @@ # produced by meanie3D-trackstats --write-center-tracks-as-vtk # Parameters -_TRACKS_DIR = "P_TRACKS_DIR" -_M3D_HOME = "P_M3D_HOME" -_CONFIG_FILE = "P_CONFIGURATION_FILE" -_RESUME = "P_RESUME" +_TRACKS_DIR = "P_TRACKS_DIR" +_M3D_HOME = "P_M3D_HOME" +_CONFIG_FILE = "P_CONFIGURATION_FILE" +_RESUME = "P_RESUME" # Import modules -import os -import pdb import sys - sys.path.append(_M3D_HOME) -import meanie3D.app.utils import meanie3D.visualisation.tracks # Parse configuration data @@ -59,4 +55,4 @@ # run it # pdb.set_trace() meanie3D.visualisation.tracks.run(configuration) -quit() \ No newline at end of file +quit() diff --git a/python/meanie3D/visualisation/__init__.py b/python/meanie3D/visualisation/__init__.py index 9d67270..675067d 100644 --- a/python/meanie3D/visualisation/__init__.py +++ b/python/meanie3D/visualisation/__init__.py @@ -1,11 +1,9 @@ -__all__ = ["clusters", "tracks","utils"] -__have_visit = None -__visitPath__ = None -__visitImportPath__ = None - import sys import meanie3D.app.utils +__all__ = ["clusters", "tracks", "utils"] +__have_visit = None + # Import visit package __visitPath__, __visitImportPath__ = meanie3D.app.utils.findVisitPaths() if __visitImportPath__ and __visitPath__: diff --git a/python/meanie3D/visualisation/clusters.py b/python/meanie3D/visualisation/clusters.py index a3f3c87..f4d4b04 100644 --- a/python/meanie3D/visualisation/clusters.py +++ b/python/meanie3D/visualisation/clusters.py @@ -1,4 +1,4 @@ -''' +""" The MIT License (MIT) (c) Juergen Simon 2014 (juergen.simon@uni-bonn.de) @@ -20,113 +20,112 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' +""" import glob import os import subprocess import sys import time + import visit # Own packages import meanie3D.visualisation import utils - from meanie3D.app import external + __have_vtk__ = external.hasCommand('meanie3D-cfm2vtk') -# ------------------------------------------------------------------------------ -# Adds clusters with names "*_infix_*.vtk" to the current visualisation window. -# The clusters are colored according to the cluster_colors colortable, -# which is created at the beginning of the visualisation code. -# -# \param infix (e.g. "_untracked_clusters_") -# \param color_table_name Name of the cluster color table -# \param color_count Number of colors in the cluster color table -# \param configuration Options to overwrite the options the clusters are -# plotted with. Clusters are added as pseudocolor plots. For information -# on the available options, check PseudocolorAttributes() -# -# ------------------------------------------------------------------------------ -def add_clusters(cluster_vtk_file,configuration): + +def add_clusters(cluster_vtk_file, configuration): + """ + Adds clusters with names "*_infix_*.vtk" to the current visualisation window. + The clusters are colored according to the cluster_colors colortable, + which is created at the beginning of the visualisation code. + + :param cluster_vtk_file: + :param configuration: Options to overwrite the options the clusters are + plotted with. Clusters are added as pseudocolor plots. For information + on the available options, check PseudocolorAttributes() + :return: + """ # Get the configuration - clusterOptions = utils.getValueForKeyPath(configuration,'postprocessing.clusters.visit.cluster') + clusterOptions = utils.getValueForKeyPath(configuration, 'postprocessing.clusters.visit.cluster') if not clusterOptions: sys.stderr.write("ERROR:no value found for postprocessing.clusters.visit.cluster") return # plot the clusters - print "Adding clusters from file %s" % cluster_vtk_file + print("Adding clusters from file %s" % cluster_vtk_file) utils.addPseudocolorPlot(cluster_vtk_file, clusterOptions) return def run(conf): - ''' + """ Visualises clusters of a given run. :param conf: :return: - ''' - #pp = pprint.PrettyPrinter() - #pp.pprint(conf) + """ + # pp = pprint.PrettyPrinter() + # pp.pprint(conf) # Make sure the global configuration is in place utils.run_global_visit_configuration(conf) - #pdb.set_trace() + # pdb.set_trace() - clusterConf = utils.getValueForKeyPath(conf,'postprocessing.clusters') + clusterConf = utils.getValueForKeyPath(conf, 'postprocessing.clusters') if not clusterConf: - print "No configuration for cluster postprocessing. Nothing to do." + print("No configuration for cluster postprocessing. Nothing to do.") return 0 - visitConf = utils.getValueForKeyPath(clusterConf,'visit') + visitConf = utils.getValueForKeyPath(clusterConf, 'visit') if not visitConf: - print "No configuration for visuals. Nothing to do." + print("No configuration for visuals. Nothing to do.") return 0 - views = utils.getValueForKeyPath(visitConf,'views') + views = utils.getValueForKeyPath(visitConf, 'views') # Set up background gradient, axis labels etc. - utils.setAnnotations(visitConf,'annotationAttributes') + utils.setAnnotations(visitConf, 'annotationAttributes') # Set view and annotation attributes - utils.setAnnotations(conf,'postprocessing.clusters.visit.annotations') + utils.setAnnotations(conf, 'postprocessing.clusters.visit.annotations') - if not utils.getValueForKeyPath(conf,'resume'): - print "Removing results from previous runs" + if not utils.getValueForKeyPath(conf, 'resume'): + print("Removing results from previous runs") subprocess.call("rm -rf images movies *.vtk *.vtr *tracking*.png *source*.png", shell=True) else: - print "Removing intermediary files from previous runs" + print("Removing intermediary files from previous runs") subprocess.call("rm -f *.vtk *.vtr *.vtu", shell=True) # Figure out the ending for the cluster vtk files - conversion_config = utils.getValueForKeyPath(conf,'postprocessing.clusters.meanie3D-cfm2vtk') + conversion_config = utils.getValueForKeyPath(conf, 'postprocessing.clusters.meanie3D-cfm2vtk') if "--write-as-xml" in conversion_config: cluster_vtk_extension = ".vtu" else: cluster_vtk_extension = ".vtk" # Glob the netcdf directory or find the single file - uses_time = utils.getValueForKeyPath(conf,'uses_time') + uses_time = utils.getValueForKeyPath(conf, 'uses_time') - print "Current work directory: " + os.path.abspath(os.getcwd()) + print("Current work directory: " + os.path.abspath(os.getcwd())) if uses_time: - print "Processing file " + conf['source_directory'] + print("Processing file " + conf['source_directory']) netcdf_file = conf['source_directory'] else: - print "Processing files in directory " + conf['source_directory'] - netcdf_files = sorted(glob.glob(conf['source_directory']+"/*.nc")) + print("Processing files in directory " + conf['source_directory']) + netcdf_files = sorted(glob.glob(conf['source_directory'] + "/*.nc")) # Keep track of number of images to allow # forced re-set in time to circumvent the # Visit memory leak image_count = 0 - index_range = [] if uses_time: - t1 = int(utils.getValueForKeyPath(conf,'start_time_index')) - t2 = int(utils.getValueForKeyPath(conf,'end_time_index')) - index_range = range(t1,t2+1) + t1 = int(utils.getValueForKeyPath(conf, 'start_time_index')) + t2 = int(utils.getValueForKeyPath(conf, 'end_time_index')) + index_range = range(t1, t2 + 1) else: index_range = range(len(netcdf_files)) time_index = -1 @@ -140,7 +139,7 @@ def run(conf): else: time_index = index - netcdf_path,filename = os.path.split(netcdf_file) + netcdf_path, filename = os.path.split(netcdf_file) basename = os.path.splitext(filename)[0] if uses_time: basename = basename + "-" + str(time_index) @@ -150,151 +149,147 @@ def run(conf): cluster_vtk_file = os.path.splitext(filename)[0] + "-clusters" + cluster_vtk_extension # label and displacement files are based on the cluster file name - label_vtk_file = basename + "-clusters-centers.vtk" + label_vtk_file = basename + "-clusters-centers.vtk" displacement_vtk_file = basename + "-clusters-displacements.vtk" - print "netcdf_file = " + netcdf_file - print "cluster_file = " + cluster_file + print("netcdf_file = " + netcdf_file) + print("cluster_file = " + cluster_file) # check if the files both exist if not os.path.exists(cluster_file): - print "Cluster file does not exist. Skipping." + print("Cluster file does not exist. Skipping.") continue # predict the filenames for checking on resume - number_postfix = str(image_count).rjust(4,'0') + number_postfix = str(image_count).rjust(4, '0') source_open = False skip_source = False - if conf['resume'] == True: - exists = utils.images_exist(visitConf['views'],"source",image_count) + if conf['resume']: + exists = utils.images_exist(visitConf['views'], "source", image_count) if exists == "all": - print "Source visualization " + number_postfix + " exists. Skipping." + print("Source visualization " + number_postfix + " exists. Skipping.") skip_source = True elif exists == "partial": - print "Deleting partial visualization " + number_postfix - utils.delete_images(conf,"source",image_count) - - if skip_source == False: + print("Deleting partial visualization " + number_postfix) + utils.delete_images(conf, "source", image_count) - if utils.getValueForKeyPath(clusterConf,'createSourceMovie'): + if not skip_source and utils.getValueForKeyPath(clusterConf, 'createSourceMovie'): - # Add ancillary background data - utils.plotMapdata(visitConf,'map') + # Add ancillary background data + utils.plotMapdata(visitConf, 'map') - # Add timestamp - if utils.getValueForKeyPath(clusterConf,'showDateTime'): - utils.add_datetime(clusterConf,netcdf_file,time_index) + # Add timestamp + if utils.getValueForKeyPath(clusterConf, 'showDateTime'): + utils.add_datetime(clusterConf, netcdf_file, time_index) - # Add source data and threshold it - print "Plotting source data ..." - start_time = time.time() + # Add source data and threshold it + print("Plotting source data ...") + start_time = time.time() + utils.addPseudocolorPlots(netcdf_file, visitConf, 'source.plots', time_index) + source_open = True + visit.DrawPlots() - utils.addPseudocolorPlots(netcdf_file,visitConf,'source.plots',time_index) - source_open = True - visit.DrawPlots() + utils.saveImagesForViews(views, "source") - utils.saveImagesForViews(views,"source") + visit.DeleteAllPlots() + visit.ClearWindow() - visit.DeleteAllPlots() - visit.ClearWindow() - - print " done. (%.2f seconds)" % (time.time()-start_time) - - if utils.getValueForKeyPath(clusterConf,'visualiseClusters') and __have_vtk__: + print(" done. (%.2f seconds)" % (time.time() - start_time)) + if utils.getValueForKeyPath(clusterConf, 'visualiseClusters') and __have_vtk__: skip = False - if conf['resume'] == True: + if conf['resume']: - exists = utils.images_exist(visitConf['views'],"tracking",image_count) + exists = utils.images_exist(visitConf['views'], "tracking", image_count) if exists == "all": - print "Cluster visualization "+number_postfix+" exists. Skipping." + print("Cluster visualization " + number_postfix + " exists. Skipping.") skip = True elif exists == "partial": - print "Deleting partial cluster visualization " + number_postfix - utils.delete_images(conf,"tracking",image_count) + print("Deleting partial cluster visualization " + number_postfix) + utils.delete_images(conf, "tracking", image_count) - if skip == False: + if not skip: # Run the conversion - print "-- Converting clusters to .vtr --" + print("-- Converting clusters to .vtr --") start_time = time.time() params = "-f %s %s" \ - % (cluster_file,utils.getValueForKeyPath(conf,'postprocessing.clusters.meanie3D-cfm2vtk')) - if utils.getValueForKeyPath(conf,'postprocessing.clusters.showDisplacementVectors'): + % (cluster_file, utils.getValueForKeyPath(conf, 'postprocessing.clusters.meanie3D-cfm2vtk')) + if utils.getValueForKeyPath(conf, 'postprocessing.clusters.showDisplacementVectors'): params += " --write-displacement-vectors" - if utils.getValueForKeyPath(conf,'data.vtkDimensions'): - vtkDimString = ",".join(utils.getValueForKeyPath(conf,'data.vtkDimensions')) + if utils.getValueForKeyPath(conf, 'data.vtkDimensions'): + vtkDimString = ",".join(utils.getValueForKeyPath(conf, 'data.vtkDimensions')) params += " --vtk-dimensions=%s" % vtkDimString # pdb.set_trace(); - print "meanie3D-cfm2vtk %s" % params + print("meanie3D-cfm2vtk %s" % params) meanie3D.app.external.execute_command('meanie3D-cfm2vtk', params) - print " done. (%.2f seconds)" % (time.time()-start_time) + print(" done. (%.2f seconds)" % (time.time() - start_time)) # Move cluster output file to individual file if uses_time: cluster_vtk_file_dst = basename + "-clusters.vtk" - os.rename(cluster_vtk_file,cluster_vtk_file_dst) + os.rename(cluster_vtk_file, cluster_vtk_file_dst) cluster_vtk_file = cluster_vtk_file_dst - print "-- Rendering cluster scene --" + print("-- Rendering cluster scene --") start_time = time.time() # Add ancillary background data - utils.plotMapdata(visitConf,'map') + utils.plotMapdata(visitConf, 'map') # Add timestamp - if utils.getValueForKeyPath(clusterConf,'showDateTime'): - utils.add_datetime(clusterConf,netcdf_file,time_index) + if utils.getValueForKeyPath(clusterConf, 'showDateTime'): + utils.add_datetime(clusterConf, netcdf_file, time_index) # Add background source data - if utils.getValueForKeyPath(clusterConf,'showSourceBackground'): - utils.addPseudocolorPlots(netcdf_file,visitConf,'sourceBackground.plots',time_index) + if utils.getValueForKeyPath(clusterConf, 'showSourceBackground'): + utils.addPseudocolorPlots(netcdf_file, visitConf, 'sourceBackground.plots', time_index) source_open = True # Add the clusters add_clusters(cluster_vtk_file, conf) # Add modes as labels - labelConf = utils.getValueForKeyPath(visitConf,'label') + labelConf = utils.getValueForKeyPath(visitConf, 'label') if labelConf: - utils.addLabelPlot(label_vtk_file,labelConf) + utils.addLabelPlot(label_vtk_file, labelConf) # Add displacement vectors - if utils.getValueForKeyPath(clusterConf,'showDisplacementVectors'): - vectorConf = utils.getValueForKeyPath(visitConf,'displacementVectors') + if utils.getValueForKeyPath(clusterConf, 'showDisplacementVectors'): + vectorConf = utils.getValueForKeyPath(visitConf, 'displacementVectors') if vectorConf: - utils.addVectorPlot(displacement_vtk_file,vectorConf) + utils.addVectorPlot(displacement_vtk_file, vectorConf) visit.DrawPlots() - utils.saveImagesForViews(views,"tracking") + utils.saveImagesForViews(views, "tracking") - print " done. (%.2f seconds)" % (time.time()-start_time) + print(" done. (%.2f seconds)" % (time.time() - start_time)) # clean up - visit.DeleteAllPlots(); + visit.DeleteAllPlots() visit.ClearWindow() if source_open: visit.CloseDatabase(netcdf_file) visit.CloseDatabase(label_vtk_file) - utils.close_pattern(basename+"*.vtr") - utils.close_pattern(basename+"*.vtk") - utils.close_pattern(basename+"*.vtu") + utils.close_pattern(basename + "*.vtr") + utils.close_pattern(basename + "*.vtk") + utils.close_pattern(basename + "*.vtu") - if utils.getValueForKeyPath(conf,'cleanup_vtk'): + if utils.getValueForKeyPath(conf, 'cleanup_vtk'): subprocess.call("rm -f *.vt*", shell=True) # periodically kill computing engine to # work around the memory leak fix - image_count=image_count+1; + image_count = image_count + 1 # TODO: check if this is still necessary and re-implement if it is. - #if image_count % 100 == 0: + # if image_count % 100 == 0: # visit.CloseComputeEngine() # close mapstuff @@ -303,21 +298,20 @@ def run(conf): # Use imagemagick to use image sequences to make movies - movieFormats = utils.getValueForKeyPath(clusterConf,'movieFormats') - if utils.getValueForKeyPath(clusterConf,'createSourceMovie'): - utils.createMoviesForViews(views,"source", movieFormats) + movieFormats = utils.getValueForKeyPath(clusterConf, 'movieFormats') + if utils.getValueForKeyPath(clusterConf, 'createSourceMovie'): + utils.createMoviesForViews(views, "source", movieFormats) - if utils.getValueForKeyPath(clusterConf,'createClusterMovie'): - utils.createMoviesForViews(views,"tracking", movieFormats) + if utils.getValueForKeyPath(clusterConf, 'createClusterMovie'): + utils.createMoviesForViews(views, "tracking", movieFormats) # clean up - print "Cleaning up ..." + print("Cleaning up ...") subprocess.call("mkdir images", shell=True) subprocess.call("mv *tracking_*.png images", shell=True) subprocess.call("mv *source_*.png images", shell=True) subprocess.call("mkdir movies", shell=True) subprocess.call("mv *source*.gif *tracking*.gif *.m4v movies", shell=True) - if utils.getValueForKeyPath(conf,'cleanup_vtk'): + if utils.getValueForKeyPath(conf, 'cleanup_vtk'): subprocess.call("rm -f *.vt* visitlog.py", shell=True) return - diff --git a/python/meanie3D/visualisation/tracks.py b/python/meanie3D/visualisation/tracks.py index 054e9bd..b238454 100644 --- a/python/meanie3D/visualisation/tracks.py +++ b/python/meanie3D/visualisation/tracks.py @@ -1,4 +1,4 @@ -''' +""" The MIT License (MIT) (c) Juergen Simon 2014 (juergen.simon@uni-bonn.de) @@ -20,44 +20,45 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' +""" import glob import os -import pprint +# import pprint + import visit + import utils -## -# Plots a list of tracks from .vtk files produced by -# meanie3D-trackstats --write-center-tracks-as-vtk. -# -# \param:conf Configuration dictionary -# -def run(conf): - pp = pprint.PrettyPrinter() +def run(conf): + """ + Plots a list of tracks from .vtk files produced by meanie3D-trackstats --write-center-tracks-as-vtk. + :param conf: Configuration dictionary + :return: + """ + # pp = pprint.PrettyPrinter() # pp.pprint(conf) # Make sure the global configuration is in place utils.run_global_visit_configuration(conf) - visitConf = utils.getValueForKeyPath(conf,'postprocessing.tracks.visit') + visitConf = utils.getValueForKeyPath(conf, 'postprocessing.tracks.visit') if not visitConf: - print "No configuration for visuals. Nothing to do." + print("No configuration for visuals. Nothing to do.") return 0 # Set up background gradient, axis labels etc. - utils.setAnnotations(conf,'postprocessing.tracks.visit.annotationAttributes') + utils.setAnnotations(conf, 'postprocessing.tracks.visit.annotationAttributes') # Set the view straight - utils.setView(conf,'postprocessing.tracks.visit.view') + utils.setView(conf, 'postprocessing.tracks.visit.view') # Plot the map data - utils.plotMapdata(conf,'postprocessing.tracks.visit.map') + utils.plotMapdata(conf, 'postprocessing.tracks.visit.map') # Plot the tracks - trackPlotConf = utils.getValueForKeyPath(conf,'postprocessing.tracks.visit.track') + trackPlotConf = utils.getValueForKeyPath(conf, 'postprocessing.tracks.visit.track') # pp.pprint(trackPlotConf) currentDirectory = os.path.abspath(os.getcwd()) @@ -71,12 +72,12 @@ def run(conf): # Plot the Tracks # track_pattern = conf['tracks_dir'] + "/*-track_*.vtk" - track_pattern = "*-track_*.vtk" - list = sorted(glob.glob(track_pattern)) - print "Looking with pattern " + track_pattern - print "Found %d track files." % len(list) - count = 0; - for trackFile in list: + track_pattern = "*-track_*.vtk" + _list = sorted(glob.glob(track_pattern)) + print("Looking with pattern " + track_pattern) + print("Found %d track files." % len(_list)) + count = 0 + for trackFile in _list: # add plot # trackFile = conf['tracks_dir'] + os.path.sep + fname @@ -87,9 +88,9 @@ def run(conf): # pp.pprint(trackPlotConf) # Plot the actual track data - file = conf['tracks_dir'] + os.path.sep + trackFile - print "Adding plot for " + file - utils.addPseudocolorPlot(file,trackPlotConf) + _file = conf['tracks_dir'] + os.path.sep + trackFile + print("Adding plot for " + _file) + utils.addPseudocolorPlot(_file, trackPlotConf) count = count + 1 @@ -101,11 +102,11 @@ def run(conf): trackPlotConf['PseudocolorAttributes']['legendFlag'] = legendFlag # pp.pprint(trackPlotConf) - print "Drawing plots" + print("Drawing plots") visit.DrawPlots() - print "Saving image to %s" % os.getcwd() - utils.saveImage("tracks",0) + print("Saving image to %s" % os.getcwd()) + utils.saveImage("tracks", 0) os.chdir(currentDirectory) diff --git a/python/meanie3D/visualisation/utils.py b/python/meanie3D/visualisation/utils.py index f743df2..ab6d40a 100644 --- a/python/meanie3D/visualisation/utils.py +++ b/python/meanie3D/visualisation/utils.py @@ -1,4 +1,4 @@ -''' +""" The MIT License (MIT) (c) Juergen Simon 2014 (juergen.simon@uni-bonn.de) @@ -20,50 +20,47 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -''' +""" import glob -import sys import os import os.path -import meanie3D.app.utils -import meanie3D.app.external +import sys from subprocess import call +from meanie3D.app import external, utils + # make sure external commands are available -meanie3D.app.external.locateCommands(['convert', 'composite','python']) +external.locateCommands(['convert', 'composite', 'python']) -ret_code, paths_string = meanie3D.app.external.execute_command('python','-c "import sys; print sys.path"', True) +ret_code, paths_string = external.run('python', '-c "import sys; print sys.path"', return_output=True) if ret_code == 0: - print "Attempting to locate python module netCDF4" - paths = paths_string.split(",") - paths.append('/usr/lib') - paths.append('/usr/local/lib') - result = meanie3D.app.utils.find_in_paths(paths,"netCDF4","site-packages") + print ("Attempting to locate python module netCDF4") + home = os.path.expanduser('~') + searchPaths = ['/usr/lib', '/usr/local/lib', home + os.path.sep + '.pyenv/versions'] + print("Looking in paths:") + print(searchPaths) + result = utils.find_in_paths(searchPaths, "netCDF4", "site-packages") if not result: - result = meanie3D.app.utils.find_in_paths(paths,"netCDF4","dist-packages") + result = utils.find_in_paths(searchPaths, "netCDF4", "dist-packages") if not result: - print "Failed to locate python module netCDF4" + print("Failed to locate python module netCDF4") exit(-1) else: - print "Found netCDF4 at %s" % result - sys.path.append(os.path.split(result)[0]); - print "Python path after adding system search directories:" + print("Found netCDF4 at %s" % result) + sys.path.append(os.path.split(result)[0]) + print("Python path after adding system search directories:") print(sys.path) - else: - print "Failed to obtain system's python path" + print("Failed to obtain system's python path") exit(-1) +# Now that the path is extended, import the rest import netCDF4 import visit - - # --------------------------------------------------------- -# # Global configuration handling -# # --------------------------------------------------------- # Global variable to help making sure the global visit configuration @@ -72,12 +69,12 @@ def run_global_visit_configuration(configuration): - ''' + """ Creates any configuration items regarded as global, such as creating color tables and named views. :param configuration: :return: - ''' + """ # Only run this once global did_global_config_execute if did_global_config_execute: @@ -87,7 +84,7 @@ def run_global_visit_configuration(configuration): if getValueForKeyPath(configuration, 'postprocessing.visit.colorTables'): createColorTables(configuration, 'postprocessing.visit.colorTables') - saveWindowAttributes = getValueForKeyPath(configuration, 'postprocessing.visit.saveWindowAttributes'); + saveWindowAttributes = getValueForKeyPath(configuration, 'postprocessing.visit.saveWindowAttributes') if saveWindowAttributes: s = visit.GetSaveWindowAttributes() updateVisitObjectFromDictionary(s, saveWindowAttributes) @@ -107,10 +104,10 @@ def run_global_visit_configuration(configuration): # # --------------------------------------------------------- -def addPseudocolorPlot(databaseFile, configuration, time_index=-1): - ''' +def addPseudocolorPlot(db_file, configuration, time_index=-1): + """ Adds a pseudocolor plot - :param databaseFile: + :param db_file: :param configuration: { "variable":"X", @@ -123,13 +120,13 @@ def addPseudocolorPlot(databaseFile, configuration, time_index=-1): } :param time_index: :return: - ''' + """ variable = getValueForKeyPath(configuration, 'variable') attributes = getValueForKeyPath(configuration, 'PseudocolorAttributes') if variable and attributes: # Open data file - visit.OpenDatabase(databaseFile) + visit.OpenDatabase(db_file) # Add the plot visit.AddPlot("Pseudocolor", variable) @@ -152,10 +149,10 @@ def addPseudocolorPlot(databaseFile, configuration, time_index=-1): return -def addContourPlot(databaseFile, configuration): - ''' +def addContourPlot(db_file, configuration): + """ Adds a contour plot. - :param databaseFile: + :param db_file: :param configuration: { "variable":"X", @@ -167,12 +164,12 @@ def addContourPlot(databaseFile, configuration): } } :return: - ''' + """ variable = getValueForKeyPath(configuration, 'variable') attributes = getValueForKeyPath(configuration, 'ContourAttributes') if variable and attributes: # Open data file - visit.OpenDatabase(databaseFile) + visit.OpenDatabase(db_file) # Add the plot visit.AddPlot("Contour", variable) p = visit.ContourAttributes() @@ -188,10 +185,10 @@ def addContourPlot(databaseFile, configuration): return -def addVectorPlot(databaseFile, configuration): - ''' +def addVectorPlot(db_file, configuration): + """ Adds a vector plot. - :param databaseFile: + :param db_file: :param configuration: { "variable":"X", @@ -200,11 +197,11 @@ def addVectorPlot(databaseFile, configuration): } } :return: - ''' + """ variable = getValueForKeyPath(configuration, 'variable') attributes = getValueForKeyPath(configuration, 'VectorAttributes') if variable and attributes: - visit.OpenDatabase(databaseFile) + visit.OpenDatabase(db_file) visit.AddPlot("Vector", variable) p = visit.VectorAttributes() updateVisitObjectFromDictionary(p, attributes) @@ -212,10 +209,10 @@ def addVectorPlot(databaseFile, configuration): return -def addLabelPlot(file, configuration): - ''' +def addLabelPlot(db_file, configuration): + """ Adds a label plot. - :param file: + :param db_file: :param configuration: { "variable":"X", @@ -224,11 +221,11 @@ def addLabelPlot(file, configuration): } } :return: - ''' + """ variable = getValueForKeyPath(configuration, 'variable') attributes = getValueForKeyPath(configuration, 'LabelAttributes') if variable and attributes: - visit.OpenDatabase(file) + visit.OpenDatabase(db_file) visit.AddPlot("Label", variable) a = visit.LabelAttributes() updateVisitObjectFromDictionary(a, attributes) @@ -238,81 +235,81 @@ def addLabelPlot(file, configuration): def addTextAnnotation(x, y, message): - ''' + """ Add a 2D text annotation of height 2% :param x: :param y: :param message: :return: - ''' + """ try: textAnnotation = visit.GetAnnotationObject("Text2D1") except visit.VisItException: textAnnotation = visit.CreateAnnotationObject("Text2D") if textAnnotation: - textAnnotation.text = message; + textAnnotation.text = message textAnnotation.position = (x, y) textAnnotation.height = 0.02 return -def setViewFromDict(viewConfig): - ''' +def setViewFromDict(view_config): + """ Sets the view up with the given perspective object from the configuration. - :param viewConfig: you can use all keys found in GetView2D() or GetView3D() + :param view_config: you can use all keys found in GetView2D() or GetView3D() :return: - ''' - if viewConfig: - if 'windowCoords' in viewConfig or 'viewportCoords' in viewConfig: + """ + if view_config: + if 'windowCoords' in view_config or 'viewportCoords' in view_config: view = visit.GetView2D() - updateVisitObjectFromDictionary(view, viewConfig) + updateVisitObjectFromDictionary(view, view_config) visit.SetView2D(view) else: view = visit.GetView3D() - updateVisitObjectFromDictionary(view, viewConfig) + updateVisitObjectFromDictionary(view, view_config) visit.SetView3D(view) return -def setView(configuration, path): - ''' +def setView(configuration, key_path): + """ Sets the view up with the given perspective object from the configuration. :param configuration: you can use all keys found in GetView2D() or GetView3D() - :param path: + :param key_path: :return: - ''' - viewConfig = meanie3D.app.utils.getValueForKeyPath(configuration, path) + """ + viewConfig = utils.getValueForKeyPath(configuration, key_path) setViewFromDict(viewConfig) -def addPseudocolorPlots(databaseFile, configuration, path, time_index = -1): - ''' +def addPseudocolorPlots(db_file, configuration, key_path, time_index=-1): + """ Plots mapdata according to the configuration given. Note that $ variables will be replaced with the value found in the environment. - :param databaseFile: + :param db_file: :param configuration: - :param path: + :param key_path: :param time_index: :return: - ''' - plots = getValueForKeyPath(configuration, path) + """ + plots = getValueForKeyPath(configuration, key_path) if plots: - visit.OpenDatabase(databaseFile) + visit.OpenDatabase(db_file) for plot in plots: - addPseudocolorPlot(databaseFile, plot, time_index) + addPseudocolorPlot(db_file, plot, time_index) return -def setAnnotations(configuration, path): - ''' +def setAnnotations(configuration, key_path): + """ Set annotation attributes from defaults and configuration. :param configuration: see visit.GetAnnotationAttributes() - :param path: + :param key_path: :return: - ''' - ca = meanie3D.app.utils.getValueForKeyPath(configuration, path) + """ + ca = utils.getValueForKeyPath(configuration, key_path) if ca: a = visit.GetAnnotationAttributes() updateVisitObjectFromDictionary(a, ca) @@ -320,14 +317,14 @@ def setAnnotations(configuration, path): return -def createColorTables(configuration, path): - ''' +def createColorTables(configuration, key_path): + """ Creates a list of colortables from the configuration. :param configuration: - :param path: + :param key_path: :return: - ''' - colorTables = getValueForKeyPath(configuration, path) + """ + colorTables = getValueForKeyPath(configuration, key_path) if colorTables: for colorTable in colorTables: colors = colorTable['colors'] @@ -344,36 +341,36 @@ def createColorTables(configuration, path): def close_pattern(pattern): - ''' + """ Closes the databases of all files matching the given pattern. :param pattern: :return: - ''' - list = glob.glob(pattern) - for file in list: - visit.CloseDatabase(file) + """ + databases = glob.glob(pattern) + for database in databases: + visit.CloseDatabase(database) return -def plotMapdata(configuration, path): - ''' +def plotMapdata(configuration, key_path): + """ Plots mapdata according to the configuration given. Note that $ variables will be replaced with the value found in the environment. :param configuration: - :param path: + :param key_path: :return: - ''' - mapConfig = getValueForKeyPath(configuration, path) + """ + mapConfig = getValueForKeyPath(configuration, key_path) if mapConfig: # Find the map data file mapFile = os.path.expandvars(mapConfig['mapDataFile']) # Check if data file exists if os.path.exists(mapFile): - addPseudocolorPlots(mapFile, configuration, path + ".plots") + addPseudocolorPlots(mapFile, configuration, key_path + ".plots") else: - print "ERROR:could not find map file at " + mapFile + print("ERROR:could not find map file at " + mapFile) return @@ -384,21 +381,20 @@ def plotMapdata(configuration, path): # --------------------------------------------------------- def path(filename): - ''' + """ Extracts the path part from the given filename. :param filename: :return: - ''' - path = os.path.dirname(filename) - return path + """ + return os.path.dirname(filename) def naked_name(filename): - ''' + """ Extracts the filename WITHOUT extension from the the given filename. :param filename: :return: - ''' + """ stripped = os.path.splitext(filename)[0] return stripped @@ -410,7 +406,7 @@ def naked_name(filename): # --------------------------------------------------------- def saveImagesForViews(views, basename): - ''' + """ Iterates over the given view objects, sets each in turn and saves an image. If there is only one view, the resulting image filenames are 'basename_.png'. If there are more than @@ -418,7 +414,7 @@ def saveImagesForViews(views, basename): :param views: :param basename: :return: - ''' + """ if views: for i in range(0, len(views)): view = views[i] @@ -434,12 +430,12 @@ def saveImagesForViews(views, basename): def saveImage(basename, progressive): - ''' + """ Saves the current window to an image file. :param basename: :param progressive: :return: - ''' + """ s = visit.GetSaveWindowAttributes() s.progressive = progressive s.fileName = basename + "_" @@ -450,16 +446,16 @@ def saveImage(basename, progressive): return -def createMovieForViews(views, basename, format): - ''' +def createMovieForViews(views, basename, extension): + """ Creates movies for the formats given from the images based on the basename given. Depending on the views, the filenames used start with 'p_basename_' or :param views: :param basename: - :param format: + :param extension: :return: - ''' + """ if views: for i in range(0, len(views)): if len(views) > 1: @@ -467,62 +463,60 @@ def createMovieForViews(views, basename, format): else: movie_fn = basename image_fn = movie_fn + "_" - create_movie(image_fn, movie_fn + os.path.extsep + format) + create_movie(image_fn, movie_fn + os.path.extsep + extension) else: image_fn = basename + "_" - create_movie(image_fn, basename + os.path.extsep + format) + create_movie(image_fn, basename + os.path.extsep + extension) def createMoviesForViews(views, basename, formats): - ''' + """ :param views: :param basename: :param formats: ("gif","m4v" etc.) :return: - ''' + """ if formats: for k in range(0, len(formats)): createMovieForViews(views, basename, formats[k]) def delete_images(views, basename, image_count): - ''' + """ Checks if the image file(s) with the given number exists and deletes them. :param views: :param basename: :param image_count: :return: - ''' - number_postfix = str(image_count).rjust(4, '0') + ".png"; - result = False + """ + number_postfix = str(image_count).rjust(4, '0') + ".png" if len(views) > 1: for i in range(0, len(views)): fn = "p" + str(i) + "_" + basename + "_" + number_postfix - if (os.path.exists(fn)): + if os.path.exists(fn): os.remove(fn) else: fn = basename + "_" + number_postfix - if (os.path.exists(fn)): + if os.path.exists(fn): os.remove(fn) def images_exist(views, basename, image_count): - ''' + """ Checks if the image with the given number exists. Takes perspectives into account. :param views: :param basename: :param image_count: :return: "all","none" or "partial" - ''' - number_postfix = str(image_count).rjust(4, '0') + ".png"; + """ + number_postfix = str(image_count).rjust(4, '0') + ".png" num_found = 0 for i in range(0, len(views)): - fn = None if len(views) > 1: fn = "p" + str(i) + "_" + basename + "_" + number_postfix else: fn = basename + "_" + number_postfix - if (os.path.exists(fn)): + if os.path.exists(fn): num_found = num_found + 1 if num_found == 0: return "none" @@ -533,17 +527,17 @@ def images_exist(views, basename, image_count): def create_movie(basename, moviename): - ''' + """ Creates a movie from a .png image series. :param basename: :param moviename: :return: - ''' - print "Creating movie '" + moviename + "' from files '" + basename + "*.png ..." + """ + print("Creating movie '" + moviename + "' from files '" + basename + "*.png ...") args = "-limit memory 4GB -delay 50 -quality 100 -dispose Background %s*.png %s" % (basename, moviename) - print "convert %s" % args - meanie3D.app.external.execute_command('convert', args, False) - print "done." + print("convert %s" % args) + external.run('convert', args, False) + print("done.") return @@ -553,114 +547,114 @@ def create_movie(basename, moviename): # # --------------------------------------------------------- -def getVisitValue(object, key): - ''' +def getVisitValue(obj, key): + """ Visit's objects are a bit of an arse. Standard getattr does not seem to work in a lot of times. - :param object: + :param obj: :param key: :return: - ''' - value = getattr(object, key) + """ + value = getattr(obj, key) if value is None: - getter = getattr(object, 'Get' + meanie3D.app.utils.capitalize(key)) + getter = getattr(obj, 'Get' + utils.capitalize(key)) if getter: value = getter() return value -def setValueForKeyPath(object, keypath, value): - ''' +def setValueForKeyPath(obj, key_path, value): + """ Sets the given object values along a keypath separated by periods. Example: axes2D.yAxis.title.units. - :param object: - :param keypath: + :param obj: + :param key_path: :param value: :return: - ''' - keys = keypath.split(".") + """ + keys = key_path.split(".") if len(keys) > 1: - if type(object) is dict: - nextObject = object[keys[0]] + if type(obj) is dict: + nextObject = obj[keys[0]] else: - nextObject = getattr(object, keys[0]) + nextObject = getattr(obj, keys[0]) keys.pop(0) remainingPath = ".".join(keys) setValueForKeyPath(nextObject, remainingPath, value) else: - objectValue = getVisitValue(object, keypath) + objectValue = getVisitValue(obj, key_path) # The specific enumerated values in Visit are configured # as strings in configuration dictionary, but are int values # in visualisation objects. Try to set an enumerated value when hitting # this combination if type(value) != type(objectValue): - if (type(value) == unicode and type(objectValue) == str): + if type(value) == unicode and type(objectValue) == str: # unicode -> str autoconversion value = str(value) - elif (type(value) == int and type(objectValue) == float): + elif type(value) == int and type(objectValue) == float: # int -> float autoconversion value = float(value) - elif (type(value) == float and type(objectValue) == int): + elif type(value) == float and type(objectValue) == int: # float -> int autoconversion value = int(value) else: # Probably a visit enumeration constant. Resolve the value of the # enumerated value and use that instead - value = getVisitValue(object, str(value)) + value = getVisitValue(obj, str(value)) - if type(object) is dict: - object[keypath] = value + if type(obj) is dict: + obj[key_path] = value else: try: - setattr(object, keypath, value) + setattr(obj, key_path, value) except: try: # try a setter - setter = getattr(object, 'Set' + meanie3D.app.utils.capitalize(keypath)) + setter = getattr(obj, 'Set' + utils.capitalize(key_path)) if setter: setter(value) except: - sys.stderr.write("Can't set value %s for key '%s' on object" % (str(value), keypath)) + sys.stderr.write("Can't set value %s for key '%s' on object" % (str(value), key_path)) raise return -def getValueForKeyPath(object, keypath): - ''' +def getValueForKeyPath(obj, key_path): + """ Gets the given object values along a keypath separated by periods. Example: axes2D.yAxis.title.units. - :param object: - :param keypath: + :param obj: + :param key_path: :return: - ''' - return meanie3D.app.utils.getValueForKeyPath(object, keypath) + """ + return utils.getValueForKeyPath(obj, key_path) -def updateVisitObjectFromDictionary(object, dictionary): - ''' +def updateVisitObjectFromDictionary(obj, dictionary): + """ Sets the values from the dictionary on the given object. - :param object: + :param obj: :param dictionary: :return: - ''' + """ for key, value in dictionary.items(): if type(value) is list: - setValueForKeyPath(object, key, tuple(value)) + setValueForKeyPath(obj, key, tuple(value)) else: - setValueForKeyPath(object, key, value) + setValueForKeyPath(obj, key, value) return -def add_datetime(conf, netcdf_file,time_index): - ''' +def add_datetime(conf, netcdf_file, time_index): + """ Reads the timestamp info from NetCDF file. Must comply to the cf-metadata convention. The timestamp position and format can be controlled through the (optional) configuration entry: :param conf: - :param filename: + :param netcdf_file: :param time_index: :return: - ''' + """ ncfile = netCDF4.Dataset(netcdf_file, "r") times = ncfile.variables['time'] ti = time_index @@ -669,34 +663,35 @@ def add_datetime(conf, netcdf_file,time_index): have_valid_time = False try: - date = netCDF4.num2date(times[:],units=times.units,calendar=times.calendar)[ti] + date = netCDF4.num2date(times[:], units=times.units, calendar=times.calendar)[ti] date = date.replace(microsecond=0) have_valid_time = True except ValueError as vi: - print "Error reading time information from file:" - print vi.message - print "Falling back on time index" + print("Error reading time information from file:") + print(vi.message) + print("Falling back on time index") date = time_index x = 0.725 y = 0.95 - format = '__default__' - if getValueForKeyPath(conf,'timestamp'): - if getValueForKeyPath(conf,'timestamp.x'): - x = getValueForKeyPath(conf,'timestamp.x') - if getValueForKeyPath(conf,'timestamp.y'): - y = getValueForKeyPath(conf,'timestamp.y') - if getValueForKeyPath(conf,'timestamp.format'): - format = getValueForKeyPath(conf,'timestamp.format') + _format = '__default__' + if getValueForKeyPath(conf, 'timestamp'): + if getValueForKeyPath(conf, 'timestamp.x'): + x = getValueForKeyPath(conf, 'timestamp.x') + if getValueForKeyPath(conf, 'timestamp.y'): + y = getValueForKeyPath(conf, 'timestamp.y') + if getValueForKeyPath(conf, 'timestamp.format'): + _format = getValueForKeyPath(conf, 'timestamp.format') if have_valid_time: - if format == '__default__': + if _format == '__default__': addTextAnnotation(x, y, date.isoformat()) else: - addTextAnnotation(x, y, date.strftime(format)) + addTextAnnotation(x, y, date.strftime(_format)) else: addTextAnnotation(x, y, "time index: %d" % time_index) + # --------------------------------------------------------- # # Code to be phased out. @@ -704,7 +699,7 @@ def add_datetime(conf, netcdf_file,time_index): # --------------------------------------------------------- def create_dual_panel(basename_left, basename_right, basename_combined): - ''' + """ Creates a series of images combined from two series of images as one. It removes the date stamp on the left side and cuts the right side such, that no legend is visible. @@ -712,11 +707,12 @@ def create_dual_panel(basename_left, basename_right, basename_combined): :param basename_right: :param basename_combined: :return: - ''' + """ left_files = sorted(glob.glob(basename_left + "*.png")) right_files = sorted(glob.glob(basename_right + "*.png")) if len(left_files) != len(right_files): - print "ERROR:the two image series " + basename_left + "*.png and " + basename_right + "*.png have different lengths!" + print("ERROR:the two image series " + basename_left + "*.png and " + + basename_right + "*.png have different lengths!") return # create a small image to blank the datestamp with @@ -726,16 +722,13 @@ def create_dual_panel(basename_left, basename_right, basename_combined): for i in range(len(left_files)): # create backdrop combined = basename_combined + str(i) + ".png" - meanie3D.app.external.execute_command("convert", "-size 1852x1024 xc:white " + combined, shell=True) + external.run("convert", "-size 1852x1024 xc:white " + combined) # copy images and blank date - meanie3D.app.external.execute_command("composite", - "-geometry +826+0 " + right_files[i] + " " + combined + " " + combined) - meanie3D.app.external.execute_command("composite", - "-geometry +0+0 " + left_files[i] + " " + combined + " " + combined) - meanie3D.app.external.execute_command("composite", - "-geometry +735+20 dateblind.png " + combined + " " + combined) + external.run("composite", "-geometry +826+0 " + right_files[i] + " " + combined + " " + combined) + external.run("composite", "-geometry +0+0 " + left_files[i] + " " + combined + " " + combined) + external.run("composite", "-geometry +735+20 dateblind.png " + combined + " " + combined) # delete the dateblind call("rm dateblind.png", shell=True) - return \ No newline at end of file + return diff --git a/python/setup.py b/python/setup.py index ae7e4fc..841adcc 100644 --- a/python/setup.py +++ b/python/setup.py @@ -1,18 +1,19 @@ -from setuptools import find_packages, setup +from setuptools import setup + setup( - name='meanie3D', - version='1.6.1', - packages=["meanie3D","meanie3D.app", "meanie3D.visualisation", "meanie3D.resources"], - include_package_data=True, - url='http://git.meteo.uni-bonn.de/projects/meanie3d', - license='MIT License', - author='Juergen Simon', - author_email='simon@webtecc.com', - zip_safe = False, - description='Python scripts for running the meanie3D clustering and tracking algorithms and visualising the results.', - entry_points = { - 'console_scripts' : [ - 'meanie3D = meanie3D.meanie3D_run:main' - ] - } + name='meanie3D', + version='1.6.1', + packages=["meanie3D", "meanie3D.app", "meanie3D.visualisation", "meanie3D.resources"], + include_package_data=True, + url='http://git.meteo.uni-bonn.de/projects/meanie3d', + license='MIT License', + author='Juergen Simon', + author_email='simon@webtecc.com', + zip_safe=False, + description='Python scripts for running the meanie3D clustering and tracking algorithms and visualising the results.', + entry_points={ + 'console_scripts': [ + 'meanie3D = meanie3D.meanie3D_run:main' + ] + } ) From d6d3ce28e8991af6418286c07c34af0f68e7ce31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 23 Jun 2020 13:45:36 +0100 Subject: [PATCH 36/65] Python: exit codes unified. Bugfixes. --- python/meanie2tic/meanie2tic.py | 6 ++-- python/meanie3D/app/external.py | 36 +++++++++++++-------- python/meanie3D/app/postprocessing.py | 3 ++ python/meanie3D/app/tracking.py | 2 -- python/meanie3D/app/utils.py | 12 +++---- python/meanie3D/resources/clusters_visit.py | 4 +-- python/meanie3D/visualisation/clusters.py | 13 +++++--- python/meanie3D/visualisation/utils.py | 18 +++++------ 8 files changed, 54 insertions(+), 40 deletions(-) diff --git a/python/meanie2tic/meanie2tic.py b/python/meanie2tic/meanie2tic.py index 7015f7a..139638b 100755 --- a/python/meanie2tic/meanie2tic.py +++ b/python/meanie2tic/meanie2tic.py @@ -55,7 +55,7 @@ def write_csv(json): objects_file = open('objects.csv', 'wb') if (not objects_file): sys.stderr.write("Could not open 'objects.csv' for writing.") - sys.exit(-1) + sys.exit(1) objects = csv.writer(objects_file, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL) @@ -122,7 +122,7 @@ def write_csv(json): tracks_file = open('tracks.csv', 'wb') if (not tracks_file): sys.stderr.write("Could not open 'tracks.csv' for writing.") - sys.exit(-1) + sys.exit(1) tracks = csv.writer(tracks_file, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL) @@ -200,7 +200,7 @@ def main(): json = load_json(filename) if not json: sys.stderr.write("ERROR:could not read file %s" % filename) - sys.exit(-1) + sys.exit(1) write_csv(json) return diff --git a/python/meanie3D/app/external.py b/python/meanie3D/app/external.py index c499a61..cd9c6c9 100644 --- a/python/meanie3D/app/external.py +++ b/python/meanie3D/app/external.py @@ -26,6 +26,7 @@ import platform import subprocess import sys +import traceback from subprocess import Popen, CalledProcessError # --------------------------------------------- @@ -172,18 +173,18 @@ def execute_command(command, parameters, return_output=False, silent=True): raise ValueError("Requested command: %s was not found") cmd = getCommand(COMMAND_MAP.get(command)) + ' ' + parameters + print("%s %s" % (command, parameters)) if return_output: p = Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = p.communicate() return p.returncode, out, err else: - if silent: - p = Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - else: - # TODO: this does seem to be broken - p = Popen(cmd, shell=True, stdout=subprocess.STDOUT, stderr=subprocess.STDOUT) + p = Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if not silent: + print(p.stdout.read()) + print(p.stderr.read()) p.communicate() - return p.returncode + return p.returncode, None, None def run(command, args, return_output=False): @@ -191,18 +192,27 @@ def run(command, args, return_output=False): Convenience wrapper around executing external commands :param command: Command to execute :param args: Arguments to the program (as string) - :param return_output: Should return output of stderr? + :param return_output: Should return output of stdout? :return: ({boolean}, {stdout}, {stderr}) """ try: - return_code, outstream, errstream = execute_command(command, args, return_output) - return (return_code == 0), outstream, errstream + return_code, stdout, errstream = execute_command(command, args, return_output) + return (return_code == 0), stdout except ValueError as ex: - print("ERROR: %s", ex.__str__()) + tb = traceback.format_exc() + error = ex.__str__() except CalledProcessError as ex: - print("ERROR: %s", ex.__str__()) + tb = traceback.format_exc() + error = ex.__str__() except IOError as ex: - print("ERROR: %s", ex.__str__()) + tb = traceback.format_exc() + error = ex.__str__() + except TypeError as ex: + tb = traceback.format_exc() + error = ex.__str__() except: - print("ERROR:%s" % sys.exc_info()[0]) + tb = traceback.format_exc() + error = sys.exc_info()[0] + print(error) + print(tb) return False diff --git a/python/meanie3D/app/postprocessing.py b/python/meanie3D/app/postprocessing.py index 45e67da..4e7fb2d 100644 --- a/python/meanie3D/app/postprocessing.py +++ b/python/meanie3D/app/postprocessing.py @@ -25,6 +25,7 @@ import glob import os import shutil +import sys import tempfile import time @@ -164,6 +165,8 @@ def run_trackstats(configuration, directory): return True success = external.run("meanie3D-trackstats", " ".join(params)) + if not success: + sys.exit(0) os.chdir("..") return success diff --git a/python/meanie3D/app/tracking.py b/python/meanie3D/app/tracking.py index 1bb2e12..2254644 100644 --- a/python/meanie3D/app/tracking.py +++ b/python/meanie3D/app/tracking.py @@ -282,7 +282,6 @@ def run(config, time_index): if config['time_operations']: start_time = time.time() - print("meanie3D-detect " + params) success = external.run("meanie3D-detect", params, return_output=True) if not success: print("ERROR: could not perform detection") @@ -314,7 +313,6 @@ def run(config, time_index): if config['time_operations']: start_time = time.time() - print("meanie3D-track" + params) success = external.run("meanie3D-track", params, return_output=True) if not success: print("ERROR: could not perform detection") diff --git a/python/meanie3D/app/utils.py b/python/meanie3D/app/utils.py index 4024a4b..b9baa92 100644 --- a/python/meanie3D/app/utils.py +++ b/python/meanie3D/app/utils.py @@ -138,7 +138,7 @@ def getSafe(dictionary, key): :param key: :return: """ - if key in dict and dictionary[key]: + if key in dictionary and dictionary[key]: return dictionary[key] else: return None @@ -198,14 +198,14 @@ def setValueForKeyPath(obj, keypath, value): return -def getValueForKeyPath(obj, keypath): +def getValueForKeyPath(obj, key_path): """ Gets the given object values along a keypath separated by periods. Example: axes2D.yAxis.title.units :param obj: - :param keypath: + :param key_path: :return: """ - keys = keypath.split(".") + keys = key_path.split(".") if len(keys) > 1: if obj is None: return None @@ -224,9 +224,9 @@ def getValueForKeyPath(obj, keypath): if obj is None: return None elif type(obj) is dict: - value = getSafe(obj, keypath) + value = getSafe(obj, key_path) else: - value = getattr(obj, keypath) + value = getattr(obj, key_path) return value diff --git a/python/meanie3D/resources/clusters_visit.py b/python/meanie3D/resources/clusters_visit.py index c65cf00..55dda7e 100644 --- a/python/meanie3D/resources/clusters_visit.py +++ b/python/meanie3D/resources/clusters_visit.py @@ -39,7 +39,7 @@ _M3D_HOME = "P_M3D_HOME" _RESUME = "P_RESUME" _CONFIG_FILE = "P_CONFIGURATION_FILE" -_USES_TIME = "P_USES_TIME" +_USES_TIME = P_USES_TIME _START_TIME = "P_START_TIME_INDEX" _END_TIME = "P_END_TIME_INDEX" @@ -63,5 +63,5 @@ # run it # pdb.set_trace() -meanie3D.visualisation.clusters.run(configuration) +meanie3D.visualisation.clusters.run(configuration, print_conf=True) quit() diff --git a/python/meanie3D/visualisation/clusters.py b/python/meanie3D/visualisation/clusters.py index f4d4b04..9db4e0c 100644 --- a/python/meanie3D/visualisation/clusters.py +++ b/python/meanie3D/visualisation/clusters.py @@ -27,6 +27,7 @@ import subprocess import sys import time +import pprint import visit @@ -61,14 +62,15 @@ def add_clusters(cluster_vtk_file, configuration): return -def run(conf): +def run(conf, print_conf=False): """ Visualises clusters of a given run. :param conf: :return: """ - # pp = pprint.PrettyPrinter() - # pp.pprint(conf) + if print_conf: + pp = pprint.PrettyPrinter() + pp.pprint(conf) # Make sure the global configuration is in place utils.run_global_visit_configuration(conf) @@ -226,8 +228,9 @@ def run(conf): params += " --vtk-dimensions=%s" % vtkDimString # pdb.set_trace(); - print("meanie3D-cfm2vtk %s" % params) - meanie3D.app.external.execute_command('meanie3D-cfm2vtk', params) + success = external.run('meanie3D-cfm2vtk', params) + if not success: + sys.exit(1) print(" done. (%.2f seconds)" % (time.time() - start_time)) # Move cluster output file to individual file diff --git a/python/meanie3D/visualisation/utils.py b/python/meanie3D/visualisation/utils.py index ab6d40a..6af6ab7 100644 --- a/python/meanie3D/visualisation/utils.py +++ b/python/meanie3D/visualisation/utils.py @@ -33,27 +33,27 @@ # make sure external commands are available external.locateCommands(['convert', 'composite', 'python']) -ret_code, paths_string = external.run('python', '-c "import sys; print sys.path"', return_output=True) -if ret_code == 0: - print ("Attempting to locate python module netCDF4") +success, paths_string = external.run('python', '-c "import sys; print sys.path"', return_output=True) +if success: home = os.path.expanduser('~') searchPaths = ['/usr/lib', '/usr/local/lib', home + os.path.sep + '.pyenv/versions'] - print("Looking in paths:") - print(searchPaths) + print ("Attempting to locate python module netCDF4 in paths %s" % ",".join(searchPaths)) + # print("Looking in paths:") + # print(searchPaths) result = utils.find_in_paths(searchPaths, "netCDF4", "site-packages") if not result: result = utils.find_in_paths(searchPaths, "netCDF4", "dist-packages") if not result: print("Failed to locate python module netCDF4") - exit(-1) + exit(1) else: print("Found netCDF4 at %s" % result) sys.path.append(os.path.split(result)[0]) - print("Python path after adding system search directories:") - print(sys.path) + # print("Python path after adding system search directories:") + # print(sys.path) else: print("Failed to obtain system's python path") - exit(-1) + exit(1) # Now that the path is extended, import the rest import netCDF4 From f4e030b041e4cf59ee512de24764d7876604e3db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 23 Jun 2020 14:38:51 +0100 Subject: [PATCH 37/65] Docker fixes. Re-introduced (mute) parameter -o (output dir) --- .dockerignore | 8 +++-- Dockerfile.vtk | 12 +++---- python/meanie3D/meanie3D_run.py | 56 +++++++++++++++------------------ 3 files changed, 36 insertions(+), 40 deletions(-) diff --git a/.dockerignore b/.dockerignore index 363ba1e..032f0f2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,12 @@ .idea .vscode build -CMakeFiles cmake-build-debug +CMakeFiles +gtest-* +html +m3D-test-* +nbproject* +radolan2netcdf Makefile.docker cmake_install.cmake -html \ No newline at end of file diff --git a/Dockerfile.vtk b/Dockerfile.vtk index ed7adf8..0f01487 100644 --- a/Dockerfile.vtk +++ b/Dockerfile.vtk @@ -2,8 +2,8 @@ FROM debian:stable ARG VISIT_VERSION="3.1.0" ARG VISIT_PKG_URL="https://portal.nersc.gov/project/visit/releases/3.1.0/visit3_1_0.linux-x86_64-debian9.tar.gz" -ARG VISIT_INSTALLER_URL=https://portal.nersc.gov/project/visit/releases/3.1.0/${VISIT_INSTALLER} ARG VISIT_INSTALLER=visit-install3_1_0 +ARG VISIT_INSTALLER_URL="https://portal.nersc.gov/project/visit/releases/3.1.0/${VISIT_INSTALLER}" RUN apt-get -y update --fix-missing RUN apt-get -y upgrade @@ -16,15 +16,13 @@ gcc g++ libomp5 \ python python-pip \ libboost-all-dev libflann1.9 libflann-dev blitz++ \ shapelib libhdf5-dev netcdf-bin libnetcdf-dev libnetcdf-c++4 libnetcdf-c++4-dev zlib1g zlib1g-dev -RUN pip install setuptools netcdf4 external utils +RUN pip install setuptools external utils # Visualisation -RUN pip install Cython h5py netcdf4 +RUN pip install Cython h5py "netCDF4>=1.3.0<1.4.0" RUN apt-get -y --fix-missing install gnuplot vtk7 libvtk7-dev -RUN wget --quiet ${VISIT_INSTALLER_URL} -RUN wget --quiet ${VISIT_PKG_URL} -RUN chmod u+x ./visit-install3_1_0 -RUN echo "1" | ./visit-install3_1_0 3.1.0 linux-x86_64 /usr/local/visit +RUN wget --quiet ${VISIT_INSTALLER_URL} && wget --quiet ${VISIT_PKG_URL} && chmod u+x ./${VISIT_INSTALLER} +RUN echo "1" | ./visit-install3_1_0 3.1.0 linux-x86_64-debian9 /usr/local/visit ENV VISIT_EXECUTABLE=/usr/local/visit/bin/visit # libradolan diff --git a/python/meanie3D/meanie3D_run.py b/python/meanie3D/meanie3D_run.py index 827eb88..28ce5dc 100755 --- a/python/meanie3D/meanie3D_run.py +++ b/python/meanie3D/meanie3D_run.py @@ -39,16 +39,19 @@ meanie3D.app.external.locateCommands(["meanie3D-detect", "meanie3D-track", "meanie3D-trackstats", "rm"]) -# ---------------------------------------------------------------------------- -## Prints usage and exits -# def usage(): - print( - "meanie3D -c= -f= [-s=] [--start=t1 --end=t2] [--resume,-r] [--help,-h] [--version] [--time-operations]") + """ + Prints usage information. + :return: + """ + print("meanie3D -c=/path/to/json -f=/path/to/netcdf [-o /path/to/output] " + "[-s=] [--start=t1 --end=t2] [--resume,-r] [--help,-h] " + "[--version] [--time-operations]") print("runs a complete set of netcdf files through the clustering/tracking") print("-c : json config file specifying variables etc.") print("-f : directory containing the files. It is assumed that") print(" the files are in the correct order when sorted alphabetically.") + print("-o : Output directory. If omitted, this is the same as the input directory") print("--scales,-s : comma separated list of scale parameters.") print("--ranges,-r : comma separated list of bandwidths, one for each spatial and value range variable.") print("--json-example : prints an example .json configuration and exits") @@ -61,16 +64,13 @@ def usage(): print("--version : prints the version information and exits") print("--time-operations: time operations and print out their elapsed time") print("--skip-trackstats: skip meanie3D-trackstats, even if the configuration is present") - sys.exit(1) - return -# ---------------------------------------------------------------------------- - -# ---------------------------------------------------------------------------- -## Prints the configuration file .json file description and exits -# def print_configuration_format(): + """ + Prints the configuration file .json file description + :return: + """ print('{') print(' "description" : "...", /* Description of file content */') print(' "data" : {...}, /* call --help config.data for help on this section */') @@ -78,18 +78,14 @@ def print_configuration_format(): print(' "tracking" : {...}, /* call --help config.tracking for help on this section */') print(' "postprocessing" : {...} /* call --help config.postprocessing for help on this section */') print('}') - sys.exit(1) - -# ---------------------------------------------------------------------------- -# ---------------------------------------------------------------------------- -## Prints version info and exits -# def print_version(): + """ + Prints version info + :return: + """ print("meanie3D: " + meanie3D.getVersion() + "\n") - sys.exit(1) - return # ---------------------------------------------------------------------------- @@ -126,9 +122,8 @@ def main(): output_dir = "." netcdf_dir = "" num_params = 0 - - start_time_index = -1; - end_time_index = -1; + start_time_index = -1 + end_time_index = -1 for o, a in opts: @@ -151,6 +146,7 @@ def main(): elif o in ["--json-example"]: print_configuration_format() + sys.exit(0) elif o in ["--resume"]: resume = True @@ -163,10 +159,11 @@ def main(): elif o in ["--help"]: usage() - sys.exit() + sys.exit(0) elif o in ["--version"]: print_version() + sys.exit(0) elif o in ["--time-operations"]: time_operations = True @@ -182,26 +179,23 @@ def main(): if num_params < 2: usage() + sys.exit(1) uses_time = False # sanity checks on time range if start_time_index != -1 or end_time_index != -1: - if (start_time_index == -1 and end_time_index != -1) or (start_time_index != -1 and end_time_index == -1): print("start-time-index and end-time-index must both be set") - sys.exit(2) - + sys.exit(1) if start_time_index > end_time_index: print("start-time-index is after end-time-index!") - sys.exit(2) - + sys.exit(1) uses_time = True # Parse configuration data and expand - - configuration = meanie3D.app.utils.load_configuration(config_file); + configuration = meanie3D.app.utils.load_configuration(config_file) print("-------------------------------------------------------------------------------------") print("Configuration: " + configuration['description']) From 4a7d89805b45d32aa985924a5cfa265194cc6927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 23 Jun 2020 14:59:42 +0100 Subject: [PATCH 38/65] cmake fix. --- CMakeLists.txt | 20 ++++++++++---------- Dockerfile.vtk | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80efd69..891bfe0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,7 +67,7 @@ OPTION(WITH_PYTHON "Add the python frontend" ON) # PRESETS # ------------------------------------- -IF(PRESET MATCHES "docker") +IF(PRESET MATCHES "^docker$") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "Release") SET(OPENMP_ENABLED ON) @@ -78,7 +78,7 @@ IF(PRESET MATCHES "docker") SET(SATELLITE_ENABLED OFF) SET(KONRAD_ENABLED OFF) SET(DOCS_ENABLED OFF) -ELSEIF(PRESET MATCHES "docker-vtk") +ELSEIF(PRESET MATCHES "^docker-vtk$") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "Release") SET(OPENMP_ENABLED ON) @@ -89,7 +89,7 @@ ELSEIF(PRESET MATCHES "docker-vtk") SET(SATELLITE_ENABLED OFF) SET(KONRAD_ENABLED OFF) SET(DOCS_ENABLED OFF) -ELSEIF (PRESET MATCHES "dev-all") +ELSEIF (PRESET MATCHES "^dev-all$") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "Debug") SET(OPENMP_ENABLED ON) @@ -100,7 +100,7 @@ ELSEIF (PRESET MATCHES "dev-all") SET(SATELLITE_ENABLED ON) SET(KONRAD_ENABLED ON) SET(DOCS_ENABLED ON) -ELSEIF(PRESET MATCHES "dev-core") +ELSEIF(PRESET MATCHES "^dev-core$") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "Debug") SET(OPENMP_ENABLED ON) @@ -111,7 +111,7 @@ ELSEIF(PRESET MATCHES "dev-core") SET(SATELLITE_ENABLED OFF) SET(KONRAD_ENABLED OFF) SET(DOCS_ENABLED ON) -ELSEIF (PRESET MATCHES "dev-vtk") +ELSEIF (PRESET MATCHES "^dev-vtk$") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "Debug") SET(OPENMP_ENABLED ON) @@ -122,7 +122,7 @@ ELSEIF (PRESET MATCHES "dev-vtk") SET(SATELLITE_ENABLED OFF) SET(KONRAD_ENABLED OFF) SET(DOCS_ENABLED ON) -ELSEIF (PRESET MATCHES "dev-all") +ELSEIF (PRESET MATCHES "^dev-all$") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "Debug") SET(OPENMP_ENABLED ON) @@ -133,7 +133,7 @@ ELSEIF (PRESET MATCHES "dev-all") SET(SATELLITE_ENABLED ON) SET(KONRAD_ENABLED ON) SET(DOCS_ENABLED ON) -ELSEIF (PRESET MATCHES "prod-core") +ELSEIF (PRESET MATCHES "^prod-core$") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "Release") SET(OPENMP_ENABLED ON) @@ -144,7 +144,7 @@ ELSEIF (PRESET MATCHES "prod-core") SET(SATELLITE_ENABLED OFF) SET(KONRAD_ENABLED OFF) SET(DOCS_ENABLED OFF) -ELSEIF (PRESET MATCHES "prod-vtk") +ELSEIF (PRESET MATCHES "^prod-vtk$") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "Release") SET(OPENMP_ENABLED ON) @@ -155,7 +155,7 @@ ELSEIF (PRESET MATCHES "prod-vtk") SET(SATELLITE_ENABLED OFF) SET(KONRAD_ENABLED OFF) SET(DOCS_ENABLED OFF) -ELSEIF (PRESET MATCHES "fast-core") +ELSEIF (PRESET MATCHES "^fast-core$") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "MinSizeRel") SET(OPENMP_ENABLED ON) @@ -166,7 +166,7 @@ ELSEIF (PRESET MATCHES "fast-core") SET(SATELLITE_ENABLED OFF) SET(KONRAD_ENABLED OFF) SET(DOCS_ENABLED OFF) -ELSEIF (PRESET MATCHES "fast-vtk") +ELSEIF (PRESET MATCHES "^fast-vtk$") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "MinSizeRel") SET(OPENMP_ENABLED ON) diff --git a/Dockerfile.vtk b/Dockerfile.vtk index 0f01487..14c699b 100644 --- a/Dockerfile.vtk +++ b/Dockerfile.vtk @@ -30,10 +30,10 @@ RUN git clone https://github.com/JuergenSimon/radolan.git RUN cd radolan && cmake . && make install && cd .. # Meanie3D -RUN git clone --recurse-submodules --depth=1 https://github.com/JuergenSimon/meanie3D +RUN git clone https://github.com/JuergenSimon/meanie3D WORKDIR /meanie3D RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch --all -RUN git checkout --track remotes/origin/dockerize && git pull +RUN git checkout --track remotes/origin/dockerize RUN git pull && cmake -DPRESET=docker-vtk . && make install # Cleanup From 8c819f59e5a73454fa15122561bc2b3c3320b14c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 23 Jun 2020 15:12:18 +0100 Subject: [PATCH 39/65] Python fix (output dir) --- python/meanie3D/app/postprocessing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/meanie3D/app/postprocessing.py b/python/meanie3D/app/postprocessing.py index 4e7fb2d..149b078 100644 --- a/python/meanie3D/app/postprocessing.py +++ b/python/meanie3D/app/postprocessing.py @@ -515,7 +515,7 @@ def run(configuration): # Otherwise it's 'clustering'. To be safe, iterate over both scales = configuration['scales'] if scales: - directories = ['scale%s' % scale for scale in scales] + directories = [ configuration['output_dir'] + 'scale%s' % scale for scale in scales] else: directories = ['clustering'] From 2520dce026b9a505587c389e11bda710a2db52ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 23 Jun 2020 15:26:27 +0100 Subject: [PATCH 40/65] Python fix (output dir) --- Dockerfile.vtk | 3 ++- python/meanie3D/app/postprocessing.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile.vtk b/Dockerfile.vtk index 14c699b..7b0138b 100644 --- a/Dockerfile.vtk +++ b/Dockerfile.vtk @@ -34,7 +34,8 @@ RUN git clone https://github.com/JuergenSimon/meanie3D WORKDIR /meanie3D RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch --all RUN git checkout --track remotes/origin/dockerize -RUN git pull && cmake -DPRESET=docker-vtk . && make install +RUN git pull +RUN cmake -DPRESET=docker-vtk . && make install # Cleanup WORKDIR / diff --git a/python/meanie3D/app/postprocessing.py b/python/meanie3D/app/postprocessing.py index 149b078..96bff5a 100644 --- a/python/meanie3D/app/postprocessing.py +++ b/python/meanie3D/app/postprocessing.py @@ -515,7 +515,8 @@ def run(configuration): # Otherwise it's 'clustering'. To be safe, iterate over both scales = configuration['scales'] if scales: - directories = [ configuration['output_dir'] + 'scale%s' % scale for scale in scales] + template = configuration['output_dir'] + os.path.sep + "scale%s" + directories = [ template % scale for scale in scales] else: directories = ['clustering'] From 8a01d90e05590fb88eef74b89bfdbfd05920bd92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 23 Jun 2020 15:52:58 +0100 Subject: [PATCH 41/65] Python fix (output dir) --- python/meanie3D/app/postprocessing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/meanie3D/app/postprocessing.py b/python/meanie3D/app/postprocessing.py index 96bff5a..e55077e 100644 --- a/python/meanie3D/app/postprocessing.py +++ b/python/meanie3D/app/postprocessing.py @@ -515,8 +515,7 @@ def run(configuration): # Otherwise it's 'clustering'. To be safe, iterate over both scales = configuration['scales'] if scales: - template = configuration['output_dir'] + os.path.sep + "scale%s" - directories = [ template % scale for scale in scales] + directories = [ configuration['output_dir'] + os.path.sep + "scale%s" % scale for scale in scales] else: directories = ['clustering'] From 8cd13bbd3e64a694a9c03f21eb0c21ad3bc52a19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 23 Jun 2020 18:01:38 +0100 Subject: [PATCH 42/65] Checking in work in progress. --- Dockerfile.vtk | 19 ++++++++++--------- python/meanie3D/app/postprocessing.py | 3 ++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Dockerfile.vtk b/Dockerfile.vtk index 7b0138b..dad8286 100644 --- a/Dockerfile.vtk +++ b/Dockerfile.vtk @@ -1,9 +1,9 @@ FROM debian:stable -ARG VISIT_VERSION="3.1.0" -ARG VISIT_PKG_URL="https://portal.nersc.gov/project/visit/releases/3.1.0/visit3_1_0.linux-x86_64-debian9.tar.gz" -ARG VISIT_INSTALLER=visit-install3_1_0 -ARG VISIT_INSTALLER_URL="https://portal.nersc.gov/project/visit/releases/3.1.0/${VISIT_INSTALLER}" +ARG VISIT_VERSION="3.1.2" +ARG VISIT_PKG_URL="https://portal.nersc.gov/project/visit/releases/3.1.2/visit3_1_2.linux-x86_64-debian9.tar.gz" +ARG VISIT_INSTALLER=visit-install3_1_2 +ARG VISIT_INSTALLER_URL="https://portal.nersc.gov/project/visit/releases/3.1.2/${VISIT_INSTALLER}" RUN apt-get -y update --fix-missing RUN apt-get -y upgrade @@ -11,7 +11,7 @@ RUN apt-get -y dist-upgrade RUN apt-get -y install software-properties-common RUN apt-get -y update RUN apt-get -y install \ -wget git cmake \ +wget git cmake cpio \ gcc g++ libomp5 \ python python-pip \ libboost-all-dev libflann1.9 libflann-dev blitz++ \ @@ -22,7 +22,9 @@ RUN pip install setuptools external utils RUN pip install Cython h5py "netCDF4>=1.3.0<1.4.0" RUN apt-get -y --fix-missing install gnuplot vtk7 libvtk7-dev RUN wget --quiet ${VISIT_INSTALLER_URL} && wget --quiet ${VISIT_PKG_URL} && chmod u+x ./${VISIT_INSTALLER} -RUN echo "1" | ./visit-install3_1_0 3.1.0 linux-x86_64-debian9 /usr/local/visit +RUN rm -rf /usr/local/visit &&\ + mkdir -p /usr/local/visit/3.1.2+/linux-x86_64/bin &&\ + echo "1" | ./visit-install3_1_2 3.1.2 linux-x86_64-debian9 /usr/local/visit ENV VISIT_EXECUTABLE=/usr/local/visit/bin/visit # libradolan @@ -34,13 +36,12 @@ RUN git clone https://github.com/JuergenSimon/meanie3D WORKDIR /meanie3D RUN git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" && git fetch --all RUN git checkout --track remotes/origin/dockerize -RUN git pull -RUN cmake -DPRESET=docker-vtk . && make install +RUN git pull && cmake -DPRESET=docker-vtk . && make install # Cleanup WORKDIR / RUN rm -rf meanie3D radolan visit* -RUN apt-get remove -y wget git cmake +RUN apt-get remove -y wget git cmake cpio RUN apt autoremove -y # Prepare for runtime diff --git a/python/meanie3D/app/postprocessing.py b/python/meanie3D/app/postprocessing.py index e55077e..d3888a3 100644 --- a/python/meanie3D/app/postprocessing.py +++ b/python/meanie3D/app/postprocessing.py @@ -515,7 +515,8 @@ def run(configuration): # Otherwise it's 'clustering'. To be safe, iterate over both scales = configuration['scales'] if scales: - directories = [ configuration['output_dir'] + os.path.sep + "scale%s" % scale for scale in scales] + base = configuration["output_dir"] + os.path.sep + "scale" + directories = [ base+"%s" % scale for scale in scales ] else: directories = ['clustering'] From b87b22e293ddcade1999aa2aaa1439ab01fd96bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Wed, 24 Jun 2020 13:24:16 +0100 Subject: [PATCH 43/65] Streamlined cmake_modules and added output to CMakeList on found libs --- CMakeLists.txt | 22 ++- cmake_modules/FindBlitz.cmake | 12 +- cmake_modules/FindCFA.cmake | 23 --- cmake_modules/FindFLANN.cmake | 16 +- cmake_modules/FindHDF5.cmake | 24 +-- cmake_modules/FindKDTREE.cmake | 19 -- cmake_modules/FindNETCDF.cmake | 24 +-- cmake_modules/FindNanoFLANN.cmake | 18 +- cmake_modules/FindOpenMP_RT.cmake | 24 --- cmake_modules/FindRADOLAN.cmake | 23 --- cmake_modules/FindSHP.cmake | 20 +- cmake_modules/FindTBB.cmake | 283 ----------------------------- cmake_modules/Findlibradolan.cmake | 6 +- cmake_modules/Findosmpbf.cmake | 20 -- 14 files changed, 69 insertions(+), 465 deletions(-) delete mode 100644 cmake_modules/FindCFA.cmake delete mode 100644 cmake_modules/FindKDTREE.cmake delete mode 100644 cmake_modules/FindOpenMP_RT.cmake delete mode 100644 cmake_modules/FindRADOLAN.cmake delete mode 100644 cmake_modules/FindTBB.cmake delete mode 100644 cmake_modules/Findosmpbf.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 891bfe0..3b73212 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,6 +195,10 @@ ENDIF() # Compilation type # ------------------------------------- MESSAGE(STATUS "Build type: ${CMAKE_BUILD_TYPE}") +IF (CMAKE_BUILD_TYPE MATCHES "Debug") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O0") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0") +ENDIF() # ------------------------------------- # Multithreading / Parallelism Choices @@ -318,6 +322,7 @@ MESSAGE(STATUS "Resolving dependencies for Meanie3D") # netcdf FIND_PACKAGE(NETCDF) IF (NETCDF_FOUND) + MESSAGE(STATUS "Found netcdf ${NETCDF_INCLUDE_DIR} ${NETCDF_VERSION}") INCLUDE_DIRECTORIES(${NETCDF_INCLUDE_DIR}) ELSE (NETCDF_FOUND) MESSAGE(FATAL_ERROR "netcdf not found INCLUDE_DIR=${NETCDF_INCLUDE_DIR} LIBRARIES=${NETCDF_LIBRARIES}") @@ -326,6 +331,7 @@ ENDIF (NETCDF_FOUND) # HDF5 FIND_PACKAGE(HDF5) IF (HDF5_FOUND) + MESSAGE(STATUS "Found HDF5 ${HDF5_INCLUDE_DIR} ${HDF5_VERSION}") INCLUDE_DIRECTORIES(${HDF5_INCLUDE_DIR}) ELSE (HDF5_FOUND) MESSAGE(FATAL_ERROR "HDF5 not found INCLUDE_DIR=${HDF5_INCLUDE_DIR} LIBRARIES=${HDF5_LIBRARIES}") @@ -334,6 +340,7 @@ ENDIF (HDF5_FOUND) # Boost FIND_PACKAGE(Boost COMPONENTS program_options thread filesystem system) IF (Boost_FOUND) + MESSAGE(STATUS "Found boost ${Boost_INCLUDE_DIR} ${Boost_VERSION}") INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIR}) ELSE (Boost_FOUND) MESSAGE(FATAL_ERROR ${Boost_ERROR_REASON}) @@ -342,6 +349,7 @@ ENDIF () # FLANN FIND_PACKAGE(FLANN) IF (FLANN_FOUND) + MESSAGE(STATUS "Found flann ${FLANN_INCLUDE_DIR} ${FLANN_VERSION}") INCLUDE_DIRECTORIES(${FLANN_INCLUDE_DIR}) ELSE (FLANN_FOUND) MESSAGE(FATAL_ERROR "libflann not found (http://www.cs.ubc.ca/research/flann/)") @@ -350,6 +358,7 @@ ENDIF (FLANN_FOUND) # Blitz++ FIND_PACKAGE(Blitz REQUIRED) IF (Blitz_FOUND) + MESSAGE(STATUS "Found blitz++ ${Blitz_INCLUDE_DIR} ${Blitz_VERSION}") INCLUDE_DIRECTORIES(${Blitz_INCLUDE_DIR}) ELSE () MESSAGE(FATAL_ERROR "blitz++ not found") @@ -357,20 +366,21 @@ ENDIF () # Shapefile FIND_PACKAGE(SHP) -IF (NOT SHP_FOUND) - ADD_DEFINITIONS(-DWITH_SHAPELIB=0) - MESSAGE(STATUS "Shapelib disabled") -ELSE () +IF (SHP_FOUND) + MESSAGE(STATUS "Found shapelib ${SHP_INCLUDE_DIR} ${SHP_VERSION}") ADD_DEFINITIONS(-DWITH_SHAPELIB=1) MESSAGE(STATUS "Shapelib enabled") INCLUDE_DIRECTORIES(${SHP_INCLUDE_DIR}) +ELSE () + ADD_DEFINITIONS(-DWITH_SHAPELIB=0) + MESSAGE(STATUS "Shapelib disabled") ENDIF () # VTK IF (VTK_ENABLED) FIND_PACKAGE(VTK REQUIRED NO_MODULE) IF (VTK_FOUND) - MESSAGE(STATUS "VTK found") + MESSAGE(STATUS "Found vtk ${VTK_INCLUDE_DIRS}") INCLUDE(${VTK_USE_FILE}) INCLUDE_DIRECTORIES(${VTK_INCLUDE_DIRS}) ELSE () @@ -380,7 +390,7 @@ ENDIF () FIND_PACKAGE(libradolan REQUIRED) IF (libradolan_FOUND) - MESSAGE(STATUS "libradolan found") + MESSAGE(STATUS "Found libradolan ${libradolan_INCLUDE_DIR}") INCLUDE_DIRECTORIES(${libradolan_INCLUDE_DIR}) ELSE() MESSAGE(FATAL_ERROR "libradolan not found") diff --git a/cmake_modules/FindBlitz.cmake b/cmake_modules/FindBlitz.cmake index 6625437..08c6a81 100644 --- a/cmake_modules/FindBlitz.cmake +++ b/cmake_modules/FindBlitz.cmake @@ -12,17 +12,19 @@ FIND_LIBRARY(Blitz_LIBRARY NAMES blitz) IF (Blitz_INCLUDE_DIR AND Blitz_LIBRARY) SET(Blitz_FOUND TRUE) -ENDIF (Blitz_INCLUDE_DIR AND Blitz_LIBRARY) +ENDIF () # handle the QUIETLY and REQUIRED arguments and set Blitz_FOUND to TRUE if # all listed variables are TRUE INCLUDE(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(Blitz DEFAULT_MSG Blitz_LIBRARY Blitz_INCLUDE_DIR) -IF(Blitz_FOUND) - SET(Blitz_LIBRARIES ${Blitz_LIBRARY} ) - SET(Blitz_INCLUDE_DIRS ${Blitz_INCLUDE_DIR} ) -ENDIF(Blitz_FOUND) +IF (Blitz_FOUND) + SET(Blitz_LIBRARIES ${Blitz_LIBRARY}) + SET(Blitz_INCLUDE_DIRS ${Blitz_INCLUDE_DIR}) +ELSEIF (BLITZ_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Could not find blitz++") +ENDIF () MARK_AS_ADVANCED(Blitz_INCLUDE_DIR Blitz_LIBRARY) diff --git a/cmake_modules/FindCFA.cmake b/cmake_modules/FindCFA.cmake deleted file mode 100644 index 0461177..0000000 --- a/cmake_modules/FindCFA.cmake +++ /dev/null @@ -1,23 +0,0 @@ -FIND_PATH(CFA_INCLUDE_DIR cf-algorithms.h PATHS /usr/include/cf-algorithms /usr/local/include/cf-algorithms /opt/local/include/cf-algorithms) - -FIND_LIBRARY(CFA NAMES cf-algorithms PATHS /usr/lib /usr/local/lib /opt/local/lib) - -IF (CFA) - SET(CFA_LIBRARIES ${CFA}) -ELSE (CFA) - SET(CFA_LIBRARIES "NOTFOUND") -ENDIF(CFA) - -IF (CFA_INCLUDE_DIR AND CFA_LIBRARIES) - SET(CFA_FOUND TRUE) -ENDIF (CFA_INCLUDE_DIR AND CFA_LIBRARIES) - -IF (CFA_FOUND) - IF (NOT CFA_FIND_QUIETLY) - MESSAGE(STATUS "Found CFA: ${CFA_LIBRARIES}") - ENDIF (NOT CFA_FIND_QUIETLY) -ELSE (CFA_FOUND) - IF (CFA_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find CFA") - ENDIF (CFA_FIND_REQUIRED) -ENDIF (CFA_FOUND) \ No newline at end of file diff --git a/cmake_modules/FindFLANN.cmake b/cmake_modules/FindFLANN.cmake index b36ea3d..92ff629 100644 --- a/cmake_modules/FindFLANN.cmake +++ b/cmake_modules/FindFLANN.cmake @@ -5,17 +5,15 @@ FIND_LIBRARY(LZ4 NAMES lz4 PATHS /usr/lib /usr/local/lib /opt/local/lib) IF (FLANN_INCLUDE_DIR AND FLANN) SET(FLANN_LIBRARIES ${FLANN}) SET(FLANN_FOUND TRUE) -ELSE (FLANN_INCLUDE_DIR AND FLANN) +ELSE () SET(FLANN_LIBRARIES "NOTFOUND") SET(FLANN_FOUND FALSE) -ENDIF(FLANN_INCLUDE_DIR AND FLANN) +ENDIF () -IF (FLANN_FOUND) - IF (NOT FLANN_FIND_QUIETLY) - MESSAGE(STATUS "Found FLANN: ${FLANN_LIBRARIES}") - ENDIF (NOT FLANN_FIND_QUIETLY) -ELSE (FLANN_FOUND) +IF (NOT FLANN_FOUND) IF (FLANN_FIND_REQUIRED) MESSAGE(FATAL_ERROR "Could not find FLANN") - ENDIF (FLANN_FIND_REQUIRED) -ENDIF (FLANN_FOUND) \ No newline at end of file + ENDIF () +ENDIF () + +MARK_AS_ADVANCED(FLANN_INCLUDE_DIR FLANN_LIBRARIES) \ No newline at end of file diff --git a/cmake_modules/FindHDF5.cmake b/cmake_modules/FindHDF5.cmake index 6ca8ef3..eab30cf 100644 --- a/cmake_modules/FindHDF5.cmake +++ b/cmake_modules/FindHDF5.cmake @@ -13,29 +13,25 @@ ELSEIF (HDF5_ROOT) FIND_PATH(HDF5_INCLUDE_DIR hdf5.h PATHS ${HDF5_ROOT}) FIND_LIBRARY(HDF5 NAMES hdf5 PATHS ${HDF5_ROOT}) FIND_LIBRARY(HDF5_HL NAMES hdf5_hl PATHS ${HDF5_ROOT}) -ELSE() +ELSE () MESSAGE(STATUS "Looking for HDF5 in standard locations") FIND_PATH(HDF5_INCLUDE_DIR hdf5.h PATHS /usr/include /usr/local/include) FIND_LIBRARY(HDF5 NAMES hdf5 PATHS /usr/lib /usr/local/lib) FIND_LIBRARY(HDF5_HL NAMES hdf5_hl /usr/lib /usr/local/lib) -ENDIF() +ENDIF () IF (HDF5 AND HDF5_HL) SET(HDF5_LIBRARIES ${HDF5} ${HDF5_HL}) -ELSE (HDF5 AND HDF5_HL) +ELSE () SET(HDF5_LIBRARIES "NOTFOUND") -ENDIF(HDF5 AND HDF5_HL) +ENDIF () IF (HDF5_INCLUDE_DIR AND HDF5_LIBRARIES) SET(HDF5_FOUND TRUE) -ENDIF (HDF5_INCLUDE_DIR AND HDF5_LIBRARIES) +ENDIF () -IF (HDF5_FOUND) - IF (NOT HDF5_FIND_QUIETLY) - MESSAGE(STATUS "Found HDF5: ${HDF5_LIBRARIES}") - ENDIF (NOT HDF5_FIND_QUIETLY) -ELSE (HDF5_FOUND) - IF (HDF5_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find HDF5") - ENDIF (HDF5_FIND_REQUIRED) -ENDIF (HDF5_FOUND) +IF (NOT HDF5_FOUND AND HDF5_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Could not find HDF5") +ENDIF () + +MARK_AS_ADVANCED(HDF5_INCLUDE_DIR HDF5_LIBRARIES) \ No newline at end of file diff --git a/cmake_modules/FindKDTREE.cmake b/cmake_modules/FindKDTREE.cmake deleted file mode 100644 index 2445621..0000000 --- a/cmake_modules/FindKDTREE.cmake +++ /dev/null @@ -1,19 +0,0 @@ -FIND_PATH(KDTREE_PATH kdtree++ PATHS /usr /opt /var) - -IF (KDTREE_PATH) - SET(KDTREE_INCLUDE_DIR ${KDTREE_PATH}) - SET(KDTREE_FOUND TRUE) -ELSE (KDTREE_PATH) - SET(KDTREE_INCLUDE_DIR "NOTFOUND") - SET(KDTREE_FOUND FALSE) -ENDIF(KDTREE_PATH) - -IF (KDTREE_FOUND) - IF (NOT KDTREE_FIND_QUIETLY) - MESSAGE(STATUS "Found KDTREE in: ${KDTREE_PATH}") - ENDIF (NOT KDTREE_FIND_QUIETLY) -ELSE (KDTREE_FOUND) - IF (KDTREE_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find KDTREE") - ENDIF (KDTREE_FIND_REQUIRED) -ENDIF (KDTREE_FOUND) \ No newline at end of file diff --git a/cmake_modules/FindNETCDF.cmake b/cmake_modules/FindNETCDF.cmake index 7e015fb..d269acc 100644 --- a/cmake_modules/FindNETCDF.cmake +++ b/cmake_modules/FindNETCDF.cmake @@ -1,30 +1,26 @@ -IF(CMAKE_SYSTEM_NAME MATCHES Linux) +IF (CMAKE_SYSTEM_NAME MATCHES Linux) MESSAGE(STATUS "looking for libnetcdf and libnetcdf_c++4 here: ${LPATH}") FIND_PATH(NETCDF_INCLUDE_DIR netcdf.h PATHS /usr/include /usr/local/include /opt/local/include) FIND_LIBRARY(NETCDF NAMES netcdf PATHS ${LPATH}) FIND_LIBRARY(NETCDF_CPP NAMES netcdf_c++4 PATHS ${LPATH}) -ELSE(CMAKE_SYSTEM_NAME MATCHES Linux) +ELSE () FIND_PATH(NETCDF_INCLUDE_DIR netcdf PATHS /usr/include /usr/local/include /opt/local/include) FIND_LIBRARY(NETCDF NAMES netcdf PATHS /usr/lib /usr/local/lib /opt/local/lib) FIND_LIBRARY(NETCDF_CPP NAMES netcdf_c++4 PATHS /usr/lib /usr/local/lib /opt/local/lib) -ENDIF(CMAKE_SYSTEM_NAME MATCHES Linux) +ENDIF () IF (NETCDF AND NETCDF_CPP) SET(NETCDF_LIBRARIES ${NETCDF_CPP} ${NETCDF}) -ELSE (NETCDF AND NETCDF_CPP) +ELSE () SET(NETCDF_LIBRARIES "NOTFOUND") -ENDIF(NETCDF AND NETCDF_CPP) +ENDIF() IF (NETCDF_INCLUDE_DIR AND NETCDF_LIBRARIES) SET(NETCDF_FOUND TRUE) ENDIF (NETCDF_INCLUDE_DIR AND NETCDF_LIBRARIES) -IF (NETCDF_FOUND) - IF (NOT NETCDF_FIND_QUIETLY) - MESSAGE(STATUS "Found NETCDF: ${NETCDF_LIBRARIES}") - ENDIF (NOT NETCDF_FIND_QUIETLY) -ELSE (NETCDF_FOUND) - IF (NETCDF_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find NETCDF") - ENDIF (NETCDF_FIND_REQUIRED) -ENDIF (NETCDF_FOUND) +IF (NOT NETCDF_FOUND AND NETCDF_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Could not find NETCDF") +ENDIF () + +MARK_AS_ADVANCED(NETCDF_INCLUDE_DIR NETCDF_LIBRARIES) \ No newline at end of file diff --git a/cmake_modules/FindNanoFLANN.cmake b/cmake_modules/FindNanoFLANN.cmake index bda5652..a124e91 100644 --- a/cmake_modules/FindNanoFLANN.cmake +++ b/cmake_modules/FindNanoFLANN.cmake @@ -3,17 +3,13 @@ FIND_PATH(NanoFLANN_PATH nanoflann.hpp PATHS /usr /opt /var) IF (NanoFLANN_PATH) SET(NanoFLANN_INCLUDE_DIR ${NanoFLANN_PATH}) SET(NanoFLANN_FOUND TRUE) -ELSE (NanoFLANN_PATH) +ELSE () SET(NanoFLANN_INCLUDE_DIR "NOTFOUND") SET(NanoFLANN_FOUND FALSE) -ENDIF(NanoFLANN_PATH) +ENDIF () -IF (NanoFLANN_FOUND) - IF (NOT NanoFLANN_FIND_QUIETLY) - MESSAGE(STATUS "Found NanoFLANN in: ${NanoFLANN_PATH}") - ENDIF (NOT NanoFLANN_FIND_QUIETLY) -ELSE (NanoFLANN_FOUND) - IF (NanoFLANN_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find NanoFLANN") - ENDIF (NanoFLANN_FIND_REQUIRED) -ENDIF (NanoFLANN_FOUND) \ No newline at end of file +IF (NOT NanoFLANN_FOUND AMD NanoFLANN_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Could not find NanoFLANN") +ENDIF () + +MARK_AS_ADVANCED(NanoFLANN_INCLUDE_DIR) \ No newline at end of file diff --git a/cmake_modules/FindOpenMP_RT.cmake b/cmake_modules/FindOpenMP_RT.cmake deleted file mode 100644 index d894e19..0000000 --- a/cmake_modules/FindOpenMP_RT.cmake +++ /dev/null @@ -1,24 +0,0 @@ -FIND_PATH(OpenMP_RT_INCLUDE_DIRS omp.h PATHS /usr /usr/local /opt/local) -FIND_LIBRARY(OpenMP_RT NAMES iomp5 PATHS /usr /usr/local /opt/local) - -IF (OpenMP_RT) - SET(OpenMP_RT_LIBRARIES ${OpenMP_RT}) -ELSE (OpenMP_RT) - SET(OpenMP_RT_LIBRARIES "NOTFOUND") - SET(OpenMP_RT_FOUND FALSE) -ENDIF(OpenMP_RT) - -IF (OpenMP_RT_INCLUDE_DIRS AND OpenMP_RT_LIBRARIES) - SET(OpenMP_RT_FOUND TRUE) -ELSE(OpenMP_RT_INCLUDE_DIRS AND OpenMP_RT_LIBRARIES) - MESSAGE(ERROR "--> Could not locate OpenMP runtime") - MESSAGE(ERROR "--> OpenMP_RT_LIBRARIES=${OpenMP_RT_LIBRARIES}") - MESSAGE(ERROR "--> OpenMP_RT_INCLUDE_DIRS=${OpenMP_RT_INCLUDE_DIRS}") - SET(OpenMP_RT_FOUND FALSE) -ENDIF (OpenMP_RT_INCLUDE_DIRS AND OpenMP_RT_LIBRARIES) - -IF (OpenMP_RT_FOUND) - IF (NOT OpenMP_RT_FIND_QUIETLY) - MESSAGE(STATUS "Found OpenMP_RT: ${OpenMP_RT_INCLUDE_DIRS}; ${OpenMP_RT_LIBRARIES}") - ENDIF (NOT OpenMP_RT_FIND_QUIETLY) -ENDIF (OpenMP_RT_FOUND) \ No newline at end of file diff --git a/cmake_modules/FindRADOLAN.cmake b/cmake_modules/FindRADOLAN.cmake deleted file mode 100644 index 7b97543..0000000 --- a/cmake_modules/FindRADOLAN.cmake +++ /dev/null @@ -1,23 +0,0 @@ -FIND_PATH(RADOLAN_INCLUDE_DIR radolan.h PATHS /usr/include/radolan /usr/local/include/radolan /opt/local/include/radolan) - -FIND_LIBRARY(RADOLAN NAMES radolan PATHS /usr/lib /usr/local/lib /opt/local/lib) - -IF (RADOLAN) - SET(RADOLAN_LIBRARIES ${RADOLAN}) -ELSE (RADOLAN) - SET(RADOLAN_LIBRARIES "NOTFOUND") -ENDIF(RADOLAN) - -IF (RADOLAN_INCLUDE_DIR AND RADOLAN_LIBRARIES) - SET(RADOLAN_FOUND TRUE) -ENDIF (RADOLAN_INCLUDE_DIR AND RADOLAN_LIBRARIES) - -IF (RADOLAN_FOUND) - IF (NOT RADOLAN_FIND_QUIETLY) - MESSAGE(STATUS "Found RADOLAN: ${RADOLAN_LIBRARIES}") - ENDIF (NOT RADOLAN_FIND_QUIETLY) -ELSE (RADOLAN_FOUND) - IF (RADOLAN_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find RADOLAN") - ENDIF (RADOLAN_FIND_REQUIRED) -ENDIF (RADOLAN_FOUND) \ No newline at end of file diff --git a/cmake_modules/FindSHP.cmake b/cmake_modules/FindSHP.cmake index dc5690b..286abcb 100755 --- a/cmake_modules/FindSHP.cmake +++ b/cmake_modules/FindSHP.cmake @@ -3,20 +3,16 @@ FIND_LIBRARY(SHP NAMES shp PATHS /usr/lib /usr/local/lib /opt/local/lib ~/radola IF (SHP) SET(SHP_LIBRARIES ${SHP}) -ELSE (SHP) +ELSE () SET(SHP_LIBRARIES "") -ENDIF(SHP) +ENDIF () IF (SHP_INCLUDE_DIR AND SHP_LIBRARIES) SET(SHP_FOUND TRUE) -ENDIF (SHP_INCLUDE_DIR AND SHP_LIBRARIES) +ENDIF () -IF (SHP_FOUND) - IF (NOT SHP_FIND_QUIETLY) - MESSAGE(STATUS "Found SHP: ${SHP_LIBRARIES}") - ENDIF (NOT SHP_FIND_QUIETLY) -ELSE (SHP_FOUND) - IF (SHP_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find SHP") - ENDIF (SHP_FIND_REQUIRED) -ENDIF (SHP_FOUND) \ No newline at end of file +IF (NOT SHP_FOUND AND SHP_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Could not find SHP") +ENDIF () + +MARK_AS_ADVANCED(SHP_LIBRARIES SHP_INCLUDE_DIR) \ No newline at end of file diff --git a/cmake_modules/FindTBB.cmake b/cmake_modules/FindTBB.cmake deleted file mode 100644 index 53299a2..0000000 --- a/cmake_modules/FindTBB.cmake +++ /dev/null @@ -1,283 +0,0 @@ -# Locate Intel Threading Building Blocks include paths and libraries -# FindTBB.cmake can be found at https://code.google.com/p/findtbb/ -# Written by Hannes Hofmann -# Improvements by Gino van den Bergen , -# Florian Uhlig , -# Jiri Marsik - -# The MIT License -# -# Copyright (c) 2011 Hannes Hofmann -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -# GvdB: This module uses the environment variable TBB_ARCH_PLATFORM which defines architecture and compiler. -# e.g. "ia32/vc8" or "em64t/cc4.1.0_libc2.4_kernel2.6.16.21" -# TBB_ARCH_PLATFORM is set by the build script tbbvars[.bat|.sh|.csh], which can be found -# in the TBB installation directory (TBB_INSTALL_DIR). -# -# GvdB: Mac OS X distribution places libraries directly in lib directory. -# -# For backwards compatibility, you may explicitely set the CMake variables TBB_ARCHITECTURE and TBB_COMPILER. -# TBB_ARCHITECTURE [ ia32 | em64t | itanium ] -# which architecture to use -# TBB_COMPILER e.g. vc9 or cc3.2.3_libc2.3.2_kernel2.4.21 or cc4.0.1_os10.4.9 -# which compiler to use (detected automatically on Windows) - -# This module respects -# TBB_INSTALL_DIR or $ENV{TBB21_INSTALL_DIR} or $ENV{TBB_INSTALL_DIR} - -# This module defines -# TBB_INCLUDE_DIRS, where to find task_scheduler_init.h, etc. -# TBB_LIBRARY_DIRS, where to find libtbb, libtbbmalloc -# TBB_DEBUG_LIBRARY_DIRS, where to find libtbb_debug, libtbbmalloc_debug -# TBB_INSTALL_DIR, the base TBB install directory -# TBB_LIBRARIES, the libraries to link against to use TBB. -# TBB_DEBUG_LIBRARIES, the libraries to link against to use TBB with debug symbols. -# TBB_FOUND, If false, don't try to use TBB. -# TBB_INTERFACE_VERSION, as defined in tbb/tbb_stddef.h - - -if (WIN32) - # has em64t/vc8 em64t/vc9 - # has ia32/vc7.1 ia32/vc8 ia32/vc9 - set(_TBB_DEFAULT_INSTALL_DIR "C:/Program Files/Intel/TBB" "C:/Program Files (x86)/Intel/TBB") - set(_TBB_LIB_NAME "tbb") - set(_TBB_LIB_MALLOC_NAME "${_TBB_LIB_NAME}malloc") - set(_TBB_LIB_DEBUG_NAME "${_TBB_LIB_NAME}_debug") - set(_TBB_LIB_MALLOC_DEBUG_NAME "${_TBB_LIB_MALLOC_NAME}_debug") - if (MSVC71) - set (_TBB_COMPILER "vc7.1") - endif(MSVC71) - if (MSVC80) - set(_TBB_COMPILER "vc8") - endif(MSVC80) - if (MSVC90) - set(_TBB_COMPILER "vc9") - endif(MSVC90) - if(MSVC10) - set(_TBB_COMPILER "vc10") - endif(MSVC10) - # Todo: add other Windows compilers such as ICL. - set(_TBB_ARCHITECTURE ${TBB_ARCHITECTURE}) -endif (WIN32) - -if (UNIX) - if (APPLE) - # MAC - set(_TBB_DEFAULT_INSTALL_DIR "/Library/Frameworks/Intel_TBB.framework/Versions") - # libs: libtbb.dylib, libtbbmalloc.dylib, *_debug - set(_TBB_LIB_NAME "tbb") - set(_TBB_LIB_MALLOC_NAME "${_TBB_LIB_NAME}malloc") - set(_TBB_LIB_DEBUG_NAME "${_TBB_LIB_NAME}_debug") - set(_TBB_LIB_MALLOC_DEBUG_NAME "${_TBB_LIB_MALLOC_NAME}_debug") - # default flavor on apple: ia32/cc4.0.1_os10.4.9 - # Jiri: There is no reason to presume there is only one flavor and - # that user's setting of variables should be ignored. - if(NOT TBB_COMPILER) - set(_TBB_COMPILER "cc4.0.1_os10.4.9") - elseif (NOT TBB_COMPILER) - set(_TBB_COMPILER ${TBB_COMPILER}) - endif(NOT TBB_COMPILER) - if(NOT TBB_ARCHITECTURE) - set(_TBB_ARCHITECTURE "ia32") - elseif(NOT TBB_ARCHITECTURE) - set(_TBB_ARCHITECTURE ${TBB_ARCHITECTURE}) - endif(NOT TBB_ARCHITECTURE) - else (APPLE) - # LINUX - set(_TBB_DEFAULT_INSTALL_DIR "/opt/intel/tbb" "/usr/local/include" "/usr/include") - set(_TBB_LIB_NAME "tbb") - set(_TBB_LIB_MALLOC_NAME "${_TBB_LIB_NAME}malloc") - set(_TBB_LIB_DEBUG_NAME "${_TBB_LIB_NAME}_debug") - set(_TBB_LIB_MALLOC_DEBUG_NAME "${_TBB_LIB_MALLOC_NAME}_debug") - # has em64t/cc3.2.3_libc2.3.2_kernel2.4.21 em64t/cc3.3.3_libc2.3.3_kernel2.6.5 em64t/cc3.4.3_libc2.3.4_kernel2.6.9 em64t/cc4.1.0_libc2.4_kernel2.6.16.21 - # has ia32/* - # has itanium/* - set(_TBB_COMPILER ${TBB_COMPILER}) - set(_TBB_ARCHITECTURE ${TBB_ARCHITECTURE}) - endif (APPLE) -endif (UNIX) - -if (CMAKE_SYSTEM MATCHES "SunOS.*") -# SUN -# not yet supported -# has em64t/cc3.4.3_kernel5.10 -# has ia32/* -endif (CMAKE_SYSTEM MATCHES "SunOS.*") - - -#-- Clear the public variables -set (TBB_FOUND "NO") - - -#-- Find TBB install dir and set ${_TBB_INSTALL_DIR} and cached ${TBB_INSTALL_DIR} -# first: use CMake variable TBB_INSTALL_DIR -if (TBB_INSTALL_DIR) - set (_TBB_INSTALL_DIR ${TBB_INSTALL_DIR}) -endif (TBB_INSTALL_DIR) -# second: use environment variable -if (NOT _TBB_INSTALL_DIR) - if (NOT "$ENV{TBB_INSTALL_DIR}" STREQUAL "") - set (_TBB_INSTALL_DIR $ENV{TBB_INSTALL_DIR}) - endif (NOT "$ENV{TBB_INSTALL_DIR}" STREQUAL "") - # Intel recommends setting TBB21_INSTALL_DIR - if (NOT "$ENV{TBB21_INSTALL_DIR}" STREQUAL "") - set (_TBB_INSTALL_DIR $ENV{TBB21_INSTALL_DIR}) - endif (NOT "$ENV{TBB21_INSTALL_DIR}" STREQUAL "") - if (NOT "$ENV{TBB22_INSTALL_DIR}" STREQUAL "") - set (_TBB_INSTALL_DIR $ENV{TBB22_INSTALL_DIR}) - endif (NOT "$ENV{TBB22_INSTALL_DIR}" STREQUAL "") - if (NOT "$ENV{TBB30_INSTALL_DIR}" STREQUAL "") - set (_TBB_INSTALL_DIR $ENV{TBB30_INSTALL_DIR}) - endif (NOT "$ENV{TBB30_INSTALL_DIR}" STREQUAL "") -endif (NOT _TBB_INSTALL_DIR) -# third: try to find path automatically -if (NOT _TBB_INSTALL_DIR) - if (_TBB_DEFAULT_INSTALL_DIR) - set (_TBB_INSTALL_DIR ${_TBB_DEFAULT_INSTALL_DIR}) - endif (_TBB_DEFAULT_INSTALL_DIR) -endif (NOT _TBB_INSTALL_DIR) -# sanity check -if (NOT _TBB_INSTALL_DIR) - message ("ERROR: Unable to find Intel TBB install directory. ${_TBB_INSTALL_DIR}") -else (NOT _TBB_INSTALL_DIR) -# finally: set the cached CMake variable TBB_INSTALL_DIR -if (NOT TBB_INSTALL_DIR) - set (TBB_INSTALL_DIR ${_TBB_INSTALL_DIR} CACHE PATH "Intel TBB install directory") - mark_as_advanced(TBB_INSTALL_DIR) -endif (NOT TBB_INSTALL_DIR) - - -#-- A macro to rewrite the paths of the library. This is necessary, because -# find_library() always found the em64t/vc9 version of the TBB libs -macro(TBB_CORRECT_LIB_DIR var_name) -# if (NOT "${_TBB_ARCHITECTURE}" STREQUAL "em64t") - string(REPLACE em64t "${_TBB_ARCHITECTURE}" ${var_name} ${${var_name}}) -# endif (NOT "${_TBB_ARCHITECTURE}" STREQUAL "em64t") - string(REPLACE ia32 "${_TBB_ARCHITECTURE}" ${var_name} ${${var_name}}) - string(REPLACE vc7.1 "${_TBB_COMPILER}" ${var_name} ${${var_name}}) - string(REPLACE vc8 "${_TBB_COMPILER}" ${var_name} ${${var_name}}) - string(REPLACE vc9 "${_TBB_COMPILER}" ${var_name} ${${var_name}}) - string(REPLACE vc10 "${_TBB_COMPILER}" ${var_name} ${${var_name}}) -endmacro(TBB_CORRECT_LIB_DIR var_content) - - -#-- Look for include directory and set ${TBB_INCLUDE_DIR} -set (TBB_INC_SEARCH_DIR ${_TBB_INSTALL_DIR}/include) -# Jiri: tbbvars now sets the CPATH environment variable to the directory -# containing the headers. -find_path(TBB_INCLUDE_DIR - tbb/task_scheduler_init.h - PATHS ${TBB_INC_SEARCH_DIR} ENV CPATH -) -mark_as_advanced(TBB_INCLUDE_DIR) - - -#-- Look for libraries -# GvdB: $ENV{TBB_ARCH_PLATFORM} is set by the build script tbbvars[.bat|.sh|.csh] -if (NOT $ENV{TBB_ARCH_PLATFORM} STREQUAL "") - set (_TBB_LIBRARY_DIR - ${_TBB_INSTALL_DIR}/lib/$ENV{TBB_ARCH_PLATFORM} - ${_TBB_INSTALL_DIR}/$ENV{TBB_ARCH_PLATFORM}/lib - ) -endif (NOT $ENV{TBB_ARCH_PLATFORM} STREQUAL "") -# Jiri: This block isn't mutually exclusive with the previous one -# (hence no else), instead I test if the user really specified -# the variables in question. -if ((NOT ${TBB_ARCHITECTURE} STREQUAL "") AND (NOT ${TBB_COMPILER} STREQUAL "")) - # HH: deprecated - message(STATUS "[Warning] FindTBB.cmake: The use of TBB_ARCHITECTURE and TBB_COMPILER is deprecated and may not be supported in future versions. Please set \$ENV{TBB_ARCH_PLATFORM} (using tbbvars.[bat|csh|sh]).") - # Jiri: It doesn't hurt to look in more places, so I store the hints from - # ENV{TBB_ARCH_PLATFORM} and the TBB_ARCHITECTURE and TBB_COMPILER - # variables and search them both. - set (_TBB_LIBRARY_DIR "${_TBB_INSTALL_DIR}/${_TBB_ARCHITECTURE}/${_TBB_COMPILER}/lib" ${_TBB_LIBRARY_DIR}) -endif ((NOT ${TBB_ARCHITECTURE} STREQUAL "") AND (NOT ${TBB_COMPILER} STREQUAL "")) - -# GvdB: Mac OS X distribution places libraries directly in lib directory. -list(APPEND _TBB_LIBRARY_DIR ${_TBB_INSTALL_DIR}/lib) - -# Jiri: No reason not to check the default paths. From recent versions, -# tbbvars has started exporting the LIBRARY_PATH and LD_LIBRARY_PATH -# variables, which now point to the directories of the lib files. -# It all makes more sense to use the ${_TBB_LIBRARY_DIR} as a HINTS -# argument instead of the implicit PATHS as it isn't hard-coded -# but computed by system introspection. Searching the LIBRARY_PATH -# and LD_LIBRARY_PATH environment variables is now even more important -# that tbbvars doesn't export TBB_ARCH_PLATFORM and it facilitates -# the use of TBB built from sources. -find_library(TBB_LIBRARY ${_TBB_LIB_NAME} HINTS ${_TBB_LIBRARY_DIR} - PATHS ENV LIBRARY_PATH ENV LD_LIBRARY_PATH) -find_library(TBB_MALLOC_LIBRARY ${_TBB_LIB_MALLOC_NAME} HINTS ${_TBB_LIBRARY_DIR} - PATHS ENV LIBRARY_PATH ENV LD_LIBRARY_PATH) - -#Extract path from TBB_LIBRARY name -get_filename_component(TBB_LIBRARY_DIR ${TBB_LIBRARY} PATH) - -#TBB_CORRECT_LIB_DIR(TBB_LIBRARY) -#TBB_CORRECT_LIB_DIR(TBB_MALLOC_LIBRARY) -mark_as_advanced(TBB_LIBRARY TBB_MALLOC_LIBRARY) - -#-- Look for debug libraries -# Jiri: Changed the same way as for the release libraries. -find_library(TBB_LIBRARY_DEBUG ${_TBB_LIB_DEBUG_NAME} HINTS ${_TBB_LIBRARY_DIR} - PATHS ENV LIBRARY_PATH ENV LD_LIBRARY_PATH) -find_library(TBB_MALLOC_LIBRARY_DEBUG ${_TBB_LIB_MALLOC_DEBUG_NAME} HINTS ${_TBB_LIBRARY_DIR} - PATHS ENV LIBRARY_PATH ENV LD_LIBRARY_PATH) - -# Jiri: Self-built TBB stores the debug libraries in a separate directory. -# Extract path from TBB_LIBRARY_DEBUG name -get_filename_component(TBB_LIBRARY_DEBUG_DIR ${TBB_LIBRARY_DEBUG} PATH) - -#TBB_CORRECT_LIB_DIR(TBB_LIBRARY_DEBUG) -#TBB_CORRECT_LIB_DIR(TBB_MALLOC_LIBRARY_DEBUG) -mark_as_advanced(TBB_LIBRARY_DEBUG TBB_MALLOC_LIBRARY_DEBUG) - - -if (TBB_INCLUDE_DIR) - if (TBB_LIBRARY) - set (TBB_FOUND "YES") - set (TBB_LIBRARIES ${TBB_LIBRARY} ${TBB_MALLOC_LIBRARY} ${TBB_LIBRARIES}) - set (TBB_DEBUG_LIBRARIES ${TBB_LIBRARY_DEBUG} ${TBB_MALLOC_LIBRARY_DEBUG} ${TBB_DEBUG_LIBRARIES}) - set (TBB_INCLUDE_DIRS ${TBB_INCLUDE_DIR} CACHE PATH "TBB include directory" FORCE) - set (TBB_LIBRARY_DIRS ${TBB_LIBRARY_DIR} CACHE PATH "TBB library directory" FORCE) - # Jiri: Self-built TBB stores the debug libraries in a separate directory. - set (TBB_DEBUG_LIBRARY_DIRS ${TBB_LIBRARY_DEBUG_DIR} CACHE PATH "TBB debug library directory" FORCE) - mark_as_advanced(TBB_INCLUDE_DIRS TBB_LIBRARY_DIRS TBB_DEBUG_LIBRARY_DIRS TBB_LIBRARIES TBB_DEBUG_LIBRARIES) - message(STATUS "Found Intel TBB (LIBRARIES=${TBB_LIBRARIES})") - endif (TBB_LIBRARY) -endif (TBB_INCLUDE_DIR) - -if (NOT TBB_FOUND) - message("ERROR: Intel TBB NOT found!") - message(STATUS "Looked for Threading Building Blocks in ${_TBB_INSTALL_DIR}") - # do only throw fatal, if this pkg is REQUIRED - if (TBB_FIND_REQUIRED) - message(FATAL_ERROR "Could NOT find TBB library.") - endif (TBB_FIND_REQUIRED) -endif (NOT TBB_FOUND) - -endif (NOT _TBB_INSTALL_DIR) - -if (TBB_FOUND) - set(TBB_INTERFACE_VERSION 0) - FILE(READ "${TBB_INCLUDE_DIRS}/tbb/tbb_stddef.h" _TBB_VERSION_CONTENTS) - STRING(REGEX REPLACE ".*#define TBB_INTERFACE_VERSION ([0-9]+).*" "\\1" TBB_INTERFACE_VERSION "${_TBB_VERSION_CONTENTS}") - set(TBB_INTERFACE_VERSION "${TBB_INTERFACE_VERSION}") -endif (TBB_FOUND) \ No newline at end of file diff --git a/cmake_modules/Findlibradolan.cmake b/cmake_modules/Findlibradolan.cmake index 084a0e9..e87d07d 100644 --- a/cmake_modules/Findlibradolan.cmake +++ b/cmake_modules/Findlibradolan.cmake @@ -14,7 +14,7 @@ FIND_LIBRARY(libradolan_LIBRARY NAMES radolan) IF (libradolan_INCLUDE_DIR AND libradolan_LIBRARY) SET(libradolan_FOUND TRUE) -ENDIF (libradolan_INCLUDE_DIR AND libradolan_LIBRARY) +ENDIF () # handle the QUIETLY and REQUIRED arguments and set libradolan_FOUND to TRUE if # all listed variables are TRUE @@ -24,6 +24,8 @@ FIND_PACKAGE_HANDLE_STANDARD_ARGS(libradolan DEFAULT_MSG libradolan_LIBRARY libr IF(libradolan_FOUND) SET(libradolan_LIBRARIES ${libradolan_LIBRARY} ) SET(libradolan_INCLUDE_DIRS ${libradolan_INCLUDE_DIR} ) -ENDIF(libradolan_FOUND) +ELSEIF (libradolan_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "libradolan not found") +ENDIF() MARK_AS_ADVANCED(libradolan_INCLUDE_DIR libradolan_LIBRARY) \ No newline at end of file diff --git a/cmake_modules/Findosmpbf.cmake b/cmake_modules/Findosmpbf.cmake deleted file mode 100644 index fbc7367..0000000 --- a/cmake_modules/Findosmpbf.cmake +++ /dev/null @@ -1,20 +0,0 @@ -FIND_PATH(osmpbf_INCLUDE_DIR osmpbf PATHS /usr/include /usr/local/include/ opt/local/include) -FIND_LIBRARY(osmpbf NAMES osmpbf PATHS /usr/lib /usr/local/lib /opt/local/lib) - -IF (osmpbf_INCLUDE_DIR AND osmpbf) - SET(osmpbf_LIBRARIES ${osmpbf}) - SET(osmpbf_FOUND TRUE) -ELSE (osmpbf_INCLUDE_DIR AND osmpbf) - SET(osmpbf_LIBRARIES "NOTFOUND") - SET(osmpbf_FOUND FALSE) -ENDIF(osmpbf_INCLUDE_DIR AND osmpbf) - -IF (osmpbf_FOUND) - IF (NOT osmpbf_FIND_QUIETLY) - MESSAGE(STATUS "Found osmpbf: ${osmpbf_LIBRARIES}") - ENDIF (NOT osmpbf_FIND_QUIETLY) -ELSE (osmpbf_FOUND) - IF (osmpbf_FIND_REQUIRED) - MESSAGE(FATAL_ERROR "Could not find osmpbf") - ENDIF (osmpbf_FIND_REQUIRED) -ENDIF (osmpbf_FOUND) \ No newline at end of file From fdbc276a1ec16925d6f19764eb1ef4e010597082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Wed, 24 Jun 2020 13:24:32 +0100 Subject: [PATCH 44/65] Fixed import problem --- include/meanie3D/namespaces.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/meanie3D/namespaces.h b/include/meanie3D/namespaces.h index 49dab5b..9551e57 100644 --- a/include/meanie3D/namespaces.h +++ b/include/meanie3D/namespaces.h @@ -24,10 +24,6 @@ #ifndef M3D_NAMESPACE_DEFINITIONS_H #define M3D_NAMESPACE_DEFINITIONS_H -#include // namespace std -#include // namespace netCDF -#include // namespace boost; - // This file manages the imports of namespaces into the // various namespaces defined in this software. // That way all dependencies can be changed in one place From 55c6b1c732d64336359afaf864cc171dd0e93221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Wed, 24 Jun 2020 13:25:20 +0100 Subject: [PATCH 45/65] Workaround for the odd boost::lexical_cast problem with numbers in scientific notation --- include/meanie3D/utils/vector_utils.h | 13 ++++++++++++- src/executables/meanie3D-cfm2vtk.cpp | 26 ++++++++++++++++---------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/include/meanie3D/utils/vector_utils.h b/include/meanie3D/utils/vector_utils.h index b53ff02..fc4f0d5 100644 --- a/include/meanie3D/utils/vector_utils.h +++ b/include/meanie3D/utils/vector_utils.h @@ -394,7 +394,18 @@ namespace m3D { vector mode; for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) { std::string token = *tok_iter; - result.push_back(boost::lexical_cast(token)); + T value; + try { + value = boost::lexical_cast(token); + result.push_back(value); + } + catch (std::exception &e) + { + std::cerr << "Could not convert token " << token << std::endl; + // NOTE: this is a workaround for the problem described in + // https: //stackoverflow.com/questions/62553744/why-does-boostlexical-cast-throw-an-exception-even-though-it-converted-the-val + result.push_back(value); + } } return result; diff --git a/src/executables/meanie3D-cfm2vtk.cpp b/src/executables/meanie3D-cfm2vtk.cpp index f2519f2..ca06e7a 100644 --- a/src/executables/meanie3D-cfm2vtk.cpp +++ b/src/executables/meanie3D-cfm2vtk.cpp @@ -26,12 +26,13 @@ using namespace boost; using namespace m3D; +using namespace m3D::utils; using namespace m3D::utils::vectors; using namespace netCDF; using namespace std; /** Feature-space data type */ -typedef double FS_TYPE; +#define FS_TYPE double /** Verbosity */ typedef enum { @@ -172,10 +173,10 @@ void convert_clusters(const string &filename, CoordinateSystem *cs = NULL; ClusterList *list = ClusterList::read(filename, &cs); - //::m3D::utils::VisitUtils::write_clusters_vtr(list, cs, list->source_file, true, false, true); - // ::m3D::utils::VisitUtils::write_clusters_vtu(list, cs, list->source_file, 5, true, extract_skin, write_as_xml); + //VisitUtils::write_clusters_vtr(list, cs, list->source_file, true, false, true); + // VisitUtils::write_clusters_vtu(list, cs, list->source_file, 5, true, extract_skin, write_as_xml); - ::m3D::utils::VisitUtils::write_clusters_vtu(list, cs, list->source_file, 5, true, extract_skin, + VisitUtils::write_clusters_vtu(list, cs, list->source_file, 5, true, extract_skin, write_as_xml); if (write_displacement_vectors) { @@ -189,12 +190,12 @@ void convert_clusters(const string &filename, } } string displacements_path = path.filename().stem().string() + "-displacements.vtk"; - ::m3D::utils::VisitUtils::write_vectors_vtk(displacements_path, origins, displacements, + VisitUtils::write_vectors_vtk(displacements_path, origins, displacements, "displacement"); } string centers_path = path.filename().stem().string() + "-centers.vtk"; - ::m3D::utils::VisitUtils::write_geometrical_cluster_centers_vtk(centers_path, list->clusters); + VisitUtils::write_geometrical_cluster_centers_vtk(centers_path, list->clusters); delete list; delete cs; @@ -315,6 +316,10 @@ int main(int argc, char **argv) { bool write_displacement_vectors = false; bool mind_the_time = false; + // string problemStr = "1.03964e-312"; + // double problemChild = boost::lexical_cast(problemStr); + // cout << problemStr << " = " << problemChild << endl; + try { parse_commmandline(vm, filename, @@ -350,11 +355,12 @@ int main(int argc, char **argv) { } catch (const netCDF::exceptions::NcException &e) { cerr << "NetCDF exception:" << e.what() << endl; exit(EXIT_FAILURE);; - } catch (const std::exception &e) { - cerr << "std::exception:" << e.what() << endl; - cerr << e.what() << endl; - exit(EXIT_FAILURE);; } + // } catch (const std::exception &e) { + // cerr << "std::exception:" << e.what() << endl; + // cerr << e.what() << endl; + // exit(EXIT_FAILURE);; + // } return 0; }; From 2a57133856464d9b96affb353193967555b5a554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Wed, 24 Jun 2020 19:17:44 +0100 Subject: [PATCH 46/65] Formatting. --- include/meanie3D/clustering/cluster_op_impl.h | 55 +++++++------------ 1 file changed, 19 insertions(+), 36 deletions(-) diff --git a/include/meanie3D/clustering/cluster_op_impl.h b/include/meanie3D/clustering/cluster_op_impl.h index 758b62c..4f818de 100644 --- a/include/meanie3D/clustering/cluster_op_impl.h +++ b/include/meanie3D/clustering/cluster_op_impl.h @@ -39,24 +39,15 @@ namespace m3D #if WRITE_MODES static size_t s_pass_counter = 0; - template - size_t - ClusterOperation::pass_counter() - { + template size_t ClusterOperation::pass_counter() { return s_pass_counter; }; - template - void - ClusterOperation::reset_pass_counter() - { + template void ClusterOperation::reset_pass_counter() { s_pass_counter = 0; }; - template - void - ClusterOperation::increment_pass_counter() - { + template void ClusterOperation::increment_pass_counter() { s_pass_counter++; } #endif @@ -64,16 +55,11 @@ namespace m3D #pragma mark - #pragma mark Clustering Code - template - ClusterList * - ClusterOperation::cluster() - { + template ClusterList* ClusterOperation::cluster() { using namespace m3D::utils::vectors; - const CoordinateSystem *cs = m_context.coord_system; vector resolution; - if (m_context.search_params->search_type() == SearchTypeRange) - { + if (m_context.search_params->search_type() == SearchTypeRange) { RangeSearchParams *p = (RangeSearchParams *)m_context.search_params; // Physical grid resolution in the // spatial range @@ -81,21 +67,16 @@ namespace m3D resolution = ((T)4.0) * resolution; // Supplement with bandwidth values for // the value range - for (size_t i = resolution.size(); i < p->bandwidth.size(); i++) - { + for (size_t i = resolution.size(); i < p->bandwidth.size(); i++) { resolution.push_back(p->bandwidth[i]); } - } - else - { + } else { KNNSearchParams *p = (KNNSearchParams *)m_context.search_params; resolution = p->resolution; } - if (m_context.show_progress) - { - cout << endl - << "Creating meanshift vector graph ..."; + if (m_context.show_progress) { + cout << endl << "Creating meanshift vector graph ..."; start_timer(); m_progress_bar = new boost::progress_display(this->feature_space->size()); } @@ -109,8 +90,7 @@ namespace m3D m_params.time_index); // Guard against empty feature-space - if (this->feature_space->points.size() == 0) - { + if (this->feature_space->points.size() == 0) { cout << "Feature space is empty" << endl; return cluster_list; } @@ -121,10 +101,8 @@ namespace m3D #if WITH_OPENMP #pragma omp parallel for schedule(dynamic) #endif - for (size_t index = 0; index < this->feature_space->size(); index++) - { - if (m_context.show_progress) - { + for (size_t index = 0; index < this->feature_space->size(); index++) { + if (m_context.show_progress) { #if WITH_OPENMP #pragma omp critical #endif @@ -133,6 +111,12 @@ namespace m3D // Get the meanshift vector for this point typename Point::ptr x = this->feature_space->points[index]; + + // If the weight function indicates no significant response, skip + // this point + + + // Calculate the mean shift vector x->shift = meanshiftOperator.meanshift(x->values, m_context.search_params, m_context.kernel, @@ -143,8 +127,7 @@ namespace m3D x->gridded_shift = this->feature_space->coordinate_system->to_gridpoints(spatial_shift); } - if (m_context.show_progress) - { + if (m_context.show_progress) { cout << "done. (" << stop_timer() << "s)" << endl; delete m_progress_bar; m_progress_bar = NULL; From 151f5d3c0340dffdada6c8e3981df77bad3c6a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 29 Jun 2020 15:30:14 +0100 Subject: [PATCH 47/65] Polishing. Updating cmake to separate VTK and Visit. Updating documentation --- CMakeLists.txt | 96 +++++++++--- README.md | 144 +++++++++--------- include/meanie3D/clustering/cluster_op_impl.h | 1 + .../clustering/detection_commandline.h | 4 +- include/meanie3D/defines.h | 2 +- 5 files changed, 146 insertions(+), 101 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b73212..e0b5bd7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,32 +40,41 @@ CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/config_h.in CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/include/meanie3D/version.h.in ${PROJECT_SOURCE_DIR}/include/meanie3D/version.h) -# Set default cmake build type to Debug -# (None, Debug, Release, RelWithDebInfo or MinSizeRel) +# Project include directory +INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/include) +# ------------------------------------- +# Set default cmake build type to Release +# (None, Debug, Release, RelWithDebInfo or MinSizeRel) +# ------------------------------------- IF (NOT CMAKE_BUILD_TYPE) - SET(CMAKE_BUILD_TYPE "Debug") + SET(CMAKE_BUILD_TYPE "Release") ENDIF () -# Project include directory - -INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/include) - # ------------------------------------- # Options # ------------------------------------- OPTION(WITH_OPENMP "Enable/disable use of OpenMP for parallelization" ON) OPTION(WITH_TESTS "Enable/Disable building of tests" OFF) -OPTION(WITH_VTK "Enable/Disable visualisation code (requires VisIt and VTK)" OFF) +OPTION(WITH_VTK "Enable/Disable visualisation code (requires VTK)" OFF) +OPTION(WITH_VISUALISATION "Enable code that runs in Visit" OFF) OPTION(WITH_RADOLAN_UTILS "Compile with radolan adaptor (requires libradolan)" OFF) OPTION(WITH_SATELLITE_UTILS "Compile with tools for satellite data" OFF) OPTION(WITH_KONRAD_UTILS "Compile with binaries for KONRAD tracks" OFF) OPTION(WITH_DOCS "Compile source code documentation" OFF) -OPTION(WITH_PYTHON "Add the python frontend" ON) # ------------------------------------- # PRESETS # ------------------------------------- +SET(OPENMP_ENABLED ${WITH_OPENMP}) +SET(PYTHON_ENABLED ${WITH_PYTHON}) +SET(VTK_ENABLED ${WITH_VTK}) +SET(VISUALISATION_ENABLED ${WITH_VISUALISATION}) +SET(TESTS_ENABLED ${WITH_TESTS}) +SET(RADOLAN_ENABLED ${WITH_RADOLAN_UTILS}) +SET(KONRAD_ENABLED ${WITH_KONRAD_UTILS}) +SET(SATELLITE_ENABLED ${WITH_SATELLITE_UTILS}) +SET(DOCS_ENABLED ${WITH_DOCS}) IF(PRESET MATCHES "^docker$") MESSAGE(STATUS "Selected preset ${PRESET}") @@ -73,6 +82,7 @@ IF(PRESET MATCHES "^docker$") SET(OPENMP_ENABLED ON) SET(PYTHON_ENABLED ON) SET(VTK_ENABLED OFF) + SET(VISUALISATION_ENABLED OFF) SET(TESTS_ENABLED OFF) SET(RADOLAN_ENABLED OFF) SET(SATELLITE_ENABLED OFF) @@ -84,6 +94,7 @@ ELSEIF(PRESET MATCHES "^docker-vtk$") SET(OPENMP_ENABLED ON) SET(PYTHON_ENABLED ON) SET(VTK_ENABLED ON) + SET(VISUALISATION_ENABLED OFF) SET(TESTS_ENABLED OFF) SET(RADOLAN_ENABLED OFF) SET(SATELLITE_ENABLED OFF) @@ -95,6 +106,7 @@ ELSEIF (PRESET MATCHES "^dev-all$") SET(OPENMP_ENABLED ON) SET(PYTHON_ENABLED ON) SET(VTK_ENABLED ON) + SET(VISUALISATION_ENABLED ON) SET(TESTS_ENABLED ON) SET(RADOLAN_ENABLED ON) SET(SATELLITE_ENABLED ON) @@ -106,6 +118,7 @@ ELSEIF(PRESET MATCHES "^dev-core$") SET(OPENMP_ENABLED ON) SET(PYTHON_ENABLED ON) SET(VTK_ENABLED OFF) + SET(VISUALISATION_ENABLED OFF) SET(TESTS_ENABLED OFF) SET(RADOLAN_ENABLED OFF) SET(SATELLITE_ENABLED OFF) @@ -117,6 +130,7 @@ ELSEIF (PRESET MATCHES "^dev-vtk$") SET(OPENMP_ENABLED ON) SET(PYTHON_ENABLED ON) SET(VTK_ENABLED ON) + SET(VISUALISATION_ENABLED OFF) SET(TESTS_ENABLED OFF) SET(RADOLAN_ENABLED OFF) SET(SATELLITE_ENABLED OFF) @@ -128,6 +142,7 @@ ELSEIF (PRESET MATCHES "^dev-all$") SET(OPENMP_ENABLED ON) SET(PYTHON_ENABLED ON) SET(VTK_ENABLED ON) + SET(VISUALISATION_ENABLED ON) SET(TESTS_ENABLED ON) SET(RADOLAN_ENABLED ON) SET(SATELLITE_ENABLED ON) @@ -150,17 +165,31 @@ ELSEIF (PRESET MATCHES "^prod-vtk$") SET(OPENMP_ENABLED ON) SET(PYTHON_ENABLED ON) SET(VTK_ENABLED ON) + SET(VISUALISATION_ENABLED OFF) SET(TESTS_ENABLED OFF) SET(RADOLAN_ENABLED OFF) SET(SATELLITE_ENABLED OFF) SET(KONRAD_ENABLED OFF) SET(DOCS_ENABLED OFF) +ELSEIF (PRESET MATCHES "^prod-all$") + MESSAGE(STATUS "Selected preset ${PRESET}") + SET(CMAKE_BUILD_TYPE "Release") + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED ON) + SET(VISUALISATION_ENABLED ON) + SET(TESTS_ENABLED ON) + SET(RADOLAN_ENABLED ON) + SET(SATELLITE_ENABLED ON) + SET(KONRAD_ENABLED ON) + SET(DOCS_ENABLED ON) ELSEIF (PRESET MATCHES "^fast-core$") MESSAGE(STATUS "Selected preset ${PRESET}") SET(CMAKE_BUILD_TYPE "MinSizeRel") SET(OPENMP_ENABLED ON) SET(PYTHON_ENABLED ON) SET(VTK_ENABLED OFF) + SET(VISUALISATION_ENABLED OFF) SET(TESTS_ENABLED OFF) SET(RADOLAN_ENABLED OFF) SET(SATELLITE_ENABLED OFF) @@ -172,23 +201,24 @@ ELSEIF (PRESET MATCHES "^fast-vtk$") SET(OPENMP_ENABLED ON) SET(PYTHON_ENABLED ON) SET(VTK_ENABLED ON) + SET(VISUALISATION_ENABLED OFF) SET(TESTS_ENABLED OFF) SET(RADOLAN_ENABLED OFF) SET(SATELLITE_ENABLED OFF) SET(KONRAD_ENABLED OFF) SET(DOCS_ENABLED OFF) -ELSE() - IF (NOT ${CMAKE_BUILD_TYPE}) - SET(CMAKE_BUILD_TYPE "Release") - ENDIF() - SET(OPENMP_ENABLED ${WITH_OPENMP}) - SET(PYTHON_ENABLED ${WITH_PYTHON}) - SET(TESTS_ENABLED ${WITH_TESTS}) - SET(VTK_ENABLED ${WITH_VTK}) - SET(RADOLAN_ENABLED ${WITH_RADOLAN_UTILS}) - SET(KONRAD_ENABLED ${WITH_KONRAD_UTILS}) - SET(SATELLITE_ENABLED ${WITH_SATELLITE_UTILS}) - SET(DOCS_ENABLED ${WITH_DOCS}) +ELSEIF (PRESET MATCHES "^fast-all$") + MESSAGE(STATUS "Selected preset ${PRESET}") + SET(CMAKE_BUILD_TYPE "MinSizeRel") + SET(OPENMP_ENABLED ON) + SET(PYTHON_ENABLED ON) + SET(VTK_ENABLED ON) + SET(VISUALISATION_ENABLED ON) + SET(TESTS_ENABLED ON) + SET(RADOLAN_ENABLED ON) + SET(SATELLITE_ENABLED ON) + SET(KONRAD_ENABLED ON) + SET(DOCS_ENABLED ON) ENDIF() # ------------------------------------- @@ -217,15 +247,26 @@ ELSE () ENDIF () # ------------------------------------- -# Visualization on/off +# VTK output on/off # ------------------------------------- IF (VTK_ENABLED) - MESSAGE(STATUS "Visualization is enabled") + MESSAGE(STATUS "VTK output is enabled") ADD_DEFINITIONS(-DWITH_VTK=1) ELSE () MESSAGE(STATUS "Visualization is disabled") ENDIF () +# ------------------------------------- +# Visualization on/off +# ------------------------------------- +IF (VISUALISATION_ENABLED) + MESSAGE(STATUS "Visualisation code is enabled.") + ADD_DEFINITIONS(-DWITH_VISUALISATION=1) +ELSE () + MESSAGE(STATUS "Visualisation code is disabled.") +ENDIF () + + # ------------------------------------- # Automated tests yes/no # ------------------------------------- @@ -237,7 +278,7 @@ IF (TESTS_ENABLED) INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0) INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/gtest-1.7.0/include) ELSE () - MESSAGE(STATUS "Tests are disabled. Use -DWITH_TESTS=ON/OFF to switch tests on or off.") + MESSAGE(STATUS "Tests are disabled.") ENDIF () # ------------------------------------- @@ -388,6 +429,13 @@ IF (VTK_ENABLED) ENDIF () ENDIF () +# Visualization +IF (VISUALISATION_ENABLED) + FIND_PROGRAM(imagemagick REQUIRED) + FIND_PROGRAM(visit REQUIRED) + FIND_PROGRAM(gnuplot REQUIRED) +ENDIF() + FIND_PACKAGE(libradolan REQUIRED) IF (libradolan_FOUND) MESSAGE(STATUS "Found libradolan ${libradolan_INCLUDE_DIR}") diff --git a/README.md b/README.md index 43b491f..fac6e11 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Meanie3D uses OpenMP by default (-DWITH_OPENMP=1) and requires an OpenMP enabled You can disable the OpenMP implementation adding the flags -DWITH_OPENMP=0 to your cmake call. -### Build instructions +### Build and install the software Meanie3D uses CMAKE to generate makefiles. You can use CMAKE's abilities to generate IDE files if you prefer. Start by cloning the master branch (for an up-to date but possibly unstable version) or one of the stable releases. @@ -90,19 +90,23 @@ flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available pr * `docker` * Code optimizations for build type 'Release' * Core functions and python frontend +* `docker-vtk` + * Code optimizations for build type 'Release' + * Core functions and python frontend + * VTK output enabled. * `dev-core` * Code optimizations for build type 'Debug' * Core functions and python frontend * `dev-vtk` * Code optimizations for build type 'Debug' * Core functions and python frontend - * Visualization code - * Tests + * VTK output enabled. * Documentation * `dev-all` * Code optimizations for build type 'Debug' * Core functions and python frontend - * Visualization code + * VTK output enabled. + * Visualization enabled. * Tests * Documentation * RADOLAN, Satellite and KONRAD utilities @@ -112,21 +116,19 @@ flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available pr * `prod-vtk` * Code optimizations for build type 'Release' * Core functions and python frontend - * Visualization code + * VTK output enabled. * Tests * Documentation * `prod-all` * Code optimizations for build type 'Release' * Core functions and python frontend - * Visualization code + * VTK output enabled. + * Visualization enabled. * Tests * Documentation * RADOLAN, Satellite and KONRAD utilities -The term "core functions" refers to the detection, tracking and track evaluation code. For local -development in most cases, the preset `dev-vtk` is sufficient. For more aggressive optimisations, -use `prod-vtk`. If you just need the core functions and none of the visuals, choose the `core` sets -(`dev-core` or `prod-core`). +The term "core functions" refers to the detection, tracking and track evaluation code. For local development in most cases, the preset `dev-vtk` is sufficient. For more aggressive optimisations, use `prod-vtk`. If you just need the core functions and none of the visuals, choose the `core` sets (`dev-core` or `prod-core`). ### Available build types The following build types are available: @@ -140,26 +142,17 @@ To select a build type, use the flag `CMAKE_BUILD_TYPE` like so: cmake -DCMAKE_BUILD_TYPE=Release ../meanie3D ``` -*Note:* there have been some problems on Linux with aggressive optimization and NetCDF. Your mileage may vary. -You should try the release build in any event, since it speeds up performance a lot. If you observe unexpected -problems in reading/writing NetCDF files, you may have fallen victim to the problem and revert to standard build -(leave the -DCMAKE_BUILD_TYPE=Debug). Once all dependencies are successfully resolved, install the product by -calling the following: +*Note:* there have been some problems on Linux with aggressive optimization and NetCDF. Your mileage may vary. You should try the release build in any event, since it speeds up performance a lot. If you observe unexpected problems in reading/writing NetCDF files, you may have fallen victim to the problem and revert to standard build (leave the -DCMAKE_BUILD_TYPE=Debug). Once all dependencies are successfully resolved, install the product by calling the following: ``` make install ``` -*Note:* When using presets, the preset defines the build type. Setting this flag together with a preset has -no effect on the preset's choice. +*Note:* When using presets, the preset defines the build type. Setting this flag together with a preset has no effect on the preset's choice. ### Options -If you would like to select your own options, you can leave the PRESET parameter and switch things -on and off. When using presets, the preset defines the options. Setting any of the following options -together with a -DPRESET=\ option does not alter the preset's settings. - -#### -DWITH_VTK=ON/OFF -Because of the large footprint of the VTK package, the visualization code is disabled by default. While -visualization is not necessary to run the algorithm, it can be useful to develop your parameters to -have visual queues as to what is happening. Setting this flag will result in the following changes: +If you would like to select your own options, you can leave the PRESET parameter and switch things on and off. When using presets, the preset defines the options. Setting any of the following options together with a -DPRESET=\ option does not alter the preset's settings. + +#### `-DWITH_VTK=ON/OFF` +Because of the large footprint of the VTK package, the visualization code is disabled by default. While visualization is not necessary to run the algorithm, it can be useful to develop your parameters to have visual queues as to what is happening. Setting this flag will result in the following changes: * The `meanie3D-cfm2vtk` binary will be compiled. This tool can visualize netCDF files in Visit/VTK * In several places in the code, visualizable output for intermediary steps becomes available: * Cluster boundaries/outlines @@ -172,78 +165,81 @@ have visual queues as to what is happening. Setting this flag will result in the * Individual variables of the original netCDF (by cluster) * Cluster tracks -*Important Notes*: Visualization is switched off in the Docker version. The visualiation code uses -libradolan. If you do switch this on, you will be required to install libradolan as well. +#### `-DWITH_VISUALISATION=ON/OFF` +The meanie3D python program can utilize `Visit`, `gnuplot` and `imagemagick` to create images of the source data and results as part of batch processing. If this flag is set, CMAKE will make sure the necessary components are installed on your system. -#### -DWITH_OPENMP=ON/OFF -In order to speed the process up, meanie3D uses OpenMP to parallelize it's computation. This option -is switched on by default. +#### `-DWITH_OPENMP=ON/OFF` +In order to speed the process up, meanie3D uses OpenMP to parallelize it's computation. This option is switched on by default. -#### -D WITH_TESTS=ON/OFF -Meanie3D has a number of regression tests, that cover the core algorithms and collection classes. This will -become important to you if you should decide to work on the core algorithms yourself. The tests are a good -method of making sure you haven't broken anything critical. The unit tests can then be run by calling +#### `-DWITH_TESTS=ON/OFF` +Meanie3D has a number of regression tests, that cover the core algorithms and collection classes. This will become important to you if you should decide to work on the core algorithms yourself. The tests are a good method of making sure you haven't broken anything critical. The unit tests can then be run by calling ``` make test ``` -#### -DWITH_PYTHON=ON/OFF -In addition to the C++ binaries, the installation also uses pip to put a python package in place. -This adds an executable simply called `meanie3D`, which is a front-end to the core functions. It allows -you to put your clustering and tracking parameters down in the form of a configuration file. The entire -pipeline is handled based on this configuration file. *This is the recommended way to run the software*. -For details on the configuration file format, see HOWTO.md. - -#### -DWITH_DOCS=ON/OFF -In order to start developing your own Meanie3D code, it might be useful to have API documentation of the -various classes in the project. If you have doxygen installed, you can call the following make command to -create a browsable HTML documentation in doc/html (open the file index.html). +#### `-DWITH_PYTHON=ON/OFF` +In addition to the C++ binaries, the installation also uses pip to put a python package in place. This adds an executable simply called `meanie3D`, which is a front-end to the core functions. It allows you to put your clustering and tracking parameters down in the form of a configuration file. The entire +pipeline is handled based on this configuration file. *This is the recommended way to run the software*. For details on the configuration file format, please refer to the user manual. + +TODO: location of user manual + +#### `-DWITH_DOCS=ON/OFF` +In order to start developing your own Meanie3D code, it might be useful to have API documentation of the various classes in the project. If you have doxygen installed, you can call the following make command to create a browsable HTML documentation in doc/html (open the file index.html). ``` make docs ``` -#### -DWITH_RADOLAN_UTILS=ON/OFF -This will result in compilation of the `meanie3D-radolan2cfm` utility, which converts files in RADOLAN -format to a cf-metadata compliant netCDF file, which then can be used to run the tracking. +#### `-DWITH_RADOLAN_UTILS=ON/OFF` +This will result in compilation of the `meanie3D-radolan2cfm` utility, which converts files in RADOLAN format to a cf-metadata compliant netCDF file, which then can be used to run the tracking. -#### -DWITH_SATELLITE_UTILS=ON/OFF -The package comes with binaries to perform some conversion on satellite data. Those binaries were -provided in the context of research work for the OASE project. The following binaries will be provided -if this flag is set: +#### `-DWITH_SATELLITE_UTILS=ON/OFF` +The package comes with binaries to perform some conversion on satellite data. Those binaries were provided in the context of research work for the OASE project. The following binaries will be provided if this flag is set: * `meanie3D-satconv` - Converts spectral radiance to equivalent brightness temperature or vice versa * `meanie3D-parallax_correction` - Applies parallax correction to mseviri satellite data in OASE composite files. -#### -DWITH_KONRAD_UTILS=ON/OFF -The package comes with a tool `meanie3D-trackstats-conrad` which analyses KONRAD tracks in a way that makes -the data comparable to meanie3D data. This is a specialized tool developed in the context of the OASE project. +#### `-DWITH_KONRAD_UTILS=ON/OFF` +The package comes with a tool `meanie3D-trackstats-conrad` which analyses KONRAD tracks in a way that makes the data comparable to meanie3D data. This is a specialized tool developed in the context of the OASE project. + +### Additional options for debugging +In addition to runtime flags to the detection program, a few more options exist to write debug output. + +#### `-DDEBUG_GRAPH_AGGREGATION=1` +If this option is present, a blow by blow description of the algorithms decisions on aggregating clusters from mean-shift vectors is given to stdout. This can sometimes be helpful to understand how certain clusters were formed. + +#### `-DWRITE_FEATURESPACE=1` +The featurespace is the data set constructed from all dimensions plus the variables in question. If this flag is set, this data set is written out as VTK files at construction and throughout the different filtering steps. This information can be useful to understand choices in building your featurespace and in filtering it. NOTE: requires the code to be compiled with -DWITH_VTK=ON or an adequate preset. + +#### `-DWRITE_ZEROSHIFT_CLUSTERS=1` +Areas with mean-shift vectors that are zero after discretiziation are aggregated and serve as condensation points for the vector graph analysis. Those areas are called 'zero-shift' clusters. When this flag is set, those clusters are written to disk as .vtk files. NOTE: requires the code to be compiled with -DWITH_VTK=ON or an adequate preset. + +#### `-DWRITE_OFF_LIMITS_MASK=1` +The CF-Metadata standard for NetCDF files allows to have a value for areas outside of the valid measurements. This applies for example to values outside the maximum range of a radar etc. When this flag is set, a file is written out +in VTK format, that contains a mask for such areas (from the original data set). NOTE: requires the code to be compiled with -DWITH_VTK=ON or an adequate preset. + +#### `-DWRITE_MODES=1` +When this flag is set, a file is written out which contains the cluster modes. NOTE: requires the code to be compiled with -DWITH_VTK=ON or an adequate preset. + +### Uninstall the software +Navigate to the build directory. Invoke the following command: +``` +make uninstall +``` + +The command will remove the C++ binaries, meanie3D dynamic library and executables from your system. ## Frequently Asked Quesions ### I'm having trouble compiling the code -The code has been tested on MacOS X and Suse Linux. Feel free to report problems using the -"New Issue":http://git.meteo.uni-bonn.de/projects/meanie3d/issues/new link above. Please -prepend "[BUILD PROBLEM]:" to your subject line. +The code has been tested on MacOS X and Suse Linux. Feel free to report problems using the "New Issue":http://git.meteo.uni-bonn.de/projects/meanie3d/issues/new link above. Please prepend "[BUILD PROBLEM]:" to your subject line. ### NetCDF/HDF5 trouble -Versions - make sure the NetCDF version is 4.2 or better. Also make certain that you have a HDF5 -installation to match it. If you want to make sure, uninstall HDF5 and NetCDF and re-install them -from source (HDF5 first, then NetCDF) Commonly there is a problem with the NetCDF C++ API - NetCDF -has a legacy C++ API from NetCDF3, which does not work with Meanie3D. Make sure you have the correct -one installed (http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz) +Versions - make sure the NetCDF version is 4.2 or better. Also make certain that you have a HDF5 installation to match it. If you want to make sure, uninstall HDF5 and NetCDF and re-install them from source (HDF5 first, then NetCDF) Commonly there is a problem with the NetCDF C++ API - NetCDF has a legacy C++ API from NetCDF3, which does not work with Meanie3D. Make sure you have the correct one installed (http://www.unidata.ucar.edu/downloads/netcdf/ftp/netcdf-cxx4-4.2.tar.gz) ### CMAKE does not find your installed packages -If cmake has trouble locating packages you installed, check the files in the cmake_modules directory -and adjust the search paths there instead of hardwiring paths into the CMakeLists.txt. If your system -layout is not considered and you have to make adjustments to the search paths, please send a copy of -your Find\.cmake file to simon@webtecc.com so I can include your changes in the next release -and maybe save someone else the trouble. +If cmake has trouble locating packages you installed, check the files in the cmake_modules directory and adjust the search paths there instead of hardwiring paths into the CMakeLists.txt. If your system layout is not considered and you have to make adjustments to the search paths, please send a copy of your Find\.cmake file to simon@webtecc.com so I can include your changes in the next release and maybe save someone else the trouble. ## Visualisation -The C++ binaries produce certain files when VTK is enabled. In order to generate imagery or tracking -movies, meanie3D relies on executing python scripts in VisIt (). You will have to install this software -to get access to the visualization. For details on what is possible by scripting VisIt with python, -see the following tutorial: https://www.visitusers.org/index.php?title=VisIt-tutorial-Python-scripting +The C++ binaries produce certain files when VTK is enabled. In order to generate imagery or tracking movies, meanie3D relies on executing python scripts in VisIt (https://wci.llnl.gov/simulation/computer-codes/visit). You will have to install this software to get access to the visualization. For details on what is possible by scripting VisIt with python, see the following tutorial: https://www.visitusers.org/index.php?title=VisIt-tutorial-Python-scripting. ## More information -Check out the Main Wiki Page at http://git.meteo.uni-bonn.de/projects/meanie3d/wiki for details on how the -method works, file formats and what the individual binaries do. \ No newline at end of file +Check out the Main Wiki Page at http://git.meteo.uni-bonn.de/projects/meanie3d/wiki for details on how the method works, file formats and what the individual binaries do. \ No newline at end of file diff --git a/include/meanie3D/clustering/cluster_op_impl.h b/include/meanie3D/clustering/cluster_op_impl.h index 4f818de..69921de 100644 --- a/include/meanie3D/clustering/cluster_op_impl.h +++ b/include/meanie3D/clustering/cluster_op_impl.h @@ -114,6 +114,7 @@ namespace m3D // If the weight function indicates no significant response, skip // this point + // TODO: what happened to this code? // Calculate the mean shift vector diff --git a/include/meanie3D/clustering/detection_commandline.h b/include/meanie3D/clustering/detection_commandline.h index b26978d..1d7bbad 100644 --- a/include/meanie3D/clustering/detection_commandline.h +++ b/include/meanie3D/clustering/detection_commandline.h @@ -205,8 +205,8 @@ namespace m3D { params.dimensions.push_back(name); NcVar dimVar = file->getVar(name); if (dimVar.isNull()) { - cerr << "No dimension variable '" << std::string(name) << "' exists!" << endl; - exit(EXIT_FAILURE);; + cerr << "No dimension variable '" << std::string(name) << "' exists in file " << params.filename << endl; + exit(EXIT_FAILURE); } params.dimension_variables.push_back(name); } diff --git a/include/meanie3D/defines.h b/include/meanie3D/defines.h index e36a907..4f47fba 100644 --- a/include/meanie3D/defines.h +++ b/include/meanie3D/defines.h @@ -59,7 +59,7 @@ #define GRID_ROUNDING_METHOD_NONE 0 // ---------------------------------------------------- // -// Debugging Flags(stdout) +// Debugging Flags (stdout) // ---------------------------------------------------- // #define DEBUG_GRAPH_AGGREGATION 0 From 16ef13372f00fbf3351d420272ea6d75b30dbc55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 29 Jun 2020 16:09:55 +0100 Subject: [PATCH 48/65] Adjusted docker file for vtk. --- Dockerfile.vtk | 21 +++++++++++---------- Makefile.docker | 2 ++ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Dockerfile.vtk b/Dockerfile.vtk index dad8286..669e981 100644 --- a/Dockerfile.vtk +++ b/Dockerfile.vtk @@ -15,17 +15,18 @@ wget git cmake cpio \ gcc g++ libomp5 \ python python-pip \ libboost-all-dev libflann1.9 libflann-dev blitz++ \ -shapelib libhdf5-dev netcdf-bin libnetcdf-dev libnetcdf-c++4 libnetcdf-c++4-dev zlib1g zlib1g-dev +shapelib libhdf5-dev netcdf-bin libnetcdf-dev libnetcdf-c++4 libnetcdf-c++4-dev zlib1g zlib1g-dev \ +vtk7 libvtk7-dev RUN pip install setuptools external utils -# Visualisation -RUN pip install Cython h5py "netCDF4>=1.3.0<1.4.0" -RUN apt-get -y --fix-missing install gnuplot vtk7 libvtk7-dev -RUN wget --quiet ${VISIT_INSTALLER_URL} && wget --quiet ${VISIT_PKG_URL} && chmod u+x ./${VISIT_INSTALLER} -RUN rm -rf /usr/local/visit &&\ - mkdir -p /usr/local/visit/3.1.2+/linux-x86_64/bin &&\ - echo "1" | ./visit-install3_1_2 3.1.2 linux-x86_64-debian9 /usr/local/visit -ENV VISIT_EXECUTABLE=/usr/local/visit/bin/visit +# # Visualisation +# RUN pip install Cython h5py "netCDF4>=1.3.0<1.4.0" +# RUN apt-get -y --fix-missing install gnuplot vtk7 libvtk7-dev +# RUN wget --quiet ${VISIT_INSTALLER_URL} && wget --quiet ${VISIT_PKG_URL} && chmod u+x ./${VISIT_INSTALLER} +# RUN rm -rf /usr/local/visit &&\ +# mkdir -p /usr/local/visit/3.1.2+/linux-x86_64/bin &&\ +# echo "1" | ./visit-install3_1_2 3.1.2 linux-x86_64-debian9 /usr/local/visit +# ENV VISIT_EXECUTABLE=/usr/local/visit/bin/visit # libradolan RUN git clone https://github.com/JuergenSimon/radolan.git @@ -40,7 +41,7 @@ RUN git pull && cmake -DPRESET=docker-vtk . && make install # Cleanup WORKDIR / -RUN rm -rf meanie3D radolan visit* +RUN rm -rf meanie3D radolan RUN apt-get remove -y wget git cmake cpio RUN apt autoremove -y diff --git a/Makefile.docker b/Makefile.docker index cae0bdd..a923094 100644 --- a/Makefile.docker +++ b/Makefile.docker @@ -55,3 +55,5 @@ clean: all: docker push docker-vtk push-vtk .PHONY: all + + From ffe2bd9fad252f78e69eb655f0b84993b3fcbe8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 29 Jun 2020 16:10:05 +0100 Subject: [PATCH 49/65] Link formatting. --- README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index fac6e11..f97d524 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,13 @@ # meanie3D This project provides a very generic implementation of mean-shift clustering on NetCDF data sets. -The data sets need to follow the "CF-Metadata":http://cfconventions.org. The mean-shift algorithm +The data sets need to follow the [CF-Metadata](http://cfconventions.org). The mean-shift algorithm is a robust unsupervised clustering algorithm with a wide range of applications. Meanie3D provides a multivariate implementation with no limits as to the number of variables that can be used in constructing the Featurespace. This software supports 2D as well as 3D data sets and comes with many configuration options. The code -was written as part of my efforts within "OASE HErZ group":http://www.herz-tb1.uni-bonn.de and contains +was written as part of my efforts within [OASE HErZ group](http://www.herz-tb1.uni-bonn.de) and contains some modules that are specific to this application as a result. If interest is there I might remove all specific components at one point. Meanie3D is implemented largely as a C++ template library with only a few object files (mostly from the numerical recipes library) for performance reason and uses OpenMP for @@ -32,30 +32,30 @@ The latest stable version is v1.6.1. Versions are tagged. ### Dependencies Meanie3D comes with a number of dependencies that need to be installed prior to attempting installation: -* Boost 1.56 or better (http://www.boost.org) -* FLANN 1.8.0 or better (http://www.cs.ubc.ca/research/flann/) -* Blitz++ (http://sourceforge.net/projects/blitz/) -* OpenMP (libomp) (https://www.openmp.org) -* NetCDF 4.2 or better (http://www.unidata.ucar.edu/software/netcdf/) *including* the netcdf4-C++ API -* HDF5 (http://www.hdfgroup.org/HDF5/) -* Python 2.5 or better (https://www.python.org) -* numpy (try running @pip install numpy@ or download and install from http://www.numpy.org) +* [Boost 1.56 or better](http://www.boost.org) +* [FLANN 1.8.0 or better](http://www.cs.ubc.ca/research/flann/) +* [Blitz++](http://sourceforge.net/projects/blitz/) +* [OpenMP (libomp)](https://www.openmp.org) +* [NetCDF 4.2 or better](http://www.unidata.ucar.edu/software/netcdf/) *including* the netcdf4-C++ API +* [HDF5](http://www.hdfgroup.org/HDF5/) +* [Python 2.7](https://www.python.org) +* NumPY (try running @pip install numpy@ or download and install from http://www.numpy.org) * NetCDF4-python (try running @pip install netCDF4@ or download and install from http://unidata.github.io/netcdf4-python) -* libradolan (http://meteo-ubonn.github.io/radolan/) +* [libradolan](http://meteo-ubonn.github.io/radolan/) Optional: The following libraries may be used, if they are switched on with the appropriate flags * Data format - * shapelib v1.3+ (http://shapelib.maptools.org) - When present, certain utilities have additional options for outputting + * [shapelib v1.3+](http://shapelib.maptools.org) - When present, certain utilities have additional options for outputting data in ESRI shapefile format. * Visualization: - * Imagemagick (https://imagemagick.org) - * gnuplot (http://gnuplot.sourceforge.net) - * VTK 7.0 or better (http://www.vtk.org) - * Visit 3.0.0 or better (https://wci.llnl.gov/simulation/computer-codes/visit) + * [Imagemagick](https://imagemagick.org) + * [gnuplot](http://gnuplot.sourceforge.net) + * [VTK 7.0 or better](http://www.vtk.org) + * [Visit 3.0.0 or better](https://wci.llnl.gov/simulation/computer-codes/visit) * Generated source code documentation: - * doxygen (https://www.doxygen.nl/download.html) + * [doxygen](https://www.doxygen.nl/download.html) ### Compiler Prerequisites From 4605ce669246816276c8bb76a4e99f8bf3fc9021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 29 Jun 2020 16:18:07 +0100 Subject: [PATCH 50/65] Documentation update. --- README.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f97d524..7cc34fe 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,9 @@ The following libraries may be used, if they are switched on with the appropriat Meanie3D uses OpenMP by default (-DWITH_OPENMP=1) and requires an OpenMP enabled compiler, such as: * GNU 9.0 or better (Linux, Mac) -* OpenMP/LLVM (Mac) - a clang implementation supporting OpenMP (http://clang-omp.github.io). Note that as of clang 10.0.0 OpenMP is supported, but you will still have to install the omp library (`brew install libomp`) +* OpenMP/LLVM (Mac) - a clang implementation supporting OpenMP5 (http://clang-omp.github.io). Note that as of clang 10.0.0 OpenMP is supported out of the box, but you will still have to install the libomp library (`brew install libomp`) -You can disable the OpenMP implementation adding the flags -DWITH_OPENMP=0 to your cmake call. +You can disable the OpenMP implementation adding the flags `-DWITH_OPENMP=0` to your cmake call (does nothing when using presets). ### Build and install the software @@ -107,9 +107,9 @@ flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available pr * Core functions and python frontend * VTK output enabled. * Visualization enabled. + * RADOLAN, Satellite and KONRAD utilities * Tests * Documentation - * RADOLAN, Satellite and KONRAD utilities * `prod-core` * Code optimizations for build type 'Release' * Core functions and python frontend. @@ -117,16 +117,31 @@ flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available pr * Code optimizations for build type 'Release' * Core functions and python frontend * VTK output enabled. - * Tests - * Documentation * `prod-all` * Code optimizations for build type 'Release' * Core functions and python frontend * VTK output enabled. * Visualization enabled. + * RADOLAN, Satellite and KONRAD utilities * Tests * Documentation +* `fast-core` + * Code optimizations for build type 'MinSizeRel' + * Core functions and python frontend. +* `fast-vtk` + * Code optimizations for build type 'MinSizeRel' + * Core functions and python frontend + * VTK output enabled. + * Tests + * Documentation +* `fast-all` + * Code optimizations for build type 'MinSizeRel' + * Core functions and python frontend + * VTK output enabled. + * Visualization enabled. * RADOLAN, Satellite and KONRAD utilities + * Tests + * Documentation The term "core functions" refers to the detection, tracking and track evaluation code. For local development in most cases, the preset `dev-vtk` is sufficient. For more aggressive optimisations, use `prod-vtk`. If you just need the core functions and none of the visuals, choose the `core` sets (`dev-core` or `prod-core`). From d6a73603309882746abaa53e91aa5722eb82a07c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 29 Jun 2020 17:58:21 +0100 Subject: [PATCH 51/65] Debug flags now configurable. Readme updated. Fixed typo on meanie3D-cfm2vtk tool's command line (clusters -> cluster) --- CMakeLists.txt | 19 ++++++++++++++++++ README.md | 15 ++++++++------ include/meanie3D/defines.h | 28 +++++++++++++-------------- include/meanie3D/weights/ci_weights.h | 14 +++++++------- src/executables/meanie3D-cfm2vtk.cpp | 4 ++-- 5 files changed, 51 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e0b5bd7..801f021 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -320,6 +320,25 @@ IF (PYTHON_ENABLED) ADD_SUBDIRECTORY(python) ENDIF(PYTHON_ENABLED) +# ------------------------------------- +# Debug output flags +# ------------------------------------- +IF (DEBUG_GRAPH_AGGREGATION) + ADD_DEFINITIONS(-DDEBUG_GRAPH_AGGREGATION=1) +ENDIF() +IF (WRITE_ZEROSHIFT_CLUSTERS) + ADD_DEFINITIONS(-DWRITE_ZEROSHIFT_CLUSTERS=1) +ENDIF() +IF (WRITE_FEATURESPACE) + ADD_DEFINITIONS(-DWRITE_FEATURESPACE=1) +ENDIF() +IF (WRITE_OFF_LIMITS_MASK) + ADD_DEFINITIONS(-DWRITE_OFF_LIMITS_MASK=1) +ENDIF() +IF (WRITE_MODES) + ADD_DEFINITIONS(-DWRITE_MODES=1) +ENDIF() + # ------------------------------------- # Compiler Flags # ------------------------------------- diff --git a/README.md b/README.md index 7cc34fe..32657a5 100644 --- a/README.md +++ b/README.md @@ -218,20 +218,23 @@ The package comes with a tool `meanie3D-trackstats-conrad` which analyses KONRAD ### Additional options for debugging In addition to runtime flags to the detection program, a few more options exist to write debug output. -#### `-DDEBUG_GRAPH_AGGREGATION=1` -If this option is present, a blow by blow description of the algorithms decisions on aggregating clusters from mean-shift vectors is given to stdout. This can sometimes be helpful to understand how certain clusters were formed. +#### `-DDEBUG_GRAPH_AGGREGATION=ON` +If this option is selected, a blow by blow description of the algorithms decisions on aggregating clusters from mean-shift vectors is given to stdout. This can sometimes be helpful to understand how certain clusters were formed. -#### `-DWRITE_FEATURESPACE=1` +#### `-DWRITE_CI_SCORE=ON` +If this option is selected, details of the CI (convective initiation) score calculation are written out to VTK files. Tracking regions of elevated CI score was one of the ideas for the OASE group. Only useful when running the algorithm on SEVIRI satellite data and the flag `--ci-use-walker-mecikalski` is set on calling `meanie3D-detect`. + +#### `-DWRITE_FEATURESPACE=ON` The featurespace is the data set constructed from all dimensions plus the variables in question. If this flag is set, this data set is written out as VTK files at construction and throughout the different filtering steps. This information can be useful to understand choices in building your featurespace and in filtering it. NOTE: requires the code to be compiled with -DWITH_VTK=ON or an adequate preset. -#### `-DWRITE_ZEROSHIFT_CLUSTERS=1` +#### `-DWRITE_ZEROSHIFT_CLUSTERS=ON` Areas with mean-shift vectors that are zero after discretiziation are aggregated and serve as condensation points for the vector graph analysis. Those areas are called 'zero-shift' clusters. When this flag is set, those clusters are written to disk as .vtk files. NOTE: requires the code to be compiled with -DWITH_VTK=ON or an adequate preset. -#### `-DWRITE_OFF_LIMITS_MASK=1` +#### `-DWRITE_OFF_LIMITS_MASK=ON` The CF-Metadata standard for NetCDF files allows to have a value for areas outside of the valid measurements. This applies for example to values outside the maximum range of a radar etc. When this flag is set, a file is written out in VTK format, that contains a mask for such areas (from the original data set). NOTE: requires the code to be compiled with -DWITH_VTK=ON or an adequate preset. -#### `-DWRITE_MODES=1` +#### `-DWRITE_MODES=ON` When this flag is set, a file is written out which contains the cluster modes. NOTE: requires the code to be compiled with -DWITH_VTK=ON or an adequate preset. ### Uninstall the software diff --git a/include/meanie3D/defines.h b/include/meanie3D/defines.h index 4f47fba..7ea86aa 100644 --- a/include/meanie3D/defines.h +++ b/include/meanie3D/defines.h @@ -62,27 +62,27 @@ // Debugging Flags (stdout) // ---------------------------------------------------- // -#define DEBUG_GRAPH_AGGREGATION 0 -#define DEBUG_CI_SCORE 0 +// #define DEBUG_GRAPH_AGGREGATION 0 +// #define WRITE_CI_SCORE 0 // ---------------------------------------------------- // // Debugging Flags (files) // ---------------------------------------------------- // -// Write the feature-space out in .vtk file format -#define WRITE_FEATURESPACE 0 +// // Write the feature-space out in .vtk file format +// #define WRITE_FEATURESPACE 0 -// Write out modes found by the iterations or graph approach -#define WRITE_MODES 0 +// // Write out modes found by the iterations or graph approach +// #define WRITE_MODES 0 -// Write out a VTK file that contains 1 and 0 for -// all points. 1 means, that the point was marked -// as 'off limits' in the original feature-space -// (one of the variables outside of valid_range). -#define WRITE_OFF_LIMITS_MASK 0 +// // Write out a VTK file that contains 1 and 0 for +// // all points. 1 means, that the point was marked +// // as 'off limits' in the original feature-space +// // (one of the variables outside of valid_range). +// #define WRITE_OFF_LIMITS_MASK 0 -// Write out the result of the initial clustering -// step into separate files. -#define WRITE_ZEROSHIFT_CLUSTERS 0 +// // Write out the result of the initial clustering +// // step into separate files. +// #define WRITE_ZEROSHIFT_CLUSTERS 0 #endif diff --git a/include/meanie3D/weights/ci_weights.h b/include/meanie3D/weights/ci_weights.h index 65c801f..0672ed0 100644 --- a/include/meanie3D/weights/ci_weights.h +++ b/include/meanie3D/weights/ci_weights.h @@ -135,7 +135,7 @@ namespace m3D { vector m_bandwidth; // search radius for numerous operations SearchParameters *m_search_params; // search params for search -#if DEBUG_CI_SCORE +#if WRITE_CI_SCORE MultiArray *m_score_108; MultiArray *m_score_108_trend; MultiArray *m_score_62_108; @@ -599,7 +599,7 @@ namespace m3D { void calculate_weight_function(FeatureSpace *fs) { -#if DEBUG_CI_SCORE +#if WRITE_CI_SCORE vector dims = m_coordinate_system->get_dimension_sizes(); m_score_108 = new MultiArrayBlitz(dims, 1000); m_score_108_trend = new MultiArrayBlitz(dims, 1000); @@ -618,7 +618,7 @@ namespace m3D { } } -#if DEBUG_CI_SCORE +#if WRITE_CI_SCORE #if WITH_VTK boost::filesystem::path ppath(m_data_store->filename()); @@ -719,7 +719,7 @@ namespace m3D { T ir_108_radiance = this->m_data_store->get(msevi_l15_ir_108, g, isInRange, isValid); T ir_108_temp = brightness_temperature(msevi_l15_ir_108, ir_108_radiance); -#if DEBUG_CI_SCORE +#if WRITE_CI_SCORE m_score_108->set(g, ir_108_temp); #endif @@ -740,7 +740,7 @@ namespace m3D { // IR 0.65 - IR 10.7 T delta_wv_062_ir_108 = wv_062_temp - ir_108_temp; -#if DEBUG_CI_SCORE +#if WRITE_CI_SCORE m_score_62_108->set(g, delta_wv_062_ir_108); #endif @@ -755,7 +755,7 @@ namespace m3D { // IR 13.3 - IR 10.7 T delta_ir_134_ir_108 = ir_134_temp - ir_108_temp; -#if DEBUG_CI_SCORE +#if WRITE_CI_SCORE m_score_134_108->set(g, delta_ir_134_ir_108); #endif @@ -804,7 +804,7 @@ namespace m3D { if (dT3 >= 1.0) score++; } -#if DEBUG_CI_SCORE +#if WRITE_CI_SCORE m_score_108_trend->set(g, dT1); m_62_108_trend->set(g, dT2); m_134_108_trend->set(g, dT3); diff --git a/src/executables/meanie3D-cfm2vtk.cpp b/src/executables/meanie3D-cfm2vtk.cpp index ca06e7a..2a9c246 100644 --- a/src/executables/meanie3D-cfm2vtk.cpp +++ b/src/executables/meanie3D-cfm2vtk.cpp @@ -270,12 +270,12 @@ int main(int argc, char **argv) { ("variable,v", program_options::value(), "Name of the variable to be used") ("destination,d", program_options::value()->default_value("."), "Name of output directory for the converted files (default '.')") - ("type,t", program_options::value(), "'clusters' or 'composite'") + ("type,t", program_options::value(), "'cluster' or 'composite'") #if WITH_VTK ("extract-skin,s", "Use delaunay filter to extract skin file") ("write-as-xml,x", "Write files in xml instead of ascii") ("write-displacement-vectors", - "Write out an extra file containing the displacement vectors (clusters only).") + "Write out an extra file containing the displacement vectors (type cluster only).") ("vtk-dimensions", program_options::value(), "VTK files are written in the order of dimensions given. This may lead to wrong results if the order of the dimensions is not x,y,z. Add the comma-separated list of dimensions here, in the order you would like them to be written as (x,y,z)") #endif From 77545aee32416979860e21799375c9de5bd5cc9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 30 Jun 2020 14:52:22 +0100 Subject: [PATCH 52/65] Small python fixes around configuration handling. --- python/meanie3D/app/postprocessing.py | 2 +- python/meanie3D/meanie3D_run.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/meanie3D/app/postprocessing.py b/python/meanie3D/app/postprocessing.py index d3888a3..a1041ed 100644 --- a/python/meanie3D/app/postprocessing.py +++ b/python/meanie3D/app/postprocessing.py @@ -159,7 +159,7 @@ def run_trackstats(configuration, directory): params.append("--vtk-dimensions=%s" % conf['vtkDimensions']) params.append("-s netcdf") - if configuration['resume'] and haveFiles: + if 'resume' in configuration and haveFiles: print("Skipping meanie3D-trackstats") os.chdir("..") return True diff --git a/python/meanie3D/meanie3D_run.py b/python/meanie3D/meanie3D_run.py index 28ce5dc..0bdbd1c 100755 --- a/python/meanie3D/meanie3D_run.py +++ b/python/meanie3D/meanie3D_run.py @@ -223,7 +223,7 @@ def main(): # Run the detection and tracking steps detection = meanie3D.app.utils.getSafe(configuration, 'detection') tracking = meanie3D.app.utils.getSafe(configuration, 'tracking') - if (detection or tracking): + if detection or tracking: # run the actual clustering/tracking script if not scales: @@ -243,7 +243,7 @@ def main(): meanie3D.app.tracking.run(configuration, time_index) # Run the postprocessing steps - if (configuration['postprocessing']): + if 'postprocessing' in configuration: meanie3D.app.postprocessing.run(configuration) return From 70f30c5a52fdd3f322dec5ca927156ac3d054858 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Sat, 4 Jul 2020 09:31:07 +0100 Subject: [PATCH 53/65] Added new weight function. Bugfix. --- CMakeLists.txt | 1 + .../meanie3D/clustering/cluster_list_impl.h | 7 +- .../clustering/detection_commandline.h | 2 +- include/meanie3D/clustering/detection_impl.h | 2 +- include/meanie3D/filters/weight_filter_impl.h | 55 +++++-- include/meanie3D/weights/inv_erfc_weights.h | 140 ++++++++++++++++++ .../weights/weight_function_factory_impl.h | 23 ++- 7 files changed, 206 insertions(+), 24 deletions(-) create mode 100644 include/meanie3D/weights/inv_erfc_weights.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 801f021..58fedbb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -579,6 +579,7 @@ ADD_LIBRARY(meanie3D SHARED include/meanie3D/weights/exp10_weight.h include/meanie3D/weights/inverse_default.h include/meanie3D/weights/oase_weights.h + include/meanie3D/weights/inv_erfc_weights.h include/meanie3D/weights/weight_function.h include/meanie3D/weights/weight_function_factory.h include/meanie3D/weights/weight_function_factory_impl.h diff --git a/include/meanie3D/clustering/cluster_list_impl.h b/include/meanie3D/clustering/cluster_list_impl.h index 35a6057..915e77f 100644 --- a/include/meanie3D/clustering/cluster_list_impl.h +++ b/include/meanie3D/clustering/cluster_list_impl.h @@ -975,6 +975,7 @@ namespace m3D { Point *current_point = fs->points[i]; // skip zeroshift and non-original points + // TODO: revisit the idea of skipping non-original points if (vector_norm(fs->spatial_component(current_point->shift)) == 0 || !current_point->isOriginalPoint) continue; // Find the predecessor through gridded shift @@ -1175,10 +1176,8 @@ namespace m3D { } } - // Finally remove all points from all clusters, that were - // not part of the original data set, as well as make their - // modes the arithmetic mean of the remaining points - + // Finally remove all points from all clusters that were not part of the original + // data set. Make their modes the arithmetic mean of the remaining points. if (show_progress) { cout << endl << "Erasing non-original points ..."; start_timer(); diff --git a/include/meanie3D/clustering/detection_commandline.h b/include/meanie3D/clustering/detection_commandline.h index 1d7bbad..d97b32e 100644 --- a/include/meanie3D/clustering/detection_commandline.h +++ b/include/meanie3D/clustering/detection_commandline.h @@ -81,7 +81,7 @@ namespace m3D { "uniform,gauss,epanechnikov or none") ("weight-function-name,w", program_options::value()->default_value(params.weight_function_name), - "default,inverse,pow10 or oase") + "default,inverse,pow10,inverfc,oase or oaseci") ("wwf-lower-threshold", program_options::value()->default_value(params.wwf_lower_threshold), "Lower threshold for weight function filter.") diff --git a/include/meanie3D/clustering/detection_impl.h b/include/meanie3D/clustering/detection_impl.h index 24d4da2..93cc092 100644 --- a/include/meanie3D/clustering/detection_impl.h +++ b/include/meanie3D/clustering/detection_impl.h @@ -303,7 +303,7 @@ namespace m3D { ctx.sf->apply(ctx.fs); #if WRITE_FEATURESPACE - std::string fn = path.stem().string() + "_scale_" + boost::lexical_cast(scale) + ".vtk"; + std::string fn = path.stem().string() + "_scale_" + boost::lexical_cast(params.scale) + ".vtk"; VisitUtils::write_featurespace_vtk(fn, ctx.fs); #endif } diff --git a/include/meanie3D/filters/weight_filter_impl.h b/include/meanie3D/filters/weight_filter_impl.h index e2463e2..a379058 100644 --- a/include/meanie3D/filters/weight_filter_impl.h +++ b/include/meanie3D/filters/weight_filter_impl.h @@ -52,41 +52,70 @@ namespace m3D { void WeightThresholdFilter::apply(FeatureSpace *fs) { using namespace std; - boost::progress_display *progress_bar = NULL; - if (this->show_progress()) { cout << endl << "Applying weight function filter ..."; - progress_bar = new boost::progress_display(fs->size()); - start_timer(); } - typename Point::list::iterator pit; - vector *> accepted; vector *> erased; - for (size_t k = 0; k < fs->points.size(); k++) { + // // Calculate the max and min values of the weight function response + // T w_max = -10e10; + // T w_min = 10e10; + // for (size_t k = 0; k < fs->points.size(); k++) + // { + // Point *p = fs->points.at(k); + // T w = m_weight_function->operator()(p); + // if (w > w_max) { + // w_max = w; + // } + // if (w < w_min) + // { + // w_min = w; + // } + // } + + // // The filter criteria refer to percentages in the weight function's range + // T weight_range = w_max - w_min; + // T lower_bound = w_min + weight_range * m_lower_threshold; + // T upper_bound = w_min + weight_range * m_upper_threshold; + +#if WITH_OPENMP +#pragma omp parallel for schedule(dynamic, 10) +#endif + for (size_t k = 0; k < fs->points.size(); k++) + { if (this->show_progress()) { progress_bar->operator++(); } - Point *p = fs->points.at(k); - T w = m_weight_function->operator()(p); - - if (w < m_lower_threshold || w > m_upper_threshold) { + if (w < m_lower_threshold || w > m_upper_threshold) + { +#if WITH_OPENMP +#pragma omp critical +#endif erased.push_back(p); - } else { + } + else + { +#if WITH_OPENMP +#pragma omp critical +#endif accepted.push_back(p); } } fs->points = accepted; - for (size_t i = 0; i < erased.size(); i++) { +#if WITH_OPENMP +#pragma omp parallel for schedule(dynamic, 10) +#endif + for (size_t i = 0; i < erased.size(); i++) + { Point *p = erased[i]; delete p; } diff --git a/include/meanie3D/weights/inv_erfc_weights.h b/include/meanie3D/weights/inv_erfc_weights.h new file mode 100644 index 0000000..636f9d2 --- /dev/null +++ b/include/meanie3D/weights/inv_erfc_weights.h @@ -0,0 +1,140 @@ +/* The MIT License (MIT) + * + * (c) Jürgen Simon 2014 (juergen.simon@uni-bonn.de) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef M3D_INVERFCWEIGHTFUNCTION_H +#define M3D_INVERFCWEIGHTFUNCTION_H + +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace m3D { + + /** + * This weight function scales from [0 .. 1] following the inverse + * complementary gaussian error function: + * + * f(x) = 1-erfc(2x) + * + * The curve has a strong effect in the beginning and levels out + * quickly at 1. This weight function can be used to help sharpen + * the boundaries in objects that are in close proximity. + * + * The weight is calculated by iterating over all variables. For each + * variable a weight is generated that varies from [0..1] as the variable + * goes from it's valid_min to valid_max (linear). The values are summed + * up and divided by the number of variables. + */ + template + class InvErfcWeightFunction : public WeightFunction + { + private: + + map m_min; + map m_max; + MultiArray *m_weight; + + public: + + /** Construct the weight function, using the given values + * for valid_min/valid_max + * @param featurespace + * @param map of lower bounds + * @param map of upper bounds + */ + InvErfcWeightFunction(const detection_params_t ¶ms, + const detection_context_t &ctx) + : m_weight(new MultiArrayBlitz(ctx.coord_system->get_dimension_sizes(), 0.0)) + { + // If scale-space filter is present, use the filtered + // limits. If not, use the original limits + if (ctx.sf == NULL) { + for (size_t index = 0; index < ctx.data_store->rank(); index++) { + m_min[index] = ctx.data_store->min(index); + m_max[index] = ctx.data_store->max(index); + } + } else { + m_min = ctx.sf->get_filtered_min(); + m_max = ctx.sf->get_filtered_max(); + } + + calculate_weight_function(ctx.fs); + } + + ~InvErfcWeightFunction() + { + if (this->m_weight != NULL) { + delete m_weight; + m_weight = NULL; + } + } + + private: + + void + calculate_weight_function(const FeatureSpace *fs) { +#if WITH_OPENMP +#pragma omp parallel for +#endif + for (size_t i = 0; i < fs->points.size(); i++) { + Point *p = fs->points[i]; + T saliency = (fs->off_limits()->get(p->gridpoint)) ? 0.0 : compute_weight(fs, p->values); + m_weight->set(p->gridpoint, saliency); + } + }; + + /* Actual weight computation happens here */ + + T compute_weight(const FeatureSpace *fs, const vector &values) const + { + T sum = 0.0; + for (size_t var_index = 0; var_index < fs->value_rank(); var_index++) { + T value = values.at(fs->spatial_rank() + var_index); + + // value scaled to [0..1] + T max = m_max.find(var_index)->second; + T min = m_min.find(var_index)->second; + T var_weight = (value - min) / (max - min); + sum += var_weight; + } + T x = sum / ((T)fs->value_rank()); + return 1 - erfc(2*x); + } + + public: + + /** @return pre-calculated weight + */ + T operator()(const typename Point::ptr p) const { + return m_weight->get(p->gridpoint); + } + }; +} + +#endif diff --git a/include/meanie3D/weights/weight_function_factory_impl.h b/include/meanie3D/weights/weight_function_factory_impl.h index 11703d1..0e18fcb 100644 --- a/include/meanie3D/weights/weight_function_factory_impl.h +++ b/include/meanie3D/weights/weight_function_factory_impl.h @@ -30,6 +30,7 @@ #include "default_weights.h" #include "inverse_default.h" #include "exp10_weight.h" +#include "inv_erfc_weights.h" #include "weight_function_factory.h" @@ -48,15 +49,27 @@ namespace m3D { WeightFunction *weight_function = NULL; - if (params.weight_function_name == "oase-ci") { + if (params.weight_function_name == "oase-ci") + { weight_function = new OASECIWeightFunction(params, ctx); - } else if (params.weight_function_name == "oase") { + } + else if (params.weight_function_name == "oase") + { weight_function = new OASEWeightFunction(params, ctx); - } else if (params.weight_function_name == "inverse") { + } + else if (params.weight_function_name == "inverse") + { weight_function = new InverseDefaultWeightFunction(params, ctx); - } else if (params.weight_function_name == "pow10") { + } + else if (params.weight_function_name == "pow10") { weight_function = new EXP10WeightFunction(params, ctx); - } else { + } + else if (params.weight_function_name == "inverfc") + { + weight_function = new InvErfcWeightFunction(params, ctx); + } + else + { weight_function = new DefaultWeightFunction(params, ctx); } From 6acc00d1e3d5c47eb8b627a10011ef5d8dad8f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 6 Jul 2020 12:13:42 +0100 Subject: [PATCH 54/65] Fix FindNetCDF for Catalina. --- CMakeLists.txt | 8 +++--- cmake_modules/FindNETCDF.cmake | 46 +++++++++++++++++++++------------- 2 files changed, 32 insertions(+), 22 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 58fedbb..2b8a75a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -380,12 +380,12 @@ ENDIF () MESSAGE(STATUS "Resolving dependencies for Meanie3D") # netcdf -FIND_PACKAGE(NETCDF) +FIND_PACKAGE(NetCDF REQUIRED NETCDF_CXX) IF (NETCDF_FOUND) - MESSAGE(STATUS "Found netcdf ${NETCDF_INCLUDE_DIR} ${NETCDF_VERSION}") - INCLUDE_DIRECTORIES(${NETCDF_INCLUDE_DIR}) + MESSAGE(STATUS "Found netcdf ${NETCDF_INCLUDES} ${NETCDF_VERSION}") + INCLUDE_DIRECTORIES(${NETCDF_INCLUDES}) ELSE (NETCDF_FOUND) - MESSAGE(FATAL_ERROR "netcdf not found INCLUDE_DIR=${NETCDF_INCLUDE_DIR} LIBRARIES=${NETCDF_LIBRARIES}") + MESSAGE(FATAL_ERROR "netcdf not found INCLUDE_DIR=${NETCDF_INCLUDES} LIBRARIES=${NETCDF_LIBRARIES}") ENDIF (NETCDF_FOUND) # HDF5 diff --git a/cmake_modules/FindNETCDF.cmake b/cmake_modules/FindNETCDF.cmake index d269acc..800394e 100644 --- a/cmake_modules/FindNETCDF.cmake +++ b/cmake_modules/FindNETCDF.cmake @@ -1,26 +1,36 @@ -IF (CMAKE_SYSTEM_NAME MATCHES Linux) - MESSAGE(STATUS "looking for libnetcdf and libnetcdf_c++4 here: ${LPATH}") +MESSAGE(STATUS "CMAKE_SYSTEM_NAME=${CMAKE_SYSTEM_NAME}") +MESSAGE(STATUS "CMAKE_SYSTEM_VERSION=${CMAKE_SYSTEM_VERSION}") + +IF(CMAKE_SYSTEM_NAME MATCHES Linux) FIND_PATH(NETCDF_INCLUDE_DIR netcdf.h PATHS /usr/include /usr/local/include /opt/local/include) - FIND_LIBRARY(NETCDF NAMES netcdf PATHS ${LPATH}) - FIND_LIBRARY(NETCDF_CPP NAMES netcdf_c++4 PATHS ${LPATH}) -ELSE () - FIND_PATH(NETCDF_INCLUDE_DIR netcdf PATHS /usr/include /usr/local/include /opt/local/include) - FIND_LIBRARY(NETCDF NAMES netcdf PATHS /usr/lib /usr/local/lib /opt/local/lib) - FIND_LIBRARY(NETCDF_CPP NAMES netcdf_c++4 PATHS /usr/lib /usr/local/lib /opt/local/lib) -ENDIF () + FIND_LIBRARY(NETCDF NAMES netcdf PATHS /usr/lib /usr/lib64 /usr/local/lib /usr/local/lib64) + FIND_LIBRARY(NETCDF_CPP NAMES netcdf_c++ PATHS /usr/lib /usr/lib64 /usr/local/lib /usr/local/lib64) +ELSEIF(CMAKE_SYSTEM_NAME MATCHES Darwin) + FIND_PATH(NETCDF_INCLUDE_DIR netcdf PATHS /usr/include /usr/local/include /opt/local/include) + FIND_LIBRARY(NETCDF NAMES netcdf PATHS /usr/lib /usr/local/lib /opt/local/lib) + FIND_LIBRARY(NETCDF_CPP NAMES netcdf-cxx4 netcdf_c++ PATHS /usr/lib /usr/local/lib /opt/local/lib) +ENDIF() -IF (NETCDF AND NETCDF_CPP) - SET(NETCDF_LIBRARIES ${NETCDF_CPP} ${NETCDF}) -ELSE () - SET(NETCDF_LIBRARIES "NOTFOUND") +IF (NOT NETCDF_INCLUDE_DIR) + MESSAGE(WARNING "Could not locate NETCDF_INCLUDE_DIR") +ENDIF() + +IF (NOT NETCDF) + MESSAGE(WARNING "Could not locate libnetcdf") ENDIF() -IF (NETCDF_INCLUDE_DIR AND NETCDF_LIBRARIES) - SET(NETCDF_FOUND TRUE) -ENDIF (NETCDF_INCLUDE_DIR AND NETCDF_LIBRARIES) +IF (NOT NETCDF_CPP) + MESSAGE(WARNING "Could not locate libnetcdf_c++") +ENDIF() + +IF (NETCDF_INCLUDE_DIR AND NETCDF AND NETCDF_CPP) + SET(NETCDF_FOUND TRUE) + # SET(NETCDF_LIBRARIES "-L /usr/local/lib -lnetcdf_c++ -lnetcdf") + SET(NETCDF_LIBRARIES ${NETCDF} ${NETCDF_CPP}) +ELSE() + SET(NETCDF_LIBRARIES "NOTFOUND") +ENDIF() IF (NOT NETCDF_FOUND AND NETCDF_FIND_REQUIRED) MESSAGE(FATAL_ERROR "Could not find NETCDF") ENDIF () - -MARK_AS_ADVANCED(NETCDF_INCLUDE_DIR NETCDF_LIBRARIES) \ No newline at end of file From bb55577e7a4bbc98a131d718f9f78457b98527b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 6 Jul 2020 16:49:21 +0100 Subject: [PATCH 55/65] New weight function inverfc working. Added informative output on cfm2vtk. Attempted to add cluster centeres to the same file as point data (failed). --- include/meanie3D/clustering/cluster_impl.h | 978 +++++++++--------- .../clustering/detection_commandline.h | 7 +- include/meanie3D/utils/visit_impl.h | 908 +++++++++------- 3 files changed, 1050 insertions(+), 843 deletions(-) diff --git a/include/meanie3D/clustering/cluster_impl.h b/include/meanie3D/clustering/cluster_impl.h index a01e116..3484ed2 100644 --- a/include/meanie3D/clustering/cluster_impl.h +++ b/include/meanie3D/clustering/cluster_impl.h @@ -1,579 +1,579 @@ -/* The MIT License (MIT) - * - * (c) Jürgen Simon 2014 (juergen.simon@uni-bonn.de) - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - - -#ifndef M3D_CLUSTER_IMPL_H -#define M3D_CLUSTER_IMPL_H - -#include -#include -#include -#include - -#include - -#include "cluster.h" - -namespace m3D { - - using namespace std; - -#pragma mark - -#pragma mark Constructor/Destructor + /* The MIT License (MIT) + * + * (c) Jürgen Simon 2014 (juergen.simon@uni-bonn.de) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + + + #ifndef M3D_CLUSTER_IMPL_H + #define M3D_CLUSTER_IMPL_H + + #include + #include + #include + #include + + #include + + #include "cluster.h" + + namespace m3D { + + using namespace std; + + #pragma mark - + #pragma mark Constructor/Destructor + + template + Cluster::Cluster() + : m_radius(-1), m_index(NULL), m_rank(0), m_spatial_rank(0), m_weight_range_calculated(false), + m_min_weight(0), m_max_weight(0), mode(vector()), displacement(vector()), id(NO_ID), uuid(NO_UUID) { + } + + template + Cluster::Cluster(const vector &mode, size_t spatial_dimension) + : m_radius(-1), m_index(NULL), m_rank(mode.size()), m_spatial_rank(spatial_dimension), + m_weight_range_calculated(false), m_min_weight(0), m_max_weight(0), mode(mode), + displacement(vector(spatial_dimension)), id(NO_ID), uuid(NO_UUID) { + assert(m_rank > m_spatial_rank); + } + + template + Cluster::Cluster(const Cluster &o) + : m_points(o.get_points()), m_radius(-1), m_index(NULL), m_rank(o.m_rank), m_spatial_rank(o.m_spatial_rank), + m_weight_range_calculated(o.m_weight_range_calculated), m_min_weight(o.m_min_weight), + m_max_weight(o.m_max_weight), mode(o.mode), displacement(o.displacement), id(o.id), uuid(o.uuid) { + } + + template + Cluster::~Cluster() { + this->clear_histogram_cache(); + this->clear_index(); + } + + #pragma mark - + #pragma mark Adding / Removing points + + template + void + Cluster::add_point(Point *point) { + m_points.push_back(point); + point->cluster = this; + } + + template + void + Cluster::add_points(const vector * + + > &list, + bool addOriginalPointsOnly + ) { + typename Point::list::const_iterator li; + for ( + li = list.begin(); + li != list. + + end(); + + li++) { + typename Point::ptr p = *li; + if (!p-> + isOriginalPoint &&addOriginalPointsOnly + ) { + continue; + } + this-> + add_point(p); + } + } + + template + void + Cluster::remove_point(Point *point) { + typename vector * >::iterator it = this->get_points().find(*point); + if (it != this->get_points().end()) { + this->get_points().erase(it); + point->cluster = NULL; + } + } template - Cluster::Cluster() - : m_radius(-1), m_index(NULL), m_rank(0), m_spatial_rank(0), m_weight_range_calculated(false), - m_min_weight(0), m_max_weight(0), mode(vector()), displacement(vector()), id(NO_ID), uuid(NO_UUID) { + bool + Cluster::has_point(typename Point::ptr point) { + typename Point::list::iterator fi; + fi = find(this->get_points().begin(), this->get_points().end(), point); + return fi != this->get_points().end(); } template - Cluster::Cluster(const vector &mode, size_t spatial_dimension) - : m_radius(-1), m_index(NULL), m_rank(mode.size()), m_spatial_rank(spatial_dimension), - m_weight_range_calculated(false), m_min_weight(0), m_max_weight(0), mode(mode), - displacement(vector(spatial_dimension)), id(NO_ID), uuid(NO_UUID) { - assert(m_rank > m_spatial_rank); + typename Point::list & + Cluster::get_points() { + return m_points; } template - Cluster::Cluster(const Cluster &o) - : m_points(o.get_points()), m_radius(-1), m_index(NULL), m_rank(o.m_rank), m_spatial_rank(o.m_spatial_rank), - m_weight_range_calculated(o.m_weight_range_calculated), m_min_weight(o.m_min_weight), - m_max_weight(o.m_max_weight), mode(o.mode), displacement(o.displacement), id(o.id), uuid(o.uuid) { + void + Cluster::set_points(const typename Point::list &points, bool delete_flag) { + this->clear(delete_flag); + m_points = points; } template - Cluster::~Cluster() { - this->clear_histogram_cache(); - this->clear_index(); + bool + Cluster::empty() const { + return m_points.empty(); } -#pragma mark - -#pragma mark Adding / Removing points + template + size_t + Cluster::size() const { + return m_points.size(); + }; template - void - Cluster::add_point(Point *point) { - m_points.push_back(point); - point->cluster = this; + typename Point::ptr + Cluster::operator[](const size_t &index) { + return m_points.at(index); + }; + + template + typename Point::ptr + Cluster::at(const size_t &index) const { + return m_points.at(index); } template void - Cluster::add_points(const vector * - - > &list, - bool addOriginalPointsOnly - ) { - typename Point::list::const_iterator li; - for ( - li = list.begin(); - li != list. - - end(); - - li++) { - typename Point::ptr p = *li; - if (!p-> - isOriginalPoint &&addOriginalPointsOnly - ) { - continue; -} -this-> -add_point(p); -} -} - -template -void -Cluster::remove_point(Point *point) { - typename vector * >::iterator it = this->get_points().find(*point); - if (it != this->get_points().end()) { - this->get_points().erase(it); - point->cluster = NULL; - } -} - -template -bool -Cluster::has_point(typename Point::ptr point) { - typename Point::list::iterator fi; - fi = find(this->get_points().begin(), this->get_points().end(), point); - return fi != this->get_points().end(); -} - -template -typename Point::list & -Cluster::get_points() { - return m_points; -} - -template -void -Cluster::set_points(const typename Point::list &points, bool delete_flag) { - this->clear(delete_flag); - m_points = points; -} - -template -bool -Cluster::empty() const { - return m_points.empty(); -} - -template -size_t -Cluster::size() const { - return m_points.size(); -}; - -template -typename Point::ptr -Cluster::operator[](const size_t &index) { - return m_points.at(index); -}; - -template -typename Point::ptr -Cluster::at(const size_t &index) const { - return m_points.at(index); -} - -template -void -Cluster::clear(bool deletion_flag) { - if (deletion_flag) { - typename Point::list::const_iterator pi; - for (pi = m_points.begin(); pi != m_points.end(); ++pi) { - typename Point::ptr p = *pi; - delete p; + Cluster::clear(bool deletion_flag) { + if (deletion_flag) { + typename Point::list::const_iterator pi; + for (pi = m_points.begin(); pi != m_points.end(); ++pi) { + typename Point::ptr p = *pi; + delete p; + } } + m_points.clear(); + } + + template + bool Cluster::has_margin_points() const { + return m_has_margin_points; + } + + template + void Cluster::set_has_margin_points(bool value) { + m_has_margin_points = value; } - m_points.clear(); -} - -template -bool Cluster::has_margin_points() const { - return m_has_margin_points; -} - -template -void Cluster::set_has_margin_points(bool value) { - m_has_margin_points = value; -} - -#pragma mark - -#pragma mark Derived properties - -template -vector -Cluster::weighed_center(const size_t &variable_index) { - throw "not implemented"; -} - -#pragma mark - -#pragma mark Operators - -template -bool -Cluster::operator==(const Cluster &o) { - return o.mode() == mode(); -} - -template -Cluster -Cluster::operator=(const Cluster &o) { - return Cluster(o); -} - -#pragma mark - -#pragma mark Histograms - -template -const typename Histogram::ptr -Cluster::histogram(size_t variable_index, - T valid_min, - T valid_max, - size_t number_of_bins) { - Histogram *h = NULL; - - try { - h = this->m_histograms.at(variable_index); - } catch (const std::exception &e) { - h = Histogram::create(m_points, variable_index, valid_min, valid_max, number_of_bins); - this->m_histograms.insert(std::pair::ptr>(variable_index, h)); + + #pragma mark - + #pragma mark Derived properties + + template + vector + Cluster::weighed_center(const size_t &variable_index) { + throw "not implemented"; } - return h; -} + #pragma mark - + #pragma mark Operators -template -void -Cluster::clear_histogram_cache() { - typename histogram_map_t::iterator it; - for (it = this->m_histograms.begin(); it != this->m_histograms.end(); ++it) { - delete it->second; + template + bool + Cluster::operator==(const Cluster &o) { + return o.mode() == mode(); } - this->m_histograms.clear(); -} - -#pragma mark - -#pragma mark Index - -template -PointIndex * -Cluster::index() { - if (this->m_index == NULL) { - // index is constructed using the spatial components only - vector indexes(m_spatial_rank); - for (size_t i = 0; i < m_spatial_rank; i++) { - indexes[i] = i; - } - this->m_index = PointIndex::create(&this->points, indexes); + + template + Cluster + Cluster::operator=(const Cluster &o) { + return Cluster(o); } - return m_index; -} - -template -void -Cluster::clear_index() { - if (this->m_index != NULL) { - delete this->m_index; - this->m_index = NULL; + + #pragma mark - + #pragma mark Histograms + + template + const typename Histogram::ptr + Cluster::histogram(size_t variable_index, + T valid_min, + T valid_max, + size_t number_of_bins) { + Histogram *h = NULL; + + try { + h = this->m_histograms.at(variable_index); + } catch (const std::exception &e) { + h = Histogram::create(m_points, variable_index, valid_min, valid_max, number_of_bins); + this->m_histograms.insert(std::pair::ptr>(variable_index, h)); + } + + return h; } -} -#pragma mark - -#pragma mark Coverage + template + void + Cluster::clear_histogram_cache() { + typename histogram_map_t::iterator it; + for (it = this->m_histograms.begin(); it != this->m_histograms.end(); ++it) { + delete it->second; + } + this->m_histograms.clear(); + } -template -vector -Cluster::geometrical_center() { - using namespace utils::vectors; + #pragma mark - + #pragma mark Index - if (m_geometrical_center.empty()) { - vector center(m_spatial_rank, 0.0); - typename Point::list::iterator pi; - for (pi = this->get_points().begin(); pi != this->get_points().end(); ++pi) { - vector tmp = center; - center += (*pi)->coordinate; - if (isnan(center[0])) { - cerr << "ERROR:ARGHH!" << endl; + template + PointIndex * + Cluster::index() { + if (this->m_index == NULL) { + // index is constructed using the spatial components only + vector indexes(m_spatial_rank); + for (size_t i = 0; i < m_spatial_rank; i++) { + indexes[i] = i; } + this->m_index = PointIndex::create(&this->points, indexes); } - m_geometrical_center = center / ((T) this->get_points().size()); + return m_index; } - return m_geometrical_center; -} - -template -vector -Cluster::weighed_center(size_t variable_index) { - vector wc; - try { - wc = this->m_weighed_centers.at(variable_index); - } catch (const std::exception &e) { - // first, figure out the min/max values + template + void + Cluster::clear_index() { + if (this->m_index != NULL) { + delete this->m_index; + this->m_index = NULL; + } + } - typename Point::list::const_iterator pi; + #pragma mark - + #pragma mark Coverage - T min = std::numeric_limits::max(); - T max = std::numeric_limits::min(); - for (pi = this->get_points().begin(); pi != this->get_points().end(); pi++) { - T val = (*pi)->values[variable_index]; - if (val < min) { - min = val; - } - if (val > max) { - max = val; + template + vector + Cluster::geometrical_center() { + using namespace utils::vectors; + + if (m_geometrical_center.empty()) { + vector center(m_spatial_rank, 0.0); + typename Point::list::iterator pi; + for (pi = this->get_points().begin(); pi != this->get_points().end(); ++pi) { + vector tmp = center; + center += (*pi)->coordinate; + if (isnan(center[0])) { + cerr << "ERROR:ARGHH!" << endl; + } } + m_geometrical_center = center / ((T) this->get_points().size()); } - T range = max - min; + return m_geometrical_center; + } + + template + vector + Cluster::weighed_center(size_t variable_index) { + vector wc; + try { + wc = this->m_weighed_centers.at(variable_index); + } catch (const std::exception &e) { + // first, figure out the min/max values + + typename Point::list::const_iterator pi; + + T min = std::numeric_limits::max(); + T max = std::numeric_limits::min(); + for (pi = this->get_points().begin(); pi != this->get_points().end(); pi++) { + T val = (*pi)->values[variable_index]; + if (val < min) { + min = val; + } + if (val > max) { + max = val; + } + } + + T range = max - min; - // weight is assigned based on percentage of the range - // within [min..max] + // weight is assigned based on percentage of the range + // within [min..max] - vector center(m_spatial_rank, 0.0); - T overall_mass_percent = 0.0; - for (pi = this->get_points().begin(); pi != this->get_points().end(); pi++) { - T mass = (*pi)->values[variable_index]; - T massPercent = (max - mass) / range; - center += (*pi)->coordinate * massPercent; - overall_mass_percent += massPercent; + vector center(m_spatial_rank, 0.0); + T overall_mass_percent = 0.0; + for (pi = this->get_points().begin(); pi != this->get_points().end(); pi++) { + T mass = (*pi)->values[variable_index]; + T massPercent = (max - mass) / range; + center += (*pi)->coordinate * massPercent; + overall_mass_percent += massPercent; + } + + center /= overall_mass_percent; + m_weighed_centers.insert(std::pair > (variable_index, center)); + wc = m_weighed_centers[variable_index]; } - center /= overall_mass_percent; - m_weighed_centers.insert(std::pair > (variable_index, center)); - wc = m_weighed_centers[variable_index]; + return wc; } - return wc; -} - -template -void -Cluster::clear_center_caches() { - m_geometrical_center.clear(); - m_weighed_centers.clear(); -} - -template -::units::values::m -Cluster::radius(const CoordinateSystem *cs) { - using namespace utils::vectors; - if (m_radius == ::units::values::m(-1)) { - vector mode_spatial(&mode[0], &mode[m_spatial_rank]); - vector mode_in_meters = cs->to_meters(mode_spatial); - typename Point::list::iterator pi; - ::units::values::m sum; - for (pi = this->get_points().begin(); pi != this->get_points().end(); pi++) { - typename Point::ptr p = *pi; - vector coord_in_meters = cs->to_meters(p->coordinate); - ::units::values::m dist = ::units::values::m(vector_norm(coord_in_meters - mode_in_meters)); - sum += dist; + template + void + Cluster::clear_center_caches() { + m_geometrical_center.clear(); + m_weighed_centers.clear(); + } + + template + ::units::values::m + Cluster::radius(const CoordinateSystem *cs) { + using namespace utils::vectors; + if (m_radius == ::units::values::m(-1)) { + vector mode_spatial(&mode[0], &mode[m_spatial_rank]); + vector mode_in_meters = cs->to_meters(mode_spatial); + typename Point::list::iterator pi; + ::units::values::m sum; + for (pi = this->get_points().begin(); pi != this->get_points().end(); pi++) { + typename Point::ptr p = *pi; + vector coord_in_meters = cs->to_meters(p->coordinate); + ::units::values::m dist = ::units::values::m(vector_norm(coord_in_meters - mode_in_meters)); + sum += dist; + } + m_radius = sum / ((T) this->get_points().size()); } - m_radius = sum / ((T) this->get_points().size()); + return m_radius; } - return m_radius; -} -template -T -Cluster::modal_weight_response(const WeightFunction *w) const { - T result = 0; + template + T + Cluster::modal_weight_response(const WeightFunction *w) const { + T result = 0; + + // pick the first point of this cluster to figure out the + // spatial and value dimensions - // pick the first point of this cluster to figure out the - // spatial and value dimensions + if (!m_points.empty() && w != NULL) { + const CoordinateSystem *cs = this->m_index->feature_space()->coordinate_system; - if (!m_points.empty() && w != NULL) { - const CoordinateSystem *cs = this->m_index->feature_space()->coordinate_system; + // extract coordinate of the mode and do a reverse + // lookup on the grid point + vector coordinate(&mode[0], &mode[cs->rank()]); + vector gridpoint; + cs->reverse_lookup(coordinate, gridpoint); - // extract coordinate of the mode and do a reverse - // lookup on the grid point - vector coordinate(&mode[0], &mode[cs->rank()]); - vector gridpoint; - cs->reverse_lookup(coordinate, gridpoint); + // Construct a point object for the weight function + Point point; + point.values = mode; + point.coordinate = coordinate; + point.gridpoint = gridpoint; - // Construct a point object for the weight function - Point point; - point.values = mode; - point.coordinate = coordinate; - point.gridpoint = gridpoint; + result = w->operator()(&point); + } - result = w->operator()(&point); + return result; } - return result; -} + template + T + Cluster::average_weight_response(const WeightFunction *w) const { + T result = 0; -template -T -Cluster::average_weight_response(const WeightFunction *w) const { - T result = 0; + // pick the first point of this cluster to figure out the + // spatial and value dimensions - // pick the first point of this cluster to figure out the - // spatial and value dimensions + for (size_t i = 0; i < this->size(); i++) { + Point *p = this->at(i); + result += w->operator()(p); + } - for (size_t i = 0; i < this->size(); i++) { - Point *p = this->at(i); - result += w->operator()(p); - } + if (m_points.empty()) { + result /= ((T) this->size()); + } - if (m_points.empty()) { - result /= ((T) this->size()); + return result; } - return result; -} - -#pragma mark - -#pragma mark Dynamic Range Calculation - -template -void -Cluster::dynamic_range(const typename Point::list &points, - const WeightFunction *weight_function, - T &lower_bound, - T &upper_bound) { - lower_bound = std::numeric_limits::max(); - upper_bound = std::numeric_limits::min(); - - typename Point::list::const_iterator pi; - for (pi = points.begin(); pi != points.end(); pi++) { - typename Point::ptr p = *pi; - T value = weight_function->operator()(p); - if (value < lower_bound) { - lower_bound = value; - } - if (value > upper_bound) { - upper_bound = value; + #pragma mark - + #pragma mark Dynamic Range Calculation + + template + void + Cluster::dynamic_range(const typename Point::list &points, + const WeightFunction *weight_function, + T &lower_bound, + T &upper_bound) { + lower_bound = std::numeric_limits::max(); + upper_bound = std::numeric_limits::min(); + + typename Point::list::const_iterator pi; + for (pi = points.begin(); pi != points.end(); pi++) { + typename Point::ptr p = *pi; + T value = weight_function->operator()(p); + if (value < lower_bound) { + lower_bound = value; + } + if (value > upper_bound) { + upper_bound = value; + } } } -} - -template -void -Cluster::dynamic_range(const WeightFunction *weight_function, - T &lower_bound, - T &upper_bound) { - if (!m_weight_range_calculated) { - Cluster::dynamic_range(this->points, weight_function, m_min_weight, m_max_weight); - m_weight_range_calculated = true; - } - lower_bound = m_min_weight; - upper_bound = m_max_weight; -} -template -void -Cluster::variable_ranges(std::vector &min, - std::vector &max, - std::vector &median) { - size_t var_rank = m_rank - m_spatial_rank; + template + void + Cluster::dynamic_range(const WeightFunction *weight_function, + T &lower_bound, + T &upper_bound) { + if (!m_weight_range_calculated) { + Cluster::dynamic_range(this->points, weight_function, m_min_weight, m_max_weight); + m_weight_range_calculated = true; + } + lower_bound = m_min_weight; + upper_bound = m_max_weight; + } - min.clear(); - min.resize(var_rank, std::numeric_limits::max()); + template + void + Cluster::variable_ranges(std::vector &min, + std::vector &max, + std::vector &median) { + size_t var_rank = m_rank - m_spatial_rank; - max.clear(); - max.resize(var_rank, std::numeric_limits::min()); + min.clear(); + min.resize(var_rank, std::numeric_limits::max()); - median.clear(); - median.resize(var_rank, 0); + max.clear(); + max.resize(var_rank, std::numeric_limits::min()); - // For median calculation - vector < vector > median_values; - median_values.resize(var_rank, vector()); + median.clear(); + median.resize(var_rank, 0); - typename Point::list::const_iterator pi; + // For median calculation + vector < vector > median_values; + median_values.resize(var_rank, vector()); - for (pi = this->get_points().begin(); pi != this->get_points().end(); ++pi) { - vector values = (*pi)->values; + typename Point::list::const_iterator pi; - for (size_t i = 0; i < var_rank; i++) { - T value = values.at(m_spatial_rank + i); - // record values for median - median_values.at(i).push_back(value); - // min/max checks - if (value < min.at(i)) min.at(i) = value; - if (value > max.at(i)) max.at(i) = value; + for (pi = this->get_points().begin(); pi != this->get_points().end(); ++pi) { + vector values = (*pi)->values; + + for (size_t i = 0; i < var_rank; i++) { + T value = values.at(m_spatial_rank + i); + // record values for median + median_values.at(i).push_back(value); + // min/max checks + if (value < min.at(i)) min.at(i) = value; + if (value > max.at(i)) max.at(i) = value; + } } - } - // finish median calculation - - for (size_t i = 0; i < var_rank; i++) { - vector values = median_values.at(i); - if (!values.empty()) { - if (values.size() == 1) { - // special case: only one element - median.at(i) = values.at(0); - } else { - // two or more elements - std::sort(values.begin(), values.end()); - if (values.size() % 2 == 0) { - size_t index = values.size() / 2 - 1; - median.at(i) = (values.at(index) + values.at(index + 1)) / 2.0; + // finish median calculation + + for (size_t i = 0; i < var_rank; i++) { + vector values = median_values.at(i); + if (!values.empty()) { + if (values.size() == 1) { + // special case: only one element + median.at(i) = values.at(0); } else { - size_t index = (values.size() - 1) / 2; - median.at(i) = values.at(index); + // two or more elements + std::sort(values.begin(), values.end()); + if (values.size() % 2 == 0) { + size_t index = values.size() / 2 - 1; + median.at(i) = (values.at(index) + values.at(index + 1)) / 2.0; + } else { + size_t index = (values.size() - 1) / 2; + median.at(i) = values.at(index); + } } } } } -} - -template -void -Cluster::print(bool includePoints) { - cout << "Cluster uuid:" << this->uuid - << " id:" << this->id - << " mode:" << this->mode - << " size:" << this->size() - << endl; - - if (includePoints) { - for (size_t i = 0; i < this->size(); i++) { - typename Point::ptr p = this->at(i); - p->print(1); + + template + void + Cluster::print(bool includePoints) { + cout << "Cluster uuid:" << this->uuid + << " id:" << this->id + << " mode:" << this->mode + << " size:" << this->size() + << endl; + + if (includePoints) { + for (size_t i = 0; i < this->size(); i++) { + typename Point::ptr p = this->at(i); + p->print(1); + } } } -} - - -template -void -Cluster::set_bounding_box_min(const vector &bounds) { - m_bounding_box_min = bounds; -}; - -template -const vector & -Cluster::get_bounding_box_min() { - if (m_bounding_box_min.empty()) { - vector inf(spatial_rank(), std::numeric_limits::max()); - typename Point::list::iterator pi; - for (pi = m_points.begin(); pi != m_points.end(); ++pi) { - typename Point::ptr p = *pi; - for (size_t j = 0; j < spatial_rank(); j++) { - if (p->coordinate[j] < inf[j]) { - inf[j] = p->coordinate[j]; + + + template + void + Cluster::set_bounding_box_min(const vector &bounds) { + m_bounding_box_min = bounds; + }; + + template + const vector & + Cluster::get_bounding_box_min() { + if (m_bounding_box_min.empty()) { + vector inf(spatial_rank(), std::numeric_limits::max()); + typename Point::list::iterator pi; + for (pi = m_points.begin(); pi != m_points.end(); ++pi) { + typename Point::ptr p = *pi; + for (size_t j = 0; j < spatial_rank(); j++) { + if (p->coordinate[j] < inf[j]) { + inf[j] = p->coordinate[j]; + } } } + m_bounding_box_min = inf; + } - m_bounding_box_min = inf; + return m_bounding_box_min; + }; - } - return m_bounding_box_min; -}; - -template -void -Cluster::set_bounding_box_max(const vector &bounds) { - m_bounding_box_max = bounds; -}; - -template -const vector & -Cluster::get_bounding_box_max() { - if (m_bounding_box_max.empty()) { - vector sup(spatial_rank(), -std::numeric_limits::max()); - typename Point::list::iterator pi; - for (pi = m_points.begin(); pi != m_points.end(); ++pi) { - typename Point::ptr p = *pi; - for (size_t j = 0; j < spatial_rank(); j++) { - if (p->coordinate[j] > sup[j]) { - sup[j] = p->coordinate[j]; + template + void + Cluster::set_bounding_box_max(const vector &bounds) { + m_bounding_box_max = bounds; + }; + + template + const vector & + Cluster::get_bounding_box_max() { + if (m_bounding_box_max.empty()) { + vector sup(spatial_rank(), -std::numeric_limits::max()); + typename Point::list::iterator pi; + for (pi = m_points.begin(); pi != m_points.end(); ++pi) { + typename Point::ptr p = *pi; + for (size_t j = 0; j < spatial_rank(); j++) { + if (p->coordinate[j] > sup[j]) { + sup[j] = p->coordinate[j]; + } } } + m_bounding_box_max = sup; + } - m_bounding_box_max = sup; + return m_bounding_box_max; + }; } - return m_bounding_box_max; -}; - -} -#endif + #endif diff --git a/include/meanie3D/clustering/detection_commandline.h b/include/meanie3D/clustering/detection_commandline.h index d97b32e..a2c925c 100644 --- a/include/meanie3D/clustering/detection_commandline.h +++ b/include/meanie3D/clustering/detection_commandline.h @@ -449,11 +449,8 @@ namespace m3D { // Weight Function params.weight_function_name = vm["weight-function-name"].as(); - if (!(params.weight_function_name == "default" - || params.weight_function_name == "inverse" - || params.weight_function_name == "pow10" - || params.weight_function_name == "oase" - || params.weight_function_name == "oase-ci")) { + if (!(params.weight_function_name == "default" || params.weight_function_name == "inverse" || params.weight_function_name == "pow10" || params.weight_function_name == "inverfc" || params.weight_function_name == "oase" || params.weight_function_name == "oase-ci")) + { cerr << "Illegal weight function name " << params.weight_function_name << ". Only 'default','inverse','pow10' or 'oase' are known." << endl; exit(EXIT_FAILURE); diff --git a/include/meanie3D/utils/visit_impl.h b/include/meanie3D/utils/visit_impl.h index ab843d3..2ea58c5 100644 --- a/include/meanie3D/utils/visit_impl.h +++ b/include/meanie3D/utils/visit_impl.h @@ -1,25 +1,25 @@ /* The MIT License (MIT) - * - * (c) Jürgen Simon 2014 (juergen.simon@uni-bonn.de) - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ +* +* (c) Jürgen Simon 2014 (juergen.simon@uni-bonn.de) +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to deal +* in the Software without restriction, including without limitation the rights +* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +* copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in all +* copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +* SOFTWARE. +*/ #ifndef M3D_VISUALIZATIONUTILS_IMPL_H #define M3D_VISUALIZATIONUTILS_IMPL_H @@ -55,6 +55,7 @@ #include #include #include +#include #include #include #include @@ -70,60 +71,68 @@ #include #include - #include #include "visit.h" -namespace m3D { - namespace utils { +namespace m3D +{ + namespace utils + { using namespace std; /** Global variable maintaining the dimension mapping */ - template + template vector VisitUtils::VTK_DIMENSION_INDEXES; - // Macro for safe access +// Macro for safe access #define get_vtk_index(i) VTK_DIMENSION_INDEXES.empty() ? i : VTK_DIMENSION_INDEXES[i] /* ---------------------------------------------------------------- */ /* General VTK data structure mapping/handling */ /* ---------------------------------------------------------------- */ - template + template void VisitUtils::update_vtk_dimension_mapping(vector dim_names, - vector vtk_dim_names) { + vector vtk_dim_names) + { // VTK dimension mapping - if (!vtk_dim_names.empty()) { + if (!vtk_dim_names.empty()) + { vector vtk_dimension_indexes; // Number of values should be the same in --dimensions and --vtk-dimensions - if (vtk_dim_names.size() != dim_names.size()) { + if (vtk_dim_names.size() != dim_names.size()) + { cerr << "FATAL: --dimensions and --vtk-dimensions must have \ - the same number of arguments" << endl; + the same number of arguments" + << endl; exit(EXIT_FAILURE); } // Check if all dimensions are accounted for in --vtk-dimensions - for (size_t i = 0; i < vtk_dim_names.size(); i++) { + for (size_t i = 0; i < vtk_dim_names.size(); i++) + { vector::iterator vi = find(dim_names.begin(), dim_names.end(), vtk_dim_names[i]); - if (vi == dim_names.end()) { + if (vi == dim_names.end()) + { cerr << "FATAL: --vtk-dimensions - entry " << vtk_dim_names[i] << " has no pendant in attribute \ - 'featurespace_dimensions'" << endl; + 'featurespace_dimensions'" + << endl; exit(EXIT_FAILURE); } int index = utils::vectors::index_of_first(dim_names, vtk_dim_names[i]); - vtk_dimension_indexes.push_back((size_t) index); + vtk_dimension_indexes.push_back((size_t)index); } utils::VisitUtils::VTK_DIMENSION_INDEXES = vtk_dimension_indexes; @@ -132,110 +141,144 @@ namespace m3D { } } - template + template void - VisitUtils::get_vtk_image_dimensions(const CoordinateSystem *cs, int &nx, int &ny, int &nz) { + VisitUtils::get_vtk_image_dimensions(const CoordinateSystem *cs, int &nx, int &ny, int &nz) + { assert(cs->rank() > 0 && cs->rank() <= 3); nx = ny = nz = 1; - if (cs->rank() == 1) { + if (cs->rank() == 1) + { nx = cs->dimensions()[get_vtk_index(0)].getSize(); - } else if (cs->rank() == 2) { + } + else if (cs->rank() == 2) + { nx = cs->dimensions()[get_vtk_index(0)].getSize(); ny = cs->dimensions()[get_vtk_index(1)].getSize(); - } else { + } + else + { nx = cs->dimensions()[get_vtk_index(0)].getSize(); ny = cs->dimensions()[get_vtk_index(1)].getSize(); nz = cs->dimensions()[get_vtk_index(2)].getSize(); } } - template + template void - VisitUtils::get_vtk_coords(const vector &coords, T &x, T &y, T &z) { + VisitUtils::get_vtk_coords(const vector &coords, T &x, T &y, T &z) + { assert(coords.size() > 0 && coords.size() <= 3); x = y = z = 0; - if (coords.size() == 1) { + if (coords.size() == 1) + { x = coords.at(get_vtk_index(0)); - } else if (coords.size() == 2) { + } + else if (coords.size() == 2) + { x = coords.at(get_vtk_index(0)); y = coords.at(get_vtk_index(1)); - } else { + } + else + { x = coords.at(get_vtk_index(0)); y = coords.at(get_vtk_index(1)); z = coords.at(get_vtk_index(2)); } } - template + template void - VisitUtils::get_vtk_gridpoint(const vector &gp, int &gx, int &gy, int &gz) { + VisitUtils::get_vtk_gridpoint(const vector &gp, int &gx, int &gy, int &gz) + { assert(gp.size() > 0 && gp.size() <= 3); gx = gy = gz = 0; - if (gp.size() == 1) { + if (gp.size() == 1) + { gx = gp.at(get_vtk_index(0)); - } else if (gp.size() == 2) { + } + else if (gp.size() == 2) + { gx = gp.at(get_vtk_index(0)); gy = gp.at(get_vtk_index(1)); - } else { + } + else + { gx = gp.at(get_vtk_index(0)); gy = gp.at(get_vtk_index(1)); gz = gp.at(get_vtk_index(2)); } } - template + template void - copy_vtk_coordinate_data(const CoordinateSystem *cs, size_t index, vtkDoubleArray *coords) { + copy_vtk_coordinate_data(const CoordinateSystem *cs, size_t index, vtkDoubleArray *coords) + { NcVar vx = cs->dimension_variables().at(index); const T *dim_data = cs->get_dimension_data_ptr(vx); NcDim dx = cs->dimensions().at(index); coords->SetNumberOfValues(dx.getSize()); - for (int i = 0; i < dx.getSize(); i++) { + for (int i = 0; i < dx.getSize(); i++) + { double value = dim_data[i]; coords->SetValue(i, value); } } - template + template vtkRectilinearGrid * - VisitUtils::allocate_vtk_rectilinear_grid(const CoordinateSystem *cs, - vector &coord_pointers) { + VisitUtils::allocate_vtk_rectilinear_grid(const CoordinateSystem *cs, + vector &coord_pointers) + { assert(cs->rank() > 0 && cs->rank() <= 3); int nx = 0, ny = 0, nz = 0; get_vtk_image_dimensions(cs, nx, ny, nz); vtkRectilinearGrid *rgrid = vtkRectilinearGrid::New(); rgrid->SetDimensions(nx, ny, nz); - for (int i = 0; i < cs->rank(); i++) { + for (int i = 0; i < cs->rank(); i++) + { vtkDoubleArray *coords = vtkDoubleArray::New(); coords->SetNumberOfComponents(1); copy_vtk_coordinate_data(cs, get_vtk_index(i), coords); coord_pointers.push_back(coords); - if (i == 0) { + if (i == 0) + { rgrid->SetXCoordinates(coords); - } else if (i == 1) { + } + else if (i == 1) + { rgrid->SetYCoordinates(coords); - } else { + } + else + { rgrid->SetZCoordinates(coords); } } // supply 1-D arrays with value 0 for the // empty dimensions - for (int i = 2; i >= 0; i--) { - if (i >= cs->rank()) { + for (int i = 2; i >= 0; i--) + { + if (i >= cs->rank()) + { vtkDoubleArray *coords = vtkDoubleArray::New(); coords->SetNumberOfComponents(1); coords->InsertNextValue(0.0); coord_pointers.push_back(coords); - if (i == 0) { + if (i == 0) + { rgrid->SetXCoordinates(coords); - } else if (i == 1) { + } + else if (i == 1) + { rgrid->SetYCoordinates(coords); - } else { + } + else + { rgrid->SetZCoordinates(coords); } } @@ -243,20 +286,26 @@ namespace m3D { return rgrid; } - template + template vtkIdType - VisitUtils::to_single_index(const CoordinateSystem *cs, + VisitUtils::to_single_index(const CoordinateSystem *cs, int nx, int ny, int nz, - int gx, int gy, int gz) { + int gx, int gy, int gz) + { vtkIdType index = 0; - if (cs->rank() == 1) { + if (cs->rank() == 1) + { // 'x' index = gx; - } else if (cs->rank() == 2) { + } + else if (cs->rank() == 2) + { // 'x' + 'y' * 'Nx' index = gx + gy * nx; - } else { + } + else + { // 'x' + 'y' * 'Nx' + 'z' * 'Nx' * 'Ny' index = gx + gy * nx + gz * nx * ny; } @@ -269,13 +318,14 @@ namespace m3D { /* ---------------------------------------------------------------- */ /** Write out the cluster list as list of points. The point's 'value' is the size of the cluster - * @param full path to filename, including extension '.vtk' - * @param cluster list - */ - template + * @param full path to filename, including extension '.vtk' + * @param cluster list + */ + template void VisitUtils::write_cluster_modes_vtk(const string &filename, const typename Cluster::list &list, - bool spatial_only) { + bool spatial_only) + { ofstream f(filename.c_str()); f << fixed << setprecision(4); @@ -287,16 +337,19 @@ namespace m3D { f << "DATASET UNSTRUCTURED_GRID" << endl; f << "POINTS " << list.size() << " FLOAT" << endl; - for (size_t ci = 0; ci < list.size(); ci++) { + for (size_t ci = 0; ci < list.size(); ci++) + { typename Cluster::ptr c = list[ci]; size_t dim_count = spatial_only ? c->spatial_rank() : c->rank(); vector mode = c->mode; size_t dims_plotted = 0; - for (size_t vi = 0; vi < dim_count; vi++) { + for (size_t vi = 0; vi < dim_count; vi++) + { size_t dim_index = vi; - if (vi < VTK_DIMENSION_INDEXES.size()) { + if (vi < VTK_DIMENSION_INDEXES.size()) + { dim_index = VTK_DIMENSION_INDEXES.empty() ? vi : VTK_DIMENSION_INDEXES[vi]; } @@ -304,7 +357,8 @@ namespace m3D { dims_plotted++; } - while (dims_plotted < 3) { + while (dims_plotted < 3) + { f << "0.0\t"; dims_plotted++; } @@ -316,13 +370,17 @@ namespace m3D { f << "POINT_DATA " << list.size() << endl; f << "SCALARS mode INT" << endl; f << "LOOKUP_TABLE default" << endl; - for (size_t pi = 0; pi < list.size(); pi++) { + for (size_t pi = 0; pi < list.size(); pi++) + { typename Cluster::ptr c = list[pi]; size_t dim_count = spatial_only ? c->spatial_rank() : c->rank(); - if (spatial_only) { + if (spatial_only) + { f << c->id << endl; - } else { + } + else + { f << c->mode[dim_count - 1]; } } @@ -330,11 +388,12 @@ namespace m3D { f.close(); } - template + template void VisitUtils::write_geometrical_cluster_centers_vtk(const string &filename, const typename Cluster::list &list, - bool at_max_height) { + bool at_max_height) + { ofstream f(filename.c_str()); f << fixed << setprecision(4); @@ -343,76 +402,70 @@ namespace m3D { typename Point::ptr p = NULL; // Write Header - f << "# vtk DataFile Version 3.0" << endl; f << "point list " << endl; f << "ASCII" << endl; f << "DATASET UNSTRUCTURED_GRID" << endl; f << "POINTS " << list.size() << " FLOAT" << endl; - for (size_t ci = 0; ci < list.size(); ci++) { + for (size_t ci = 0; ci < list.size(); ci++) + { typename Cluster::ptr c = list[ci]; - // obtain the geometrical center - size_t spatial_dims = c->get_points().at(0)->coordinate.size(); - vector mode = c->geometrical_center(); - - if (spatial_dims == 3 && at_max_height) { + if (spatial_dims == 3 && at_max_height) + { T max = 0; - for (size_t pi = 0; pi < c->size(); pi++) { + for (size_t pi = 0; pi < c->size(); pi++) + { Point *p = c->get_points().at(pi); - if (p->coordinate[0] > max) { + if (p->coordinate[0] > max) + { max = p->coordinate[0]; } } - mode[0] = 1.5 * max; } - size_t dims_plotted = 0; - - for (size_t vi = 0; vi < spatial_dims; vi++) { + for (size_t vi = 0; vi < spatial_dims; vi++) + { size_t dim_index = vi; - - if (vi < VTK_DIMENSION_INDEXES.size()) { + if (vi < VTK_DIMENSION_INDEXES.size()) + { dim_index = VTK_DIMENSION_INDEXES.empty() ? vi : VTK_DIMENSION_INDEXES[vi]; } - f << mode[dim_index] << "\t"; - dims_plotted++; } - while (dims_plotted < 3) { + while (dims_plotted < 3) + { f << "0.0\t"; - dims_plotted++; } - f << endl; } - f << endl; f << "POINT_DATA " << list.size() << endl; f << "SCALARS geometrical_center INT" << endl; f << "LOOKUP_TABLE default" << endl; - for (size_t pi = 0; pi < list.size(); pi++) { + for (size_t pi = 0; pi < list.size(); pi++) + { typename Cluster::ptr c = list[pi]; - f << c->id << endl; } - f.close(); } static vtkSmartPointer _vtk_cluster_table = NULL; - template + template vtkSmartPointer - VisitUtils::cluster_lookup_table() { - if (_vtk_cluster_table == NULL) { + VisitUtils::cluster_lookup_table() + { + if (_vtk_cluster_table == NULL) + { // Lookup table _vtk_cluster_table = vtkSmartPointer::New(); _vtk_cluster_table->SetNumberOfTableValues(6); @@ -441,15 +494,16 @@ namespace m3D { return _vtk_cluster_table; } - template + template void - VisitUtils::write_clusters_vtu(const ClusterList *list, - const CoordinateSystem *cs, + VisitUtils::write_clusters_vtu(const ClusterList *list, + const CoordinateSystem *cs, const string &base_name, unsigned int max_colors, bool use_ids, bool include_boundary, - bool write_xml) { + bool write_xml) + { // escape dangerous characters from basename string basename = boost::filesystem::path(base_name).stem().string(); string mesh_filename = basename + "-clusters" + (write_xml ? ".vtu" : ".vtk"); @@ -459,9 +513,11 @@ namespace m3D { size_t point_dim = 0; cout << "Cluster list has " << list->clusters.size() << " clusters" << endl; - for (size_t ci = 0; ci < list->clusters.size(); ci++) { + for (size_t ci = 0; ci < list->clusters.size(); ci++) + { num_points += list->clusters[ci]->size(); - if (point_dim == 0) { + if (point_dim == 0) + { point_dim = list->clusters[ci]->get_points()[0]->coordinate.size(); } } @@ -471,6 +527,9 @@ namespace m3D { vtkSmartPointer cellArray = vtkSmartPointer::New(); vtkSmartPointer points = vtkSmartPointer::New(); + vtkSmartPointer centerPoints = vtkSmartPointer::New(); + vtkSmartPointer centerCellArray = vtkSmartPointer::New(); + vtkSmartPointer pointData = vtkSmartPointer::New(); pointData->SetName("point_data"); @@ -485,27 +544,36 @@ namespace m3D { cellColors->SetName("cell_color"); cellColors->SetLookupTable(cluster_lookup_table()); - for (size_t ci = 0; ci < list->clusters.size(); ci++) { + vtkSmartPointer clusterIds = vtkSmartPointer::New(); + clusterIds->SetName("geometrical_center"); + for (size_t ci = 0; ci < list->clusters.size(); ci++) + { m3D::id_t id = use_ids ? list->clusters[ci]->id : ci; int color = id % 6; - typename Cluster::ptr cluster = list->clusters[ci]; - for (vtkIdType pi = 0; pi < cluster->size(); pi++) { + // Write ID + T x, y, z; + VisitUtils::get_vtk_coords(cluster->geometrical_center(), x, y, z); + vtkIdType center; + center = centerPoints->InsertNextPoint(x, y, point_dim == 2 ? 0 : z); + vtkSmartPointer centerPixel = vtkSmartPointer::New(); + centerPixel->GetPointIds()->SetId(0, center); + centerCellArray->InsertNextCell(centerPixel); + clusterIds->InsertNextValue(id); + + // Write point_color and value information + for (vtkIdType pi = 0; pi < cluster->size(); pi++) + { Point *p = list->clusters[ci]->get_points()[pi]; - double scalarValue = p->values[point_dim]; cellData->InsertNextValue(scalarValue); cellColors->InsertNextValue(color); - - T x, y, z; - x = y = z = 0.0; - VisitUtils::get_vtk_coords(p->coordinate, x, y, z); - - if (point_dim == 2) { + if (point_dim == 2) + { T rx = cs->resolution()[VisitUtils::index_of(0)] / 2.0; T ry = cs->resolution()[VisitUtils::index_of(1)] / 2.0; @@ -533,7 +601,9 @@ namespace m3D { quad->GetPointIds()->SetId(3, p4); cellArray->InsertNextCell(quad); - } else { + } + else + { T rx = cs->resolution()[VisitUtils::index_of(0)] / 2.0; T ry = cs->resolution()[VisitUtils::index_of(1)] / 2.0; T rz = cs->resolution()[VisitUtils::index_of(2)] / 2.0; @@ -587,27 +657,34 @@ namespace m3D { } } + // 2D is a square, 3D a hexahedron + VTKCellType type = point_dim == 2 ? VTK_QUAD : VTK_HEXAHEDRON; + + #ifdef FOOBAR + // TODO: research and finish this part. The problem seems to be to have two unstructured + // grids with different numbers of points in one file. + // Create an unstructured grid for center labels + vtkSmartPointer centerGrid = vtkSmartPointer::New(); + centerGrid->SetPoints(centerPoints); + centerGrid->SetCells(type, centerCellArray); + centerGrid->GetPointData()->AddArray(clusterIds); + centerGrid->GetCellData()->AddArray(clusterIds); + #endif + // Create an unstructured grid and write it off vtkSmartPointer unstructuredGrid = vtkSmartPointer::New(); - - // Set point data unstructuredGrid->SetPoints(points); - - // Set cell data - if (point_dim == 2) { - unstructuredGrid->SetCells(VTK_QUAD, cellArray); - } else { - unstructuredGrid->SetCells(VTK_HEXAHEDRON, cellArray); - } - + unstructuredGrid->SetCells(type, cellArray); unstructuredGrid->GetPointData()->AddArray(pointData); unstructuredGrid->GetPointData()->AddArray(pointColors); unstructuredGrid->GetCellData()->AddArray(cellData); unstructuredGrid->GetCellData()->AddArray(cellColors); + unstructuredGrid->GetCellData()->AddArray(clusterIds); // Extract the envelope vtkSmartPointer enveloper; - if (include_boundary) { + if (include_boundary) + { // Geometry filter vtkSmartPointer geometry = vtkSmartPointer::New(); geometry->SetInputData(unstructuredGrid); @@ -617,11 +694,14 @@ namespace m3D { // Delauny 2D/3D - if (point_dim == 2) { + if (point_dim == 2) + { vtkSmartPointer delauny = vtkSmartPointer::New(); delauny->SetInputConnection(cleaner->GetOutputPort()); enveloper = delauny; - } else { + } + else + { vtkSmartPointer delauny = vtkSmartPointer::New(); delauny->SetInputConnection(cleaner->GetOutputPort()); enveloper = delauny; @@ -629,41 +709,50 @@ namespace m3D { } // Write .vtu/.vtk file + // The reason this absurd macro is used is, that to much surprise + // vtkXMLUnstructuredGridWriter and vtkUnstructuredGridWriter have + // the same interface, but do not formally implement one or inherit + // from the same base class. Without this construct, the code would + // have to be duplicated ... + +#define WRITE_CLUSTERS(writer, centerGrid, unstructuredGrid, include_boundary, poly_filename, enveloper) \ + writer->SetFileName(mesh_filename.c_str()); \ + /* TODO: comment back in when the problem with the labels on the grid is solved */ \ + /* writer->SetInputData(centerGrid); */ \ + /* writer->Write(); */ \ + writer->SetInputData(unstructuredGrid); \ + writer->Write(); \ + if (include_boundary) \ + { \ + writer->SetFileName(poly_filename.c_str()); \ + writer->SetInputConnection(enveloper->GetOutputPort()); \ + writer->Write(); \ + } + if (write_xml) { vtkSmartPointer writer = vtkSmartPointer::New(); - writer->SetFileName(mesh_filename.c_str()); - writer->SetInputData(unstructuredGrid); - writer->Write(); - if (include_boundary) { - writer->SetFileName(poly_filename.c_str()); - writer->SetInputConnection(enveloper->GetOutputPort()); - writer->Write(); - } + WRITE_CLUSTERS(writer, centerGrid, unstructuredGrid, include_boundary, poly_filename, enveloper); } else { vtkSmartPointer writer = vtkSmartPointer::New(); - writer->SetFileName(mesh_filename.c_str()); - writer->SetInputData(unstructuredGrid); - writer->Write(); - if (include_boundary) { - writer->SetFileName(poly_filename.c_str()); - writer->SetInputConnection(enveloper->GetOutputPort()); - writer->Write(); - } + WRITE_CLUSTERS(writer, centerGrid, unstructuredGrid, include_boundary, poly_filename, enveloper); } } - template + template void VisitUtils::write_center_tracks_vtk(typename Track::trackmap &track_map, const std::string &basename, size_t spatial_dimensions, - bool exclude_degenerates) { + bool exclude_degenerates) + { typename Track::trackmap::iterator tmi; - for (tmi = track_map.begin(); tmi != track_map.end(); tmi++) { + for (tmi = track_map.begin(); tmi != track_map.end(); tmi++) + { typename Track::ptr track = tmi->second; - if (exclude_degenerates && track->clusters.size() == 1) { + if (exclude_degenerates && track->clusters.size() == 1) + { continue; } @@ -682,18 +771,21 @@ namespace m3D { // Write point coordinates out as unstructured grid typename std::list::ptr>::iterator ti; - for (ti = track->clusters.begin(); ti != track->clusters.end(); ++ti) { - TrackCluster *c = (TrackCluster *) (*ti); + for (ti = track->clusters.begin(); ti != track->clusters.end(); ++ti) + { + TrackCluster *c = (TrackCluster *)(*ti); vector center = c->geometrical_center(); - for (size_t vi = 0; vi < center.size(); vi++) { + for (size_t vi = 0; vi < center.size(); vi++) + { size_t index = VTK_DIMENSION_INDEXES.empty() ? vi : VTK_DIMENSION_INDEXES[vi]; f << center[index] << "\t"; } - if (center.size() < 3) { + if (center.size() < 3) + { f << "0.0"; } @@ -709,7 +801,8 @@ namespace m3D { size_t step = 0; - for (ti = track->clusters.begin(); ti != track->clusters.end(); ti++) { + for (ti = track->clusters.begin(); ti != track->clusters.end(); ti++) + { //f << ti->points.size() << endl; f << step++ << endl; } @@ -718,17 +811,20 @@ namespace m3D { } } - template + template void VisitUtils::write_center_tracks_vtk(typename ConradCluster::trackmap_t &track_map, const std::string &basename, - bool exclude_degenerates) { + bool exclude_degenerates) + { typename ConradCluster::trackmap_t::const_iterator tmi; - for (tmi = track_map.begin(); tmi != track_map.end(); tmi++) { + for (tmi = track_map.begin(); tmi != track_map.end(); tmi++) + { typename ConradCluster::track_t *track = tmi->second; - if (exclude_degenerates && track->size() == 1) { + if (exclude_degenerates && track->size() == 1) + { continue; } @@ -748,14 +844,17 @@ namespace m3D { typename ConradCluster::track_t::const_iterator ti; - for (ti = track->begin(); ti != track->end(); ti++) { + for (ti = track->begin(); ti != track->end(); ti++) + { vector center = ti->center(); - for (size_t vi = 0; vi < center.size(); vi++) { + for (size_t vi = 0; vi < center.size(); vi++) + { f << center[vi] << "\t"; } - if (center.size() < 3) { + if (center.size() < 3) + { f << "0.0"; } @@ -771,7 +870,8 @@ namespace m3D { size_t step = 0; - for (ti = track->begin(); ti != track->end(); ti++) { + for (ti = track->begin(); ti != track->end(); ti++) + { //f << ti->points.size() << endl; f << step++ << endl; } @@ -780,15 +880,17 @@ namespace m3D { } } - template + template void VisitUtils::write_cluster_weight_response_vtk(const string &base_name, const typename Cluster::list &clusters, - WeightFunction *w, - bool useMode) { + WeightFunction *w, + bool useMode) + { string basename(base_name); - for (size_t ci = 0; ci < clusters.size(); ci++) { + for (size_t ci = 0; ci < clusters.size(); ci++) + { Cluster *cluster = clusters[ci]; vector mode = cluster->mode; @@ -814,16 +916,19 @@ namespace m3D { size_t point_dim = cluster->at(0)->coordinate.size(); - for (size_t pi = 0; pi < cluster->size(); pi++) { + for (size_t pi = 0; pi < cluster->size(); pi++) + { Point *p = cluster->at(pi); - for (size_t vi = 0; vi < point_dim; vi++) { + for (size_t vi = 0; vi < point_dim; vi++) + { size_t index = VTK_DIMENSION_INDEXES.empty() ? vi : VTK_DIMENSION_INDEXES[vi]; f << p->coordinate[index] << "\t"; } - for (size_t j = p->coordinate.size(); j < 3; j++) { + for (size_t j = p->coordinate.size(); j < 3; j++) + { f << "0.0\t"; } @@ -832,7 +937,6 @@ namespace m3D { // Write point data out. Only take the first value after coordinates - T wr = useMode ? cluster->modal_weight_response(w) : cluster->average_weight_response(w); f << endl; @@ -840,7 +944,8 @@ namespace m3D { f << "SCALARS weight FLOAT" << endl; f << "LOOKUP_TABLE default" << endl; - for (size_t pi = 0; pi < cluster->size() - 1; pi++) { + for (size_t pi = 0; pi < cluster->size() - 1; pi++) + { f << wr << endl; } @@ -848,35 +953,41 @@ namespace m3D { } } - template + template void VisitUtils::write_cluster_meanshift_vtk(const string &base_name, const typename Cluster::list &clusters, bool use_ids, - bool spatial_only) { + bool spatial_only) + { std::string basename(base_name); boost::replace_all(basename, "/", "_"); boost::replace_all(basename, "..", ""); - for (size_t ci = 0; ci < clusters.size(); ci++) { + for (size_t ci = 0; ci < clusters.size(); ci++) + { Cluster *cluster = clusters[ci]; - typedef vector > vvector_t; + typedef vector> vvector_t; vvector_t origins; vvector_t shifts; - for (size_t pi = 0; pi < cluster->size(); pi++) { + for (size_t pi = 0; pi < cluster->size(); pi++) + { typename Point::ptr p = cluster->at(pi); - if (spatial_only) { + if (spatial_only) + { origins.push_back(p->coordinate); vector shift = p->shift; shifts.push_back(vector(&shift[0], &shift[p->coordinate.size()])); - } else { + } + else + { origins.push_back(p->values); shifts.push_back(p->shift); } @@ -886,26 +997,27 @@ namespace m3D { VisitUtils::write_vectors_vtk(filename, origins, shifts, "shift"); } - } /** Helper function for drawing an ellipse (representing bandwidth and somesuch) - * in a .curve file format. - * @param filename - * @param half axis data - * @param number of segments in the curve (default 1000) - * @param pointer to origin vector, defaults to NULL = (0,0) - */ - template + * in a .curve file format. + * @param filename + * @param half axis data + * @param number of segments in the curve (default 1000) + * @param pointer to origin vector, defaults to NULL = (0,0) + */ + template void VisitUtils::write_ellipsis_2d(const string &filename, const vector &half_axis, size_t number_of_segements, - const vector *origin) { + const vector *origin) + { ofstream fs(filename.c_str()); // fs.imbue( locale("de_DE.UTF-8") ); fs << "# bandwidth" /* << fixed << setprecision(0)*/ << endl; vector x0 = (origin == NULL) ? vector(2, 0.0) : *origin; - for (size_t i = 0; i < number_of_segements; i++) { - float phi = i * 2 * M_PI / ((float) number_of_segements); + for (size_t i = 0; i < number_of_segements; i++) + { + float phi = i * 2 * M_PI / ((float)number_of_segements); float x = x0[1] + half_axis[0] * cos(phi); float y = x0[0] + half_axis[1] * sin(phi); fs << y << "\t" << x << endl; @@ -914,28 +1026,32 @@ namespace m3D { } /** Helper function for drawing an ellipse (representing bandwidth and somesuch) - * in a .curve file format. - * @param filename - * @param half axis data - * @param number of segments in the curve (default 1000) - * @param pointer to origin vector, defaults to NULL = (0,0,0) - */ - template + * in a .curve file format. + * @param filename + * @param half axis data + * @param number of segments in the curve (default 1000) + * @param pointer to origin vector, defaults to NULL = (0,0,0) + */ + template void VisitUtils::write_ellipsis_3d(const string &filename, const vector &half_axis, size_t number_of_segements, - const vector *origin) { + const vector *origin) + { ofstream fs(filename.c_str()); // fs.imbue( locale("de_DE.UTF-8") ); fs << "x\ty\tz\tv" << endl; vector x0 = (origin == NULL) ? vector(2, 0.0) : *origin; - for (size_t i = 0; i < number_of_segements; i++) { - float u = i * 2 * M_PI / ((float) number_of_segements); - for (size_t j = 0; j < number_of_segements; j++) { - float v = j * 2 * M_PI / ((float) number_of_segements); + for (size_t i = 0; i < number_of_segements; i++) + { + float u = i * 2 * M_PI / ((float)number_of_segements); + for (size_t j = 0; j < number_of_segements; j++) + { + float v = j * 2 * M_PI / ((float)number_of_segements); float x = half_axis[0] * cos(u) * sin(v); float y = half_axis[1] * sin(u) * sin(v); float z = half_axis[2] * cos(v); - if (origin != NULL) { + if (origin != NULL) + { x += origin->at(0); y += origin->at(1); z += origin->at(2); @@ -947,9 +1063,10 @@ namespace m3D { fs.close(); } - template + template void - VisitUtils::write_featurespace_vtk(const string &filename, FeatureSpace *fs, string variable_name) { + VisitUtils::write_featurespace_vtk(const string &filename, FeatureSpace *fs, string variable_name) + { ofstream f(filename.c_str()); f << fixed << setprecision(8); @@ -961,17 +1078,21 @@ namespace m3D { f << "POINTS " << fs->points.size() << " FLOAT" << endl; // Write point coordinates out as unstructured grid - for (size_t pi = 0; pi < fs->points.size(); pi++) { + for (size_t pi = 0; pi < fs->points.size(); pi++) + { typename Point::ptr p = fs->points[pi]; vector c = p->values; - for (size_t ri = 0; ri < c.size(); ri++) { + for (size_t ri = 0; ri < c.size(); ri++) + { size_t index = ri; - if (ri < p->coordinate.size()) { + if (ri < p->coordinate.size()) + { index = VTK_DIMENSION_INDEXES.empty() ? ri : VTK_DIMENSION_INDEXES[ri]; } f << c[index] << "\t"; } - if (c.size() < 3) { + if (c.size() < 3) + { f << "0.0"; } f << endl; @@ -982,7 +1103,8 @@ namespace m3D { f << "POINT_DATA " << fs->points.size() << endl; f << "SCALARS " << variable_name << " FLOAT" << endl; f << "LOOKUP_TABLE default" << endl; - for (size_t pi = 0; pi < fs->points.size(); pi++) { + for (size_t pi = 0; pi < fs->points.size(); pi++) + { typename Point::ptr p = fs->points[pi]; vector v = p->values; T value = v.back(); @@ -991,13 +1113,14 @@ namespace m3D { f.close(); } - template + template void VisitUtils::write_featurespace_variables_vtk(const string &filename, - FeatureSpace *fs, + FeatureSpace *fs, const vector &feature_variables, const vector &vtk_variables, - bool write_legacy) { + bool write_legacy) + { const CoordinateSystem *cs = fs->coordinate_system; int nx, ny, nz; @@ -1008,7 +1131,8 @@ namespace m3D { rgrid->DebugOn(); rgrid->CheckAttributes(); - for (size_t vi = 0; vi < vtk_variables.size(); vi++) { + for (size_t vi = 0; vi < vtk_variables.size(); vi++) + { std::string var = vtk_variables[vi]; size_t value_index = fs->spatial_rank() + vectors::index_of_first(feature_variables, var); cout << "Writing variable " << var << " (index=" << value_index << ")" << endl; @@ -1017,9 +1141,10 @@ namespace m3D { variable->SetNumberOfComponents(1); // scalar variable->SetNumberOfValues(nx * ny * nz); variable->FillComponent(0, 0); - for (size_t i = 0; i < fs->points.size(); ++i) { + for (size_t i = 0; i < fs->points.size(); ++i) + { typename Point::ptr p = fs->points.at(i); - double value = (double) p->values[value_index]; + double value = (double)p->values[value_index]; int gx, gy, gz; VisitUtils::get_vtk_gridpoint(p->gridpoint, gx, gy, gz); int gridIndex = to_single_index(cs, nx, ny, nz, gx, gy, gz); @@ -1031,10 +1156,11 @@ namespace m3D { // Write out - if (write_legacy) { + if (write_legacy) + { // ASCII vtkSmartPointer - writer = vtkSmartPointer::New(); + writer = vtkSmartPointer::New(); std::string fn = filename + ".vtk"; writer->SetFileName(fn.c_str()); #if VTK_MAJOR_VERSION <= 5 @@ -1043,10 +1169,12 @@ namespace m3D { writer->SetInputData(rgrid); #endif writer->Write(); - } else { + } + else + { // XML vtkSmartPointer - xmlWriter = vtkSmartPointer::New(); + xmlWriter = vtkSmartPointer::New(); std::string xml_fn = filename + "." + xmlWriter->GetDefaultFileExtension(); xmlWriter->SetFileName(xml_fn.c_str()); @@ -1059,18 +1187,20 @@ namespace m3D { } // Clean up - for (size_t ci = 0; ci < coords.size(); ci++) { + for (size_t ci = 0; ci < coords.size(); ci++) + { coords.at(ci)->Delete(); } rgrid->Delete(); } - template + template void VisitUtils::write_vectors_vtk(const string &filename, - const vector > &origins, - const vector > &vectors, - string var_name) { + const vector> &origins, + const vector> &vectors, + string var_name) + { assert(origins.size() == vectors.size()); ofstream f(filename.c_str()); @@ -1087,16 +1217,19 @@ namespace m3D { size_t spatial_dims = origins.front().size(); - for (size_t index = 0; index < origins.size(); index++) { + for (size_t index = 0; index < origins.size(); index++) + { vector v = origins[index]; - for (size_t ri = 0; ri < spatial_dims; ri++) { + for (size_t ri = 0; ri < spatial_dims; ri++) + { size_t index = VisitUtils::index_of(ri); f << v[index] << "\t"; } - for (int i = spatial_dims; i < 3; i++) { + for (int i = spatial_dims; i < 3; i++) + { f << "0.0\t"; } @@ -1106,29 +1239,32 @@ namespace m3D { f << endl; f << "POINT_DATA " << origins.size() << endl; f << "VECTORS " << var_name << " FLOAT" << endl; - for (size_t index = 0; index < vectors.size(); index++) { + for (size_t index = 0; index < vectors.size(); index++) + { vector v = vectors[index]; - for (size_t ri = 0; ri < spatial_dims; ri++) { + for (size_t ri = 0; ri < spatial_dims; ri++) + { size_t index = VisitUtils::index_of(ri); f << v[index] << "\t"; } - for (int i = spatial_dims; i < 3; i++) { + for (int i = spatial_dims; i < 3; i++) + { f << "0.0\t"; } f << endl; } f << endl; - } - template + template void VisitUtils::write_matrix_vtk(const string &filename, const boost::numeric::ublas::matrix &matrix, - string var_name) { + string var_name) + { ofstream f(filename.c_str()); f << fixed << setprecision(4); @@ -1141,11 +1277,14 @@ namespace m3D { // Write point coordinates out as unstructured grid - for (size_t pi = 0; pi < matrix.size1(); pi++) { - for (size_t ri = 0; ri < matrix.size2(); ri++) { + for (size_t pi = 0; pi < matrix.size1(); pi++) + { + for (size_t ri = 0; ri < matrix.size2(); ri++) + { f << matrix.operator()(pi, ri) << "\t"; } - if (matrix.size2() < 3) { + if (matrix.size2() < 3) + { f << "0.0"; } @@ -1158,17 +1297,19 @@ namespace m3D { f << "POINT_DATA " << matrix.size1() << endl; f << "SCALARS " << var_name << " FLOAT" << endl; f << "LOOKUP_TABLE default" << endl; - for (size_t pi = 0; pi < matrix.size1(); pi++) { + for (size_t pi = 0; pi < matrix.size1(); pi++) + { f << "0.0" << endl; } f.close(); } - template + template void - VisitUtils::write_ublas_matrix_vtk(const string &filename, const vector > &origins, - const vector > &vectors) { + VisitUtils::write_ublas_matrix_vtk(const string &filename, const vector> &origins, + const vector> &vectors) + { assert(origins.size() == vectors.size()); ofstream f(filename.c_str()); @@ -1180,10 +1321,12 @@ namespace m3D { f << "ASCII" << endl; f << "DATASET UNSTRUCTURED_GRID" << endl; f << "POINTS " << origins.size() << " FLOAT" << endl; - for (size_t index = 0; index < origins.size(); index++) { + for (size_t index = 0; index < origins.size(); index++) + { vector v = origins[index]; - for (size_t ri = 0; ri < v.size(); ri++) { + for (size_t ri = 0; ri < v.size(); ri++) + { f << v[ri] << "\t"; } @@ -1194,10 +1337,12 @@ namespace m3D { f << "POINT_DATA " << origins.size() << endl; f << "FIELD FieldData 1" << endl; f << "vectors " << vectors.front().size() << " " << origins.size() << " FLOAT" << endl; - for (size_t index = 0; index < vectors.size(); index++) { + for (size_t index = 0; index < vectors.size(); index++) + { vector v = vectors[index]; - for (size_t ri = 0; ri < v.size(); ri++) { + for (size_t ri = 0; ri < v.size(); ri++) + { f << v[ri] << "\t"; } @@ -1206,10 +1351,11 @@ namespace m3D { f << endl; } - template + template void VisitUtils::write_pointlist_vtk(const string &filename, typename Point::list *list, size_t dim, - string var_name) { + string var_name) + { ofstream f(filename.c_str()); f << fixed << setprecision(4); @@ -1221,12 +1367,15 @@ namespace m3D { f << "POINTS " << list->size() << " FLOAT" << endl; // Write point coordinates out as unstructured grid - for (size_t pi = 0; pi < list->size(); pi++) { + for (size_t pi = 0; pi < list->size(); pi++) + { Point *p = list->at(pi); - for (size_t ri = 0; ri < dim; ri++) { + for (size_t ri = 0; ri < dim; ri++) + { f << p->values[get_vtk_index(ri)] << "\t"; } - if (dim < 3) { + if (dim < 3) + { f << "0.0"; } f << endl; @@ -1237,7 +1386,8 @@ namespace m3D { f << "SCALARS " << var_name << " FLOAT" << endl; f << "LOOKUP_TABLE default" << endl; - for (size_t pi = 0; pi < list->size(); pi++) { + for (size_t pi = 0; pi < list->size(); pi++) + { Point *p = list->at(pi); f << p->values[dim] << endl; @@ -1246,10 +1396,11 @@ namespace m3D { f.close(); } - template + template void VisitUtils::write_pointlist_all_vars_vtk(const string &filename, typename Point::list *list, - const vector &var_names) { + const vector &var_names) + { ofstream f(filename.c_str()); f << fixed << setprecision(4); @@ -1265,30 +1416,35 @@ namespace m3D { Point *p = NULL; - for (size_t pi = 0; pi < list->size(); pi++) { + for (size_t pi = 0; pi < list->size(); pi++) + { p = list->at(pi); - for (size_t ri = 0; ri < p->coordinate.size(); ri++) { + for (size_t ri = 0; ri < p->coordinate.size(); ri++) + { f << p->values[get_vtk_index(ri)] << "\t"; } - if (p->coordinate.size() < 3) { + if (p->coordinate.size() < 3) + { f << "0.0"; } f << endl; } - for (size_t k = p->coordinate.size(); k < p->values.size(); k++) { + for (size_t k = p->coordinate.size(); k < p->values.size(); k++) + { std::string var_name = var_names.empty() - ? "var_" + boost::lexical_cast(k) - : var_names[k - p->coordinate.size()]; + ? "var_" + boost::lexical_cast(k) + : var_names[k - p->coordinate.size()]; f << endl; f << "POINT_DATA " << list->size() << endl; f << "SCALARS " << var_name << " FLOAT" << endl; f << "LOOKUP_TABLE default" << endl; - for (size_t pi = 0; pi < list->size(); pi++) { + for (size_t pi = 0; pi < list->size(); pi++) + { Point *vp = list->at(pi); f << vp->values[k] << endl; @@ -1298,9 +1454,10 @@ namespace m3D { f.close(); } - template + template void - VisitUtils::write_pointlist_vtk(const string &filename, vector > *list, string var_name) { + VisitUtils::write_pointlist_vtk(const string &filename, vector> *list, string var_name) + { ofstream f(filename.c_str()); f << fixed << setprecision(4); @@ -1314,14 +1471,17 @@ namespace m3D { size_t dim = 0; - if (list->size() > 0) { + if (list->size() > 0) + { dim = list->front().size(); } // Write point coordinates out as unstructured grid - for (size_t pi = 0; pi < list->size(); pi++) { - for (size_t ri = 0; ri < dim; ri++) { + for (size_t pi = 0; pi < list->size(); pi++) + { + for (size_t ri = 0; ri < dim; ri++) + { f << list->at(pi)[get_vtk_index(ri)] << "\t"; } @@ -1335,8 +1495,9 @@ namespace m3D { f << "SCALARS " << var_name << " FLOAT" << endl; f << "LOOKUP_TABLE default" << endl; - for (size_t pi = 0; pi < list->size(); pi++) { - float value = static_cast (pi) * 255.0f / static_cast (list->size()); + for (size_t pi = 0; pi < list->size(); pi++) + { + float value = static_cast(pi) * 255.0f / static_cast(list->size()); f << value << endl; } @@ -1344,10 +1505,11 @@ namespace m3D { f.close(); } - template + template void - VisitUtils::write_modes_vtk(const string &filename, const vector > &list, - const vector &trajectory_lenghts, string var_name) { + VisitUtils::write_modes_vtk(const string &filename, const vector> &list, + const vector &trajectory_lenghts, string var_name) + { ofstream f(filename.c_str()); f << fixed << setprecision(4); @@ -1360,14 +1522,17 @@ namespace m3D { f << "POINTS " << list.size() << " FLOAT" << endl; size_t dim = 0; - if (list.size() > 0) { + if (list.size() > 0) + { dim = list.front().size(); } // Write point coordinates out as unstructured grid - for (size_t pi = 0; pi < list.size(); pi++) { - for (size_t ri = 0; ri < dim; ri++) { + for (size_t pi = 0; pi < list.size(); pi++) + { + for (size_t ri = 0; ri < dim; ri++) + { f << list[pi][get_vtk_index(ri)] << "\t"; } @@ -1381,7 +1546,8 @@ namespace m3D { f << "SCALARS " << var_name << " FLOAT" << endl; f << "LOOKUP_TABLE default" << endl; - for (size_t pi = 0; pi < list.size(); pi++) { + for (size_t pi = 0; pi < list.size(); pi++) + { float value = trajectory_lenghts[pi]; f << value << endl; @@ -1390,25 +1556,32 @@ namespace m3D { f.close(); } - template + template void VisitUtils::write_radolan_vtk(const string &filename, const string &outfile, - Radolan::RDDataType *threshold) { + Radolan::RDDataType *threshold) + { using namespace Radolan; RDScan *scan = RDAllocateScan(); - if (RDReadScan(filename.c_str(), scan, true) >= 0) { + if (RDReadScan(filename.c_str(), scan, true) >= 0) + { // figure out how many valid values we have size_t number_of_valid_points = 0; - if (threshold == NULL) { + if (threshold == NULL) + { number_of_valid_points = scan->dimLat * scan->dimLon; - } else { - for (size_t i = 0; i < (scan->dimLat * scan->dimLon); i++) { + } + else + { + for (size_t i = 0; i < (scan->dimLat * scan->dimLon); i++) + { RDDataType value = scan->data[i]; - if (value >= *threshold) { + if (value >= *threshold) + { number_of_valid_points++; } } @@ -1429,11 +1602,14 @@ namespace m3D { int lat, lon; - for (lat = 0; lat < scan->dimLat; lat++) { - for (lon = 0; lon < scan->dimLon; lon++) { + for (lat = 0; lat < scan->dimLat; lat++) + { + for (lon = 0; lon < scan->dimLon; lon++) + { RDDataType value = scan->data[lat * scan->dimLon + lon]; - if (threshold == NULL || (threshold != NULL && value >= *threshold)) { + if (threshold == NULL || (threshold != NULL && value >= *threshold)) + { RDCartesianPoint p = rcs.cartesianCoordinate(rdGridPoint(lon, lat)); f << p.x << "\t" << p.y << "\t0.0" << endl; @@ -1445,13 +1621,17 @@ namespace m3D { f << endl; f << "POINT_DATA " << number_of_valid_points << endl; - f << "SCALARS " << "reflectivity" << " FLOAT" << endl; + f << "SCALARS " + << "reflectivity" + << " FLOAT" << endl; f << "LOOKUP_TABLE default" << endl; - for (size_t i = 0; i < scan->dimLat * scan->dimLon; i++) { + for (size_t i = 0; i < scan->dimLat * scan->dimLon; i++) + { RDDataType value = scan->data[i]; - if (threshold == NULL || (threshold != NULL && value >= *threshold)) { + if (threshold == NULL || (threshold != NULL && value >= *threshold)) + { f << value << endl; } } @@ -1460,13 +1640,14 @@ namespace m3D { } } - template + template void VisitUtils::write_weights(const string &filename, const string &var_name, typename Point::list *list, const vector &weights, - bool restrict_to_2D) { + bool restrict_to_2D) + { ofstream f(filename.c_str()); f << fixed << setprecision(4); @@ -1479,15 +1660,18 @@ namespace m3D { f << "POINTS " << list->size() << " FLOAT" << endl; size_t dim = 0; - if (list->size() > 0) { + if (list->size() > 0) + { dim = list->front()->values.size(); } // Write point coordinates out as unstructured grid - for (size_t pi = 0; pi < list->size(); pi++) { + for (size_t pi = 0; pi < list->size(); pi++) + { Point *p = list->at(pi); // NOTE: modified to be put on top of 2D plots - for (size_t ri = 0; ri < (restrict_to_2D ? 2 : dim); ri++) { + for (size_t ri = 0; ri < (restrict_to_2D ? 2 : dim); ri++) + { f << p->values[get_vtk_index(ri)] << "\t"; } f << "0.0"; @@ -1499,28 +1683,34 @@ namespace m3D { f << "POINT_DATA " << list->size() << endl; f << "SCALARS " << var_name << " FLOAT" << endl; f << "LOOKUP_TABLE default" << endl; - for (size_t pi = 0; pi < list->size(); pi++) { + for (size_t pi = 0; pi < list->size(); pi++) + { f << weights[pi] << endl; } } - template + template void VisitUtils::write_shift_vectors(const string &filename, - FeatureSpace *fs, - bool spatial_only) { - vector > origins, vectors; + FeatureSpace *fs, + bool spatial_only) + { + vector> origins, vectors; // Write point coordinates out as unstructured grid - for (size_t pi = 0; pi < fs->points.size(); pi++) { + for (size_t pi = 0; pi < fs->points.size(); pi++) + { Point *p = fs->points[pi]; - if (spatial_only) { + if (spatial_only) + { origins.push_back(p->coordinate); vectors.push_back(fs->spatial_component(p->shift)); - } else { + } + else + { origins.push_back(p->values); vectors.push_back(p->shift); @@ -1530,12 +1720,13 @@ namespace m3D { write_vectors_vtk(filename, origins, vectors); } - template + template void VisitUtils::write_weight_function_response(const string &filename, - FeatureSpace *fs, - WeightFunction *weight_function, - bool write_legacy) { + FeatureSpace *fs, + WeightFunction *weight_function, + bool write_legacy) + { const CoordinateSystem *cs = fs->coordinate_system; int nx, ny, nz; VisitUtils::get_vtk_image_dimensions(cs, nx, ny, nz); @@ -1548,9 +1739,10 @@ namespace m3D { variable->SetNumberOfValues(nx * ny * nz); variable->FillComponent(0, 0); - for (size_t i = 0; i < fs->points.size(); ++i) { + for (size_t i = 0; i < fs->points.size(); ++i) + { typename Point::ptr p = fs->points.at(i); - double value = (double) weight_function->operator()(p); + double value = (double)weight_function->operator()(p); int gx, gy, gz; VisitUtils::get_vtk_gridpoint(p->gridpoint, gx, gy, gz); int gridIndex = to_single_index(cs, nx, ny, nz, gx, gy, gz); @@ -1559,10 +1751,11 @@ namespace m3D { rgrid->GetPointData()->AddArray(variable); // Write out - if (write_legacy) { + if (write_legacy) + { // ASCII vtkSmartPointer - writer = vtkSmartPointer::New(); + writer = vtkSmartPointer::New(); std::string fn = filename + ".vtk"; writer->SetFileName(fn.c_str()); @@ -1572,10 +1765,12 @@ namespace m3D { writer->SetInputData(rgrid); #endif writer->Write(); - } else { + } + else + { // XML vtkSmartPointer - xmlWriter = vtkSmartPointer::New(); + xmlWriter = vtkSmartPointer::New(); std::string xml_fn = filename + "." + xmlWriter->GetDefaultFileExtension(); xmlWriter->SetFileName(xml_fn.c_str()); #if VTK_MAJOR_VERSION <= 5 @@ -1587,36 +1782,38 @@ namespace m3D { } // Clean up - for (size_t ci = 0; ci < coords.size(); ci++) { + for (size_t ci = 0; ci < coords.size(); ci++) + { coords.at(ci)->Delete(); } rgrid->Delete(); } - template + template void VisitUtils::write_multiarray_vtk(const std::string &filename, const std::string &variable_name, - const CoordinateSystem *cs, - MultiArray *array, - bool write_legacy) { + const CoordinateSystem *cs, + MultiArray *array, + bool write_legacy) + { class Functor : public MultiArray::ForEachFunctor { private: - vtkDoubleArray *m_array; const CoordinateSystem *m_cs; int nx, ny, nz; public: - - Functor(vtkDoubleArray *array, const CoordinateSystem *cs) : m_array(array), m_cs(cs) { + Functor(vtkDoubleArray *array, const CoordinateSystem *cs) : m_array(array), m_cs(cs) + { VisitUtils::get_vtk_image_dimensions(cs, nx, ny, nz); }; void - operator()(const vector &index, const T value) { + operator()(const vector &index, const T value) + { int gx, gy, gz; VisitUtils::get_vtk_gridpoint(index, gx, gy, gz); int gridIndex = VisitUtils::to_single_index(m_cs, nx, ny, nz, gx, gy, gz); @@ -1645,11 +1842,12 @@ namespace m3D { boost::filesystem::path path(filename); std::string basename = path.filename().stem().generic_string(); - if (write_legacy) { + if (write_legacy) + { // ASCII vtkSmartPointer - writer = vtkSmartPointer::New(); + writer = vtkSmartPointer::New(); std::string fn = basename + ".vtk"; writer->SetFileName(fn.c_str()); @@ -1660,11 +1858,13 @@ namespace m3D { writer->SetInputData(rgrid); #endif writer->Write(); - } else { + } + else + { // XML vtkSmartPointer - xmlWriter = vtkSmartPointer::New(); + xmlWriter = vtkSmartPointer::New(); std::string xml_fn = basename + "." + xmlWriter->GetDefaultFileExtension(); xmlWriter->SetFileName(xml_fn.c_str()); @@ -1679,19 +1879,21 @@ namespace m3D { // Clean up - for (size_t ci = 0; ci < coords.size(); ci++) { + for (size_t ci = 0; ci < coords.size(); ci++) + { coords.at(ci)->Delete(); } rgrid->Delete(); }; - template + template void VisitUtils::write_multiarray_vtk(const std::string &filename, const std::string &variable_name, - const CoordinateSystem *cs, - const MultiArray *array) { + const CoordinateSystem *cs, + const MultiArray *array) + { class Functor : public MultiArray::ForEachFunctor { @@ -1700,27 +1902,35 @@ namespace m3D { bool m_write_coords; public: - Functor(std::ofstream &stream, const CoordinateSystem *cs, bool write_coords) - : m_stream(stream), m_coord_system(cs), m_write_coords(write_coords) { + : m_stream(stream), m_coord_system(cs), m_write_coords(write_coords) + { } void - operator()(const vector &index, const bool value) { - if (m_write_coords) { - for (size_t ri = 0; ri < 3; ri++) { - if (ri < index.size()) { + operator()(const vector &index, const bool value) + { + if (m_write_coords) + { + for (size_t ri = 0; ri < 3; ri++) + { + if (ri < index.size()) + { vector coord(index.size()); m_coord_system->lookup(index, coord); m_stream << coord[get_vtk_index(ri)] << "\t"; - } else { + } + else + { m_stream << 0.0 << "\t"; } } m_stream << endl; - } else { + } + else + { m_stream << (value ? 1.0 : 0.0) << endl; } } @@ -1752,8 +1962,8 @@ namespace m3D { Functor f2(f, cs, false); array->for_each(&f2); }; - } -} + } // namespace utils +} // namespace m3D #endif #endif \ No newline at end of file From 97ee9ac7ed37e663dcb2a9d080552a48ca168d80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 6 Jul 2020 16:49:41 +0100 Subject: [PATCH 56/65] Checking in missed changes. --- src/executables/meanie3D-cfm2vtk.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/executables/meanie3D-cfm2vtk.cpp b/src/executables/meanie3D-cfm2vtk.cpp index 2a9c246..9dc0584 100644 --- a/src/executables/meanie3D-cfm2vtk.cpp +++ b/src/executables/meanie3D-cfm2vtk.cpp @@ -176,11 +176,13 @@ void convert_clusters(const string &filename, //VisitUtils::write_clusters_vtr(list, cs, list->source_file, true, false, true); // VisitUtils::write_clusters_vtu(list, cs, list->source_file, 5, true, extract_skin, write_as_xml); + cout << "Writing clusters as vtk/vtu" << endl; VisitUtils::write_clusters_vtu(list, cs, list->source_file, 5, true, extract_skin, write_as_xml); if (write_displacement_vectors) { - vector > origins; + cout << "Writing displacement vectors" << endl; + vector> origins; vector > displacements; for (size_t i = 0; i < list->size(); i++) { Cluster::ptr c = list->clusters.at(i); @@ -195,10 +197,12 @@ void convert_clusters(const string &filename, } string centers_path = path.filename().stem().string() + "-centers.vtk"; + cout << "Writing cluster centers" << endl; VisitUtils::write_geometrical_cluster_centers_vtk(centers_path, list->clusters); delete list; delete cs; + cout << "Done." << endl; } void convert_composite(const string &filename, From 14bb2e3264a86e4c96a9d8e03d348be437ed3c54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 6 Jul 2020 16:49:55 +0100 Subject: [PATCH 57/65] Work on README --- README.md | 103 +++++++++++++++++++++--------------------------------- 1 file changed, 39 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 32657a5..e10c680 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,22 @@ -# meanie3D +# meanie3D -This project provides a very generic implementation of mean-shift clustering on NetCDF data sets. -The data sets need to follow the [CF-Metadata](http://cfconventions.org). The mean-shift algorithm -is a robust unsupervised clustering algorithm with a wide range of applications. Meanie3D provides -a multivariate implementation with no limits as to the number of variables that can be used in -constructing the Featurespace. +This project provides a generic implementation of [mean-shift clustering](https://en.wikipedia.org/wiki/Mean_shift) on [CF-Metadata](http://cfconventions.org) compliant NetCDF data sets. The mean-shift algorithm is a robust unsupervised clustering algorithm with a wide range of applications. Meanie3D provides a multivariate implementation with no limits as to the number of variables that can be used in constructing the Featurespace. This software supports 2D as well as 3D data sets and comes with many configuration options. The code -was written as part of my efforts within [OASE HErZ group](http://www.herz-tb1.uni-bonn.de) and contains -some modules that are specific to this application as a result. If interest is there I might remove all -specific components at one point. Meanie3D is implemented largely as a C++ template library with only a -few object files (mostly from the numerical recipes library) for performance reason and uses OpenMP for -parallelisation. - -The software package comes with a single command in the form of a python script 'meanie3D' which makes -it very easy to use. The detection, tracking and post-processing steps (such as statistics, visualisation -of tracks, source data and clusters as well as the track graph (a simple, graphical representation of -a tracking dictionary for obtaining detailed view on the history of individual clusters and tracks) are -all controlled through a single configuration file. +was written as part of research efforts of the [OASE HErZ group](http://www.herz-tb1.uni-bonn.de) and contains some modules that are specific to this application as a result. -## License +Meanie3D is implemented largely as a C++ template library with only a few object files (mostly from the numerical recipes library) for performance reason and uses OpenMP for parallelisation. + +The software package comes with a python frontend `meanie3D` which makes it very easy to use. The detection, tracking and post-processing steps (such as statistics, visualisation of tracks, source data and clusters as well as the track graph (a simple, graphical representation of a tracking dictionary for obtaining detailed view on the history of individual clusters and tracks) are all controlled through a single configuration file. +## License Meanie3D comes under [[MIT license]]. ## Version - The latest stable version is v1.6.1. Versions are tagged. ## Build Instructions - ### Dependencies - Meanie3D comes with a number of dependencies that need to be installed prior to attempting installation: * [Boost 1.56 or better](http://www.boost.org) * [FLANN 1.8.0 or better](http://www.cs.ubc.ca/research/flann/) @@ -43,12 +29,9 @@ Meanie3D comes with a number of dependencies that need to be installed prior to * NetCDF4-python (try running @pip install netCDF4@ or download and install from http://unidata.github.io/netcdf4-python) * [libradolan](http://meteo-ubonn.github.io/radolan/) -Optional: - The following libraries may be used, if they are switched on with the appropriate flags * Data format - * [shapelib v1.3+](http://shapelib.maptools.org) - When present, certain utilities have additional options for outputting - data in ESRI shapefile format. + * [shapelib v1.3+](http://shapelib.maptools.org) - When present, certain utilities have additional options for writing out data in ESRI shapefile format. * Visualization: * [Imagemagick](https://imagemagick.org) * [gnuplot](http://gnuplot.sourceforge.net) @@ -58,7 +41,6 @@ The following libraries may be used, if they are switched on with the appropriat * [doxygen](https://www.doxygen.nl/download.html) ### Compiler Prerequisites - Meanie3D uses OpenMP by default (-DWITH_OPENMP=1) and requires an OpenMP enabled compiler, such as: * GNU 9.0 or better (Linux, Mac) * OpenMP/LLVM (Mac) - a clang implementation supporting OpenMP5 (http://clang-omp.github.io). Note that as of clang 10.0.0 OpenMP is supported out of the box, but you will still have to install the libomp library (`brew install libomp`) @@ -66,34 +48,18 @@ Meanie3D uses OpenMP by default (-DWITH_OPENMP=1) and requires an OpenMP enabled You can disable the OpenMP implementation adding the flags `-DWITH_OPENMP=0` to your cmake call (does nothing when using presets). ### Build and install the software +Meanie3D uses [cmake](https://cmake.org). Start by cloning the master branch (for an up-to date but possibly unstable version) or one of the stable releases. -Meanie3D uses CMAKE to generate makefiles. You can use CMAKE's abilities to generate IDE files if you prefer. Start -by cloning the master branch (for an up-to date but possibly unstable version) or one of the stable releases. -``` git clone https://github.com/meteo-ubonn/meanie3D.git -``` -TODO: revise handling of map data -If you want to download the OASE topology and mapdata file for visualisation, you can obtain this file by adding: -``` - git clone http://git.meteo.uni-bonn.de/git/oase-mapdata -``` -Create a build directory: -``` - mkdir meanie3D-make - cd meanie3D-make - cmake ../meanie3D -``` + cd meanie3D + mkdir build + cd build + cmake -DPRESET=fast-vtk .. + make install ### Build presets A number of presets are provided to make the process easier. Those are selected via the -DPRESET=\ flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available presets are: -* `docker` - * Code optimizations for build type 'Release' - * Core functions and python frontend -* `docker-vtk` - * Code optimizations for build type 'Release' - * Core functions and python frontend - * VTK output enabled. * `dev-core` * Code optimizations for build type 'Debug' * Core functions and python frontend @@ -142,8 +108,15 @@ flag to cmake. (Example: `cmake -DPRESET=dev-all ../meanie3D`). The available pr * RADOLAN, Satellite and KONRAD utilities * Tests * Documentation +* `docker` + * Code optimizations for build type 'Release' + * Core functions and python frontend +* `docker-vtk` + * Code optimizations for build type 'Release' + * Core functions and python frontend + * VTK output enabled. -The term "core functions" refers to the detection, tracking and track evaluation code. For local development in most cases, the preset `dev-vtk` is sufficient. For more aggressive optimisations, use `prod-vtk`. If you just need the core functions and none of the visuals, choose the `core` sets (`dev-core` or `prod-core`). +The term "core functions" refers to the detection, tracking and track evaluation code. For local development in most cases, the preset `dev-vtk` is sufficient. For more aggressive optimisations, use `prod-vtk`. If you just need the core functions and none of the visuals, choose the `core` sets (`dev-core` or `prod-core`). For best performance, use the `fast` presets. ### Available build types The following build types are available: @@ -153,18 +126,15 @@ The following build types are available: * MinSizeRel (minimal footprint, aggressive optimizations, no debug symbols) To select a build type, use the flag `CMAKE_BUILD_TYPE` like so: -``` + cmake -DCMAKE_BUILD_TYPE=Release ../meanie3D -``` -*Note:* there have been some problems on Linux with aggressive optimization and NetCDF. Your mileage may vary. You should try the release build in any event, since it speeds up performance a lot. If you observe unexpected problems in reading/writing NetCDF files, you may have fallen victim to the problem and revert to standard build (leave the -DCMAKE_BUILD_TYPE=Debug). Once all dependencies are successfully resolved, install the product by calling the following: -``` - make install -``` -*Note:* When using presets, the preset defines the build type. Setting this flag together with a preset has no effect on the preset's choice. +*Note:* there have been some problems on Linux with aggressive optimization and NetCDF. Your mileage may vary. You should try the `Release` build type in any event, since it speeds up performance a lot. If you observe unexpected problems in reading/writing NetCDF files, you may have fallen victim to the problem and revert to standard build (leave the -DCMAKE_BUILD_TYPE=Debug). + +**Note:** When using presets, the preset defines the build type. Setting this flag together with a preset has no effect on the preset's choice. ### Options -If you would like to select your own options, you can leave the PRESET parameter and switch things on and off. When using presets, the preset defines the options. Setting any of the following options together with a -DPRESET=\ option does not alter the preset's settings. +If you would like to select your own options, you can leave the PRESET parameter and switch components on and off selectively (Remember: setting any of the following options together with a -DPRESET=\ option does not alter the preset's settings). #### `-DWITH_VTK=ON/OFF` Because of the large footprint of the VTK package, the visualization code is disabled by default. While visualization is not necessary to run the algorithm, it can be useful to develop your parameters to have visual queues as to what is happening. Setting this flag will result in the following changes: @@ -188,9 +158,8 @@ In order to speed the process up, meanie3D uses OpenMP to parallelize it's compu #### `-DWITH_TESTS=ON/OFF` Meanie3D has a number of regression tests, that cover the core algorithms and collection classes. This will become important to you if you should decide to work on the core algorithms yourself. The tests are a good method of making sure you haven't broken anything critical. The unit tests can then be run by calling -``` + make test -``` #### `-DWITH_PYTHON=ON/OFF` In addition to the C++ binaries, the installation also uses pip to put a python package in place. This adds an executable simply called `meanie3D`, which is a front-end to the core functions. It allows you to put your clustering and tracking parameters down in the form of a configuration file. The entire @@ -198,11 +167,18 @@ pipeline is handled based on this configuration file. *This is the recommended w TODO: location of user manual +##### Python2 vs Python3 + + Because the meanie3D python package contains code that is executed in Visit's own python CLI, you will have to use a python2 version if you want the visualisation code switched on. For your own visualisations from VTK output generated by the software, you may run Python3. In order to configure a specific python2 installation, use the following hint: + + cmake -DPython2_ROOT_DIR=/path/to/python/root ... + + For Python3 use ```-DPython3_ROOT_DIR``` instead. + #### `-DWITH_DOCS=ON/OFF` In order to start developing your own Meanie3D code, it might be useful to have API documentation of the various classes in the project. If you have doxygen installed, you can call the following make command to create a browsable HTML documentation in doc/html (open the file index.html). -``` + make docs -``` #### `-DWITH_RADOLAN_UTILS=ON/OFF` This will result in compilation of the `meanie3D-radolan2cfm` utility, which converts files in RADOLAN format to a cf-metadata compliant netCDF file, which then can be used to run the tracking. @@ -239,9 +215,8 @@ When this flag is set, a file is written out which contains the cluster modes. N ### Uninstall the software Navigate to the build directory. Invoke the following command: -``` -make uninstall -``` + + make uninstall The command will remove the C++ binaries, meanie3D dynamic library and executables from your system. From eeef57408ba37eac17c424c44fce3d397772d084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Mon, 6 Jul 2020 22:22:49 +0100 Subject: [PATCH 58/65] Options cleanup --- .../clustering/detection_commandline.h | 537 +++++++++--------- 1 file changed, 273 insertions(+), 264 deletions(-) diff --git a/include/meanie3D/clustering/detection_commandline.h b/include/meanie3D/clustering/detection_commandline.h index a2c925c..b2e0e10 100644 --- a/include/meanie3D/clustering/detection_commandline.h +++ b/include/meanie3D/clustering/detection_commandline.h @@ -6,7 +6,7 @@ */ #ifndef M3D_DETECTION_COMMANDLINE_H -#define M3D_DETECTION_COMMANDLINE_H +#define M3D_DETECTION_COMMANDLINE_H #include #include @@ -26,8 +26,8 @@ #include "detection.h" -namespace m3D { - +namespace m3D +{ /** * Add the command line description for the detection related paramaters. @@ -35,219 +35,155 @@ namespace m3D { * @param desc * @param params */ - template + template void add_detection_options(program_options::options_description &desc, - const detection_params_t ¶ms) { + const detection_params_t ¶ms) + { desc.add_options() - ("file,f", - program_options::value(), - "CF-Metadata compliant NetCDF-file") - ("output,o", - program_options::value(), - "Name of output file for clustering results") - ("previous-output,p", - program_options::value(), - "Previous cluster result file. Used for tracking or to keep the " - "clustering more stable over time (--postprocess-with-previous-output)") - ("dimensions,d", - program_options::value(), - "Comma-separated list of the dimensions to be used. The program " - "expects dimension variables with identical names.") - ("variables,v", - program_options::value(), - "Comma-separated variables used to construct feature space. " - "Do not include dimension variables") - ("time-index,t", - program_options::value()->default_value(params.time_index), - "Index of the point in time you wish to use in files with a " - "time dimension. -1 means no time dimension in the variables.") - ("lower-thresholds", - program_options::value(), - "Comma-separated list var1=val,var2=val,... of lower thresholds. " - "Values below this are ignored when constructing feature space") - ("upper-thresholds", - program_options::value(), - "Comma-separated list var1=val,var2=val,... of lower thresholds. " - "Values above this are ignored when constructing feature space") - ("replacement-values", - program_options::value(), - "Comma-separated list var1=val,var2=val,... of values to replace " - "missing values with in feature space construction. If no replacement " - "value is specified while even one variable is out of valid range at " - "one point, the whole point is discarded") - ("kernel-name,k", - program_options::value()->default_value(params.kernel_name), - "uniform,gauss,epanechnikov or none") - ("weight-function-name,w", - program_options::value()->default_value(params.weight_function_name), - "default,inverse,pow10,inverfc,oase or oaseci") - ("wwf-lower-threshold", - program_options::value()->default_value(params.wwf_lower_threshold), - "Lower threshold for weight function filter.") - ("wwf-upper-threshold", - program_options::value()->default_value(params.wwf_upper_threshold), - "Upper threshold for weight function filter.") - ("inline-tracking", - "If present, tracking step is performed immediately after " - "clustering. Required --previous-output and other " - "clustering parameters to be set (check meanie3D-track)") - ("replacement-filter", - program_options::value(), - "Comma-separated list varname--[percentage]," - "... to replace values with average of lowest/highest percent or " - "median of neighbours") - ("ci-comparison-file", - program_options::value(), - "File for calculating time trends for CI-score according to " - "Walker et al. 2012.") - ("ci-comparison-protocluster-file", - program_options::value(), - "Protoclusters from the comparison file") - ("ci-satellite-only", - "If present, only satellite values are used (original score), otherwise ") - ("ci-use-walker-mecikalski", - "If present, the original limits by Walker and Mecicalski are " - "used for CI score. If absent, the modified version is used") - ("ci-protocluster-scale", - program_options::value()->default_value(params.ci_protocluster_scale), - "Scale parameter for protocluster detection") - ("ci-protocluster-min-size", - program_options::value()->default_value(params.ci_protocluster_min_size), - "Minimum size of protoclusters") - ("coalesce-with-strongest-neighbour", - "If present, clusters are post-processed, coalescing each cluster " - "with their strongest neighbour") - ("postprocess-with-previous-output", - "If present, the --previous-output file is used to consolidate " - "current results. This is time consuming and has a propensity to " - "form larger clusters. Use with discretion.") - ("scale,s", - program_options::value()->default_value(params.scale), - "Scale parameter to pre-smooth the data with. Filter size is " - "calculated from this automatically.") - ("filter-size,l", - program_options::value()->default_value(0.0), - "Scale parameter to pre-smooth the data with. Scale parameter " - "is calculated from this automatically.") - ("ranges,r", - program_options::value(), - "Override the automatic bandwidth calculation with a set of given " - "bandwidths. Use in the order of (dim1,...dimN,var1,...,varN).") - ("min-cluster-size,m", - program_options::value < unsigned - int > ()->default_value(params.min_cluster_size), - "Discard clusters smaller than this number of points.") - ("previous-cluster-coverage-threshold", - program_options::value()->default_value(params.cluster_coverage_threshold), - "Minimum overlap in percent between current and previous clusters " - "to be taken into consideration.") - ("include-weight-function-in-results,i", - "Add a netcdf variable 'weight' to the result file, containing the " - "weight function response at each point in the feature-space") + ("file,f", program_options::value(), "CF-Metadata compliant NetCDF-file") + ("output,o", program_options::value(), "Name of output file for clustering results") + ("dimensions,d", program_options::value(), "Comma-separated list of the dimensions to be used. The program expects dimension variables with identical names.") + ("variables,v",program_options::value(),"Comma-separated variables used to construct feature space. Do not include dimension variables") + ("time-index,t",program_options::value()->default_value(params.time_index), "Index of the point in time you wish to use in files with a time dimension. -1 means no time dimension in the variables.") + ("lower-thresholds",program_options::value(),"Comma-separated list var1=val,var2=val,... of lower thresholds. Values below this are ignored when constructing feature space") + ("upper-thresholds", program_options::value(), "Comma-separated list var1=val,var2=val,... of lower thresholds. Values above this are ignored when constructing feature space") + ("replacement-values", program_options::value(),"Comma-separated list var1=val,var2=val,... of values to replace missing values with in feature space construction. If no replacement value is specified while even one variable is out of valid range at one point, the whole point is discarded") + ("replacement-filter", program_options::value(), "Comma-separated list varname--[percentage], to replace values with average of lowest/highest percent or median of neighbours") + ("kernel-name,k", program_options::value()->default_value(params.kernel_name), "uniform,gauss,epanechnikov or none") + ("weight-function-name,w", program_options::value()->default_value(params.weight_function_name), "default,inverse,pow10,inverfc,oase or oaseci") + ("wwf-lower-threshold",program_options::value()->default_value(params.wwf_lower_threshold), "Lower threshold for weight function filter.") + ("wwf-upper-threshold", program_options::value()->default_value(params.wwf_upper_threshold), "Upper threshold for weight function filter.") + ("scale,s", program_options::value()->default_value(params.scale), "Scale parameter to pre-smooth the data with. Filter size is calculated from this automatically.") + ("filter-size,l", program_options::value()->default_value(0.0), "Scale parameter to pre-smooth the data with. Scale parameter is calculated from this automatically.") + ("ranges,r", program_options::value(), "Override the automatic bandwidth calculation with a set of given bandwidths. Use in the order of (dim1,...dimN,var1,...,varN).") +#if WITH_SATELLITE + ("ci-comparison-file", program_options::value(), "File for calculating time trends for CI-score according to Walker et al. 2012.") + ("ci-comparison-protocluster-file", program_options::value(), "Protoclusters from the comparison file") + ("ci-satellite-only", "If present, only satellite values are used (original score), otherwise ") + ("ci-use-walker-mecikalski","used for CI score. If absent, the modified version is used") + ("ci-protocluster-scale", program_options::value()->default_value(params.ci_protocluster_scale), "Scale parameter for protocluster detection") + ("ci-protocluster-min-size", program_options::value()->default_value(params.ci_protocluster_min_size), "Minimum size of protoclusters") +#endif + ("inline-tracking", "If present, tracking step is performed immediately after clustering. Required --previous-output and other clustering parameters to be set (check meanie3D-track)") + ("coalesce-with-strongest-neighbour", "If present, clusters are post-processed, coalescing each cluster their strongest neighbour") + ("postprocess-with-previous-output", "If present, the --previous-output file is used to consolidate current results. This is time consuming and has a propensity to form larger clusters") + ("previous-output,p", program_options::value(), "Previous cluster result file. Used for tracking or to keep the clustering more stable over time (--postprocess-with-previous-output)") + ("previous-cluster-coverage-threshold", program_options::value()->default_value(params.cluster_coverage_threshold), "Minimum overlap in percent between current and previous clusters to be taken into consideration.") + ("min-cluster-size,m", program_options::value()->default_value(params.min_cluster_size), "Discard clusters smaller than this number of points.") + ("include-weight-function-in-results,i", "Add a netcdf variable 'weight' to the result file, containing the weight function response at each point in the feature-space") #if WITH_VTK - ("write-variables-as-vtk", - program_options::value(), - "Comma separated list of variables that should be written out " - "as VTK files (after applying scale/threshold)") - ("write-weight-function", - "write weight function out as .vtk file") - ("write-meanshift-vectors", - "write out .vtk files containing the meanshift vectors") - ("write-clusters-as-vtk", - "write clusters out as .vtk files") - ("write-cluster-modes", - "write the final meanshift modes in .vtk file format") - ("write-cluster-centers", - "write cluster centers out in .vtk file format") - ("write-cluster-weight-response", - "write out the clusters with weight responses as value") + ("write-variables-as-vtk", program_options::value(), "Comma separated list of variables that should be written out as VTK files (after applying scale/threshold)") + ("write-weight-function", "Write weight function out as .vtk file") + ("write-meanshift-vectors", "Write out .vtk files containing the meanshift vectors") + ("write-clusters-as-vtk", "Write clusters out as .vtk files") + ("write-cluster-modes", "Write the final meanshift modes in .vtk file format") + ("write-cluster-centers","Write cluster centers out in .vtk file format") + ("write-cluster-weight-response", "Write out the clusters with weight responses as value") #endif - ; + ; } - template + template void get_detection_parameters(program_options::variables_map vm, - detection_params_t ¶ms) { - if (vm.count("file") == 0) { + detection_params_t ¶ms) + { + if (vm.count("file") == 0) + { cerr << "Missing input file argument" << endl; exit(EXIT_FAILURE); } params.filename = vm["file"].as(); - try { + try + { params.output_filename = vm["output"].as(); - } catch (const boost::exception &e) { + } + catch (const boost::exception &e) + { cerr << "Missing parameter -o " << endl; exit(EXIT_FAILURE); } // Open NetCDF file NcFile *file = NULL; - try { + try + { file = new NcFile(params.filename, NcFile::read); - } catch (const netCDF::exceptions::NcException &e) { + } + catch (const netCDF::exceptions::NcException &e) + { cerr << "Error opening file '" << params.filename << "' : " << e.what() << endl; exit(EXIT_FAILURE); } // Extract dimensions - if (vm.count("dimensions") == 0) { + if (vm.count("dimensions") == 0) + { cerr << "Missing parameter --dimensions" << endl; exit(EXIT_FAILURE); } // parse dimension list - typedef boost::tokenizer > tokenizer; + typedef boost::tokenizer> tokenizer; boost::char_separator sep(","); tokenizer dim_tokens(vm["dimensions"].as(), sep); - for (tokenizer::iterator tok_iter = dim_tokens.begin(); tok_iter != dim_tokens.end(); ++tok_iter) { + for (tokenizer::iterator tok_iter = dim_tokens.begin(); tok_iter != dim_tokens.end(); ++tok_iter) + { const char *name = (*tok_iter).c_str(); params.dimensions.push_back(name); NcVar dimVar = file->getVar(name); - if (dimVar.isNull()) { + if (dimVar.isNull()) + { cerr << "No dimension variable '" << std::string(name) << "' exists in file " << params.filename << endl; exit(EXIT_FAILURE); } params.dimension_variables.push_back(name); } - params.parameters = params.parameters - + "dimensions=" + vm["dimensions"].as() + " "; + params.parameters = params.parameters + "dimensions=" + vm["dimensions"].as() + " "; // parse variables - if (vm.count("variables") == 0) { + if (vm.count("variables") == 0) + { cerr << "Missing mandatory parameter --variables" << endl; - exit(EXIT_FAILURE);; + exit(EXIT_FAILURE); + ; } tokenizer var_tokens(vm["variables"].as(), sep); - for (tokenizer::iterator tok_iter = var_tokens.begin(); tok_iter != var_tokens.end(); ++tok_iter) { + for (tokenizer::iterator tok_iter = var_tokens.begin(); tok_iter != var_tokens.end(); ++tok_iter) + { std::string name = *tok_iter; NcVar var = file->getVar(name); - if (var.isNull()) { + if (var.isNull()) + { cerr << "No variable '" << std::string(*tok_iter) << "' exists!" << endl; - exit(EXIT_FAILURE);; + exit(EXIT_FAILURE); + ; } params.variables.push_back(name); } - params.parameters += params.parameters - + "variables=" + vm["variables"].as() + " "; + params.parameters += params.parameters + "variables=" + vm["variables"].as() + " "; // time params.time_index = vm["time-index"].as(); // Convection filter index params.convection_filter_index = -1; - if (vm.count("convection-filter-variable") > 0) { + if (vm.count("convection-filter-variable") > 0) + { std::string cf_var_name = vm["convection-filter-variable"].as(); - for (int index = 0; index < params.variables.size(); index++) { + for (int index = 0; index < params.variables.size(); index++) + { std::string name = params.variables.at(index); - if (name == cf_var_name) { + if (name == cf_var_name) + { // give the index in the complete feature-space vector // not just the list of variables handed in params.convection_filter_index = params.dimensions.size() + index; } } - if (params.convection_filter_index < 0) { + if (params.convection_filter_index < 0) + { cerr << "Bad value for convection-filter-variable. Variable '" << cf_var_name << "' is not a feature space variable" << endl; @@ -256,58 +192,78 @@ namespace m3D { } // replacement filter - if (vm.count("replacement-filter") > 0) { + if (vm.count("replacement-filter") > 0) + { // format: variableName-replacement-mode[-percentage] // RX-median // RX-lowest-25 // RX-highest-50 std::string rf = vm["replacement-filter"].as(); tokenizer tokens(rf, sep); - for (tokenizer::iterator ti = tokens.begin(); ti != tokens.end(); ++ti) { + for (tokenizer::iterator ti = tokens.begin(); ti != tokens.end(); ++ti) + { tokenizer filter_tokens(*ti, boost::char_separator("-")); size_t i = 0; tokenizer::iterator fi = filter_tokens.begin(); typename ReplacementFilter::ReplacementMode mode; int variable_index = -1; float percentage = -1.0; - while (fi != filter_tokens.end()) { - if (i == 0) { + while (fi != filter_tokens.end()) + { + if (i == 0) + { std::string varName = *fi; - for (size_t vi = 0; vi < params.variables.size(); vi++) { - if (params.variables[vi] == varName) { + for (size_t vi = 0; vi < params.variables.size(); vi++) + { + if (params.variables[vi] == varName) + { variable_index = vi; break; } } - if (variable_index < 0) { + if (variable_index < 0) + { cerr << "Illegal variable name for --replacement-filter" << endl; - exit(EXIT_FAILURE);; + exit(EXIT_FAILURE); + ; } - } else if (i == 1) { + } + else if (i == 1) + { std::string modeName = *fi; - if (modeName == "median") { + if (modeName == "median") + { mode = ReplacementFilter::ReplaceWithMedian; - } else if (modeName == "lowest") { + } + else if (modeName == "lowest") + { mode = ReplacementFilter::ReplaceWithLowest; - } else if (modeName == "highest") { + } + else if (modeName == "highest") + { mode = ReplacementFilter::ReplaceWithHighest; - } else { + } + else + { cerr << "Illegal mode name for --replacement-filter" << endl; - exit(EXIT_FAILURE);; + exit(EXIT_FAILURE); + ; } - } else if (i == 2) { + } + else if (i == 2) + { std::string perc = *fi; - percentage = strtof(perc.c_str(), (char **) NULL) / 100.0f; + percentage = strtof(perc.c_str(), (char **)NULL) / 100.0f; } i++; fi++; } - if ((mode == ReplacementFilter::ReplaceWithHighest - || mode == ReplacementFilter::ReplaceWithLowest) - && percentage < 0) { + if ((mode == ReplacementFilter::ReplaceWithHighest || mode == ReplacementFilter::ReplaceWithLowest) && percentage < 0) + { cerr << "Missing percentage in --replacement-filter" << endl; - exit(EXIT_FAILURE);; + exit(EXIT_FAILURE); + ; } params.replacementFilterVariableIndex.push_back(variable_index); @@ -317,48 +273,55 @@ namespace m3D { } // parse ranges if there - if (vm.count("ranges") > 0) { + if (vm.count("ranges") > 0) + { tokenizer bw_tokens(vm["ranges"].as(), sep); - for (tokenizer::iterator tok_iter = bw_tokens.begin(); tok_iter != bw_tokens.end(); ++tok_iter) { + for (tokenizer::iterator tok_iter = bw_tokens.begin(); tok_iter != bw_tokens.end(); ++tok_iter) + { const char *bw = (*tok_iter).c_str(); - params.ranges.push_back((T) strtod(bw, (char **) NULL)); + params.ranges.push_back((T)strtod(bw, (char **)NULL)); } - if (params.ranges.size() != params.dimension_variables.size() + params.variables.size()) { - cerr << "Please provide " << params.dimension_variables.size() - + params.variables.size() << " bandwidth values" << endl; + if (params.ranges.size() != params.dimension_variables.size() + params.variables.size()) + { + cerr << "Please provide " << params.dimension_variables.size() + params.variables.size() << " bandwidth values" << endl; exit(EXIT_FAILURE); } - params.parameters = params.parameters - + "ranges=" + vm["ranges"].as(); + params.parameters = params.parameters + "ranges=" + vm["ranges"].as(); } // Lower Thresholds - if (vm.count("lower-thresholds") > 0) { + if (vm.count("lower-thresholds") > 0) + { boost::char_separator equals("="); tokenizer tokens(vm["lower-thresholds"].as(), sep); - for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) { + for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) + { std::string pair = *tok_iter; tokenizer subtokens(pair, equals); tokenizer::iterator subtoken_iter = subtokens.begin(); std::string variableName = *subtoken_iter; bool have_var = false; - for (int i = 0; i < params.variables.size() && !have_var; i++) { - if (params.variables[i] == variableName) { + for (int i = 0; i < params.variables.size() && !have_var; i++) + { + if (params.variables[i] == variableName) + { subtoken_iter++; - if (subtoken_iter == subtokens.end()) { + if (subtoken_iter == subtokens.end()) + { cerr << "Missing threshold value for variable " << variableName << endl; exit(EXIT_FAILURE); } const char *value = (*subtoken_iter).c_str(); - params.lower_thresholds[i] = boost::numeric_cast(strtod(value, (char **) NULL)); + params.lower_thresholds[i] = boost::numeric_cast(strtod(value, (char **)NULL)); have_var = true; } } - if (!have_var) { + if (!have_var) + { cerr << "No variable named " << variableName << " found. Check --lower-thresholds parameter" << endl; exit(EXIT_FAILURE); @@ -367,28 +330,34 @@ namespace m3D { } // Upper Thresholds - if (vm.count("upper-thresholds") > 0) { + if (vm.count("upper-thresholds") > 0) + { boost::char_separator equals("="); tokenizer tokens(vm["upper-thresholds"].as(), sep); - for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) { + for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) + { std::string pair = *tok_iter; tokenizer subtokens(pair, equals); tokenizer::iterator subtoken_iter = subtokens.begin(); std::string variableName = *subtoken_iter; bool have_var = false; - for (int i = 0; i < params.variables.size(); i++) { - if (params.variables[i] == variableName) { + for (int i = 0; i < params.variables.size(); i++) + { + if (params.variables[i] == variableName) + { subtoken_iter++; - if (subtoken_iter == subtokens.end()) { + if (subtoken_iter == subtokens.end()) + { cerr << "Missing threshold value for variable " << variableName << endl; exit(EXIT_FAILURE); } const char *value = (*subtoken_iter).c_str(); - params.upper_thresholds[i] = boost::numeric_cast(strtod(value, (char **) NULL)); + params.upper_thresholds[i] = boost::numeric_cast(strtod(value, (char **)NULL)); have_var = true; } } - if (!have_var) { + if (!have_var) + { cerr << "No variable named " << variableName << " found. Check --upper-thresholds parameter" << endl; exit(EXIT_FAILURE); @@ -397,10 +366,12 @@ namespace m3D { } // Fill values - if (vm.count("replacement-values") > 0) { + if (vm.count("replacement-values") > 0) + { boost::char_separator equals("="); tokenizer tokens(vm["replacement-values"].as(), sep); - for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) { + for (tokenizer::iterator tok_iter = tokens.begin(); tok_iter != tokens.end(); ++tok_iter) + { std::string pair = *tok_iter; tokenizer subtokens(pair, equals); tokenizer::iterator subtoken_iter = subtokens.begin(); @@ -410,40 +381,40 @@ namespace m3D { // should be removed to decouple this from netCDF layer. NcVar variable; - for (size_t i = 0; i < params.variables.size(); i++) { - if (params.variables[i] == variableName) { + for (size_t i = 0; i < params.variables.size(); i++) + { + if (params.variables[i] == variableName) + { variable = file->getVar(params.variables[i]); } } - if (variable.isNull()) { + if (variable.isNull()) + { cerr << "No variable named " << variableName << " found. Check --replacement-values parameter" << endl; exit(EXIT_FAILURE); } subtoken_iter++; - if (subtoken_iter == subtokens.end()) { + if (subtoken_iter == subtokens.end()) + { cerr << "Missing replacement value for variable " << variableName << endl; exit(EXIT_FAILURE); } const char *value = (*subtoken_iter).c_str(); - double doubleValue = strtod(value, (char **) NULL); + double doubleValue = strtod(value, (char **)NULL); params.replacement_values[variable.getId()] = doubleValue; } } - // Scale parameter params.scale = vm["scale"].as(); // Kernel params.kernel_name = vm["kernel-name"].as(); - if (!(params.kernel_name == "uniform" - || params.kernel_name == "epanechnikov" - || params.kernel_name == "gauss" - || params.kernel_name == "none")) { - cerr << "Illegal kernel name " << params.kernel_name << - ". Only 'none','uniform','gauss' or 'epanechnikov' are accepted." << endl; + if (!(params.kernel_name == "uniform" || params.kernel_name == "epanechnikov" || params.kernel_name == "gauss" || params.kernel_name == "none")) + { + cerr << "Illegal kernel name " << params.kernel_name << ". Only 'none','uniform','gauss' or 'epanechnikov' are accepted." << endl; exit(EXIT_FAILURE); } @@ -451,8 +422,7 @@ namespace m3D { params.weight_function_name = vm["weight-function-name"].as(); if (!(params.weight_function_name == "default" || params.weight_function_name == "inverse" || params.weight_function_name == "pow10" || params.weight_function_name == "inverfc" || params.weight_function_name == "oase" || params.weight_function_name == "oase-ci")) { - cerr << "Illegal weight function name " << params.weight_function_name << - ". Only 'default','inverse','pow10' or 'oase' are known." << endl; + cerr << "Illegal weight function name " << params.weight_function_name << ". Only 'default','inverse','pow10' or 'oase' are known." << endl; exit(EXIT_FAILURE); } params.wwf_lower_threshold = vm["wwf-lower-threshold"].as(); @@ -478,13 +448,16 @@ namespace m3D { params.include_weight_in_result = vm.count("include-weight-function-in-results") > 0; // Previous file - if (vm.count("previous-output") > 0) { + if (vm.count("previous-output") > 0) + { std::string previous = vm["previous-output"].as(); boost::filesystem::path previous_path(previous); - if (boost::filesystem::exists(previous_path) - && boost::filesystem::is_regular_file(previous_path)) { + if (boost::filesystem::exists(previous_path) && boost::filesystem::is_regular_file(previous_path)) + { params.previous_clusters_filename = new std::string(previous); - } else { + } + else + { cerr << "Illegal value for parameter --previous-output:" << "does not exist or is no regular file" << endl; exit(EXIT_FAILURE); @@ -495,16 +468,21 @@ namespace m3D { params.postprocess_with_previous_output = vm.count("postprocess-with-previous-output") > 0; // CI flags +#if WITH_SATELLITE params.ci_satellite_only = vm.count("ci-satellite-only") > 0; params.ci_use_walker_mecikalski = vm.count("ci-use-walker-mecikalski") > 0; // ci_comparison_file - if (vm.count("ci-comparison-file") > 0) { + if (vm.count("ci-comparison-file") > 0) + { std::string previous = vm["ci-comparison-file"].as(); boost::filesystem::path previous_path(previous); - if (boost::filesystem::exists(previous_path) && boost::filesystem::is_regular_file(previous_path)) { + if (boost::filesystem::exists(previous_path) && boost::filesystem::is_regular_file(previous_path)) + { params.ci_comparison_file = new std::string(previous); - } else { + } + else + { cerr << "FATAL:illegal value for parameter --ci-comparison-file" << ": does not exist or is no regular file" << endl; exit(EXIT_FAILURE); @@ -512,36 +490,43 @@ namespace m3D { } // ci-comparison-protocluster-file - if (vm.count("ci-comparison-protocluster-file") > 0) { + if (vm.count("ci-comparison-protocluster-file") > 0) + { std::string previous = vm["ci-comparison-protocluster-file"].as(); boost::filesystem::path previous_path(previous); - if (boost::filesystem::exists(previous_path) && boost::filesystem::is_regular_file(previous_path)) { + if (boost::filesystem::exists(previous_path) && boost::filesystem::is_regular_file(previous_path)) + { params.ci_comparison_protocluster_file = new std::string(previous); - } else { + } + else + { cerr << "FATAL:illegal value for parameter --ci-comparison-protocluster-file: " << "does not exist or is no regular file" << endl; exit(EXIT_FAILURE); } } - if (vm.count("ci-protocluster-scale") > 0) { + if (vm.count("ci-protocluster-scale") > 0) + { params.ci_protocluster_scale = vm["ci-protocluster-scale"].as(); } - if (vm.count("ci-protocluster-min-size") > 0) { + if (vm.count("ci-protocluster-min-size") > 0) + { params.ci_protocluster_min_size = vm["ci-protocluster-min-size"].as(); } +#endif // previous-cluster-coverage-threshold params.cluster_coverage_threshold = vm["previous-cluster-coverage-threshold"].as(); // min cluster size - params.min_cluster_size = vm["min-cluster-size"].as < unsigned - int > (); + params.min_cluster_size = vm["min-cluster-size"].as(); // Inline tracking? params.inline_tracking = vm.count("inline-tracking") > 0; - if (params.inline_tracking && params.previous_clusters_filename == NULL) { + if (params.inline_tracking && params.previous_clusters_filename == NULL) + { cerr << "FATAL:when --inline-tracking is set you must give " << "--previous-output as well" << endl; exit(EXIT_FAILURE); @@ -549,20 +534,26 @@ namespace m3D { // vtk-variables #if WITH_VTK - if (vm.count("write-variables-as-vtk") > 0) { + if (vm.count("write-variables-as-vtk") > 0) + { tokenizer bw_tokens(vm["write-variables-as-vtk"].as(), sep); - for (tokenizer::iterator tok_iter = bw_tokens.begin(); tok_iter != bw_tokens.end(); ++tok_iter) { + for (tokenizer::iterator tok_iter = bw_tokens.begin(); tok_iter != bw_tokens.end(); ++tok_iter) + { std::string name = *tok_iter; - try { + try + { NcVar var = file->getVar(name); - if (var.isNull()) { + if (var.isNull()) + { cerr << "Can't open variable " << name << " from NetCDF file. Check --write-variables-as-vtk" << endl; exit(EXIT_FAILURE); } params.vtk_variables.push_back(name); - } catch (const netCDF::exceptions::NcException &e) { + } + catch (const netCDF::exceptions::NcException &e) + { cerr << "Can't find variable " << name << " from NetCDF file. Check --write-variables-as-vtk" << endl; @@ -580,40 +571,52 @@ namespace m3D { * @param ctx pre-initialised detection context * @param vm */ - template + template void print_detection_params(const detection_params_t ¶ms, const detection_context_t &ctx, - const program_options::variables_map &vm) { + const program_options::variables_map &vm) + { cout << "\tinput file = " << params.filename << endl; - if (params.time_index >= 0) { + if (params.time_index >= 0) + { cout << "\tusing point in time at index " << params.time_index << " (timestamp=" << ctx.timestamp << ")" << endl; - } else { + } + else + { cout << "\ttime is not a variable dimension. Using timestamp " << ctx.timestamp << endl; } cout << "\tdimensions = " << vm["dimensions"].as() << endl; cout << "\tvariables = " << vm["variables"].as() << endl; - if (!params.ranges.empty()) { + if (!params.ranges.empty()) + { cout << "\tranges = " << params.ranges << endl; - } else { + } + else + { cout << "\tautomatic bandwidth selection" << endl; } - if (!params.lower_thresholds.empty()) { + if (!params.lower_thresholds.empty()) + { cout << "\tusing lower thresholds " << vm["lower-thresholds"].as() << endl; } - if (!params.upper_thresholds.empty()) { + if (!params.upper_thresholds.empty()) + { cout << "\tusing upper thresholds " << vm["upper-thresholds"].as() << endl; } - if (params.scale != Detection::NO_SCALE) { + if (params.scale != Detection::NO_SCALE) + { cout << "\tpre-smoothing data with scale parameter " << params.scale << " (kernel width = " << ctx.kernel_width << ")" << endl; - } else { + } + else + { cout << "\tno scale-space smoothing" << endl; } @@ -624,44 +627,52 @@ namespace m3D { cout << "\t\tupper weight-function threshold: " << params.wwf_upper_threshold << endl; - if (params.ci_comparison_file != NULL) { + if (params.ci_comparison_file != NULL) + { cout << "\tCI comparison file for time trends:" << *params.ci_comparison_file << endl; } - if (params.previous_clusters_filename != NULL) { + if (params.previous_clusters_filename != NULL) + { cout << "\tprevious file:" << *params.previous_clusters_filename << endl; } - if (params.postprocess_with_previous_output) { - if (vm.count("previous-cluster-coverage-threshold") > 0) { + if (params.postprocess_with_previous_output) + { + if (vm.count("previous-cluster-coverage-threshold") > 0) + { cout << "\tprevious file coverage threshold:" << params.cluster_coverage_threshold << endl; } } - if (params.min_cluster_size > 1) { + if (params.min_cluster_size > 1) + { cout << "\tminimum cluster size = " << params.min_cluster_size << endl; } - if (!params.replacementFilterVariableIndex.empty()) { + if (!params.replacementFilterVariableIndex.empty()) + { cout << "\treplacement filters on " << endl; - for (size_t i = 0; i < params.replacementFilterVariableIndex.size(); i++) { + for (size_t i = 0; i < params.replacementFilterVariableIndex.size(); i++) + { int rvi = params.replacementFilterVariableIndex[i]; cout << "\t\t" << params.variables[rvi] << " replace with "; typename ReplacementFilter::ReplacementMode mode = params.replacementFilterModes.at(rvi); float percent = params.replacementFilterPercentages.at(rvi) * 100.0; - switch (mode) { - case ReplacementFilter::ReplaceWithHighest: - cout << "highest " << percent << "%"; - break; - case ReplacementFilter::ReplaceWithLowest: - cout << "lowest " << percent << "%"; - break; - case ReplacementFilter::ReplaceWithMedian: - cout << "median"; - break; + switch (mode) + { + case ReplacementFilter::ReplaceWithHighest: + cout << "highest " << percent << "%"; + break; + case ReplacementFilter::ReplaceWithLowest: + cout << "lowest " << percent << "%"; + break; + case ReplacementFilter::ReplaceWithMedian: + cout << "median"; + break; } cout << " of neighboring points" << endl; } @@ -671,15 +682,14 @@ namespace m3D { << (params.spatial_range_only ? "yes" : "no") << endl; cout << "\tcoalesce results with strongest neighbor: " - << (params.coalesceWithStrongestNeighbour ? "yes" : "no") << - endl; + << (params.coalesceWithStrongestNeighbour ? "yes" : "no") << endl; cout << "\toutput written to file: " << params.output_filename << endl; #if WITH_VTK - if (!params.vtk_variables.empty()) { - cout << "\twriting out these variables as vtk after processing:" << - vm["write-variables-as-vtk"].as() << endl; + if (!params.vtk_variables.empty()) + { + cout << "\twriting out these variables as vtk after processing:" << vm["write-variables-as-vtk"].as() << endl; } cout << "\twriting weight function to vtk:" << (params.write_weight_function ? "yes" : "no") << endl; cout << "\twriting mean-shift vectors to vtk:" << (params.write_meanshift_vectors ? "yes" : "no") << endl; @@ -688,7 +698,6 @@ namespace m3D { cout << "\twriting cluster weights as vtk: " << (params.write_weight_response ? "yes" : "no") << endl; #endif } -} - -#endif /* M3D_DETECTION_COMMANDLINE_H */ +} // namespace m3D +#endif /* M3D_DETECTION_COMMANDLINE_H */ From c25c2dbf5e4975bfa0678736c30decb2d690b2a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 7 Jul 2020 12:41:55 +0100 Subject: [PATCH 59/65] Options improved --- CMakeLists.txt | 4 ++-- README.md | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b8a75a..169d18b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,7 +59,7 @@ OPTION(WITH_TESTS "Enable/Disable building of tests" OFF) OPTION(WITH_VTK "Enable/Disable visualisation code (requires VTK)" OFF) OPTION(WITH_VISUALISATION "Enable code that runs in Visit" OFF) OPTION(WITH_RADOLAN_UTILS "Compile with radolan adaptor (requires libradolan)" OFF) -OPTION(WITH_SATELLITE_UTILS "Compile with tools for satellite data" OFF) +OPTION(WITH_SATELLITE "Compile with tools for satellite data" OFF) OPTION(WITH_KONRAD_UTILS "Compile with binaries for KONRAD tracks" OFF) OPTION(WITH_DOCS "Compile source code documentation" OFF) @@ -73,7 +73,7 @@ SET(VISUALISATION_ENABLED ${WITH_VISUALISATION}) SET(TESTS_ENABLED ${WITH_TESTS}) SET(RADOLAN_ENABLED ${WITH_RADOLAN_UTILS}) SET(KONRAD_ENABLED ${WITH_KONRAD_UTILS}) -SET(SATELLITE_ENABLED ${WITH_SATELLITE_UTILS}) +SET(SATELLITE_ENABLED ${WITH_SATELLITE}) SET(DOCS_ENABLED ${WITH_DOCS}) IF(PRESET MATCHES "^docker$") diff --git a/README.md b/README.md index e10c680..60fac03 100644 --- a/README.md +++ b/README.md @@ -183,10 +183,11 @@ In order to start developing your own Meanie3D code, it might be useful to have #### `-DWITH_RADOLAN_UTILS=ON/OFF` This will result in compilation of the `meanie3D-radolan2cfm` utility, which converts files in RADOLAN format to a cf-metadata compliant netCDF file, which then can be used to run the tracking. -#### `-DWITH_SATELLITE_UTILS=ON/OFF` -The package comes with binaries to perform some conversion on satellite data. Those binaries were provided in the context of research work for the OASE project. The following binaries will be provided if this flag is set: +#### `-DWITH_SATELLITE=ON/OFF` +When this option is switched on, the package comes with binaries to perform some conversion on satellite data. Those binaries were provided in the context of research work for the OASE project. The following binaries will be provided if this flag is set: * `meanie3D-satconv` - Converts spectral radiance to equivalent brightness temperature or vice versa * `meanie3D-parallax_correction` - Applies parallax correction to mseviri satellite data in OASE composite files. +In addition the detection code has special options for calculating a convective initiation score (Mecikalski, John R., and Kristopher M. Bedka. “Forecasting Convective Initiation by Monitoring the Evolution of Moving Cumulus in Daytime GOES Imagery.” Monthly Weather Review 134, no. 1 (2006). https://doi.org/10.1175/MWR3062.1.). This can be used to track convective cells in SEVIRI satellite data. #### `-DWITH_KONRAD_UTILS=ON/OFF` The package comes with a tool `meanie3D-trackstats-conrad` which analyses KONRAD tracks in a way that makes the data comparable to meanie3D data. This is a specialized tool developed in the context of the OASE project. From d591b4c76f9513d173f38c4618565fa995124dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 7 Jul 2020 13:21:13 +0100 Subject: [PATCH 60/65] Fix includes. --- include/meanie3D/filters/convection_filter.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/meanie3D/filters/convection_filter.h b/include/meanie3D/filters/convection_filter.h index acd3537..aa77b50 100644 --- a/include/meanie3D/filters/convection_filter.h +++ b/include/meanie3D/filters/convection_filter.h @@ -25,6 +25,7 @@ #define M3D_REFLECTIVITYCONVECTIONFILTER_H #include +#include #include #include @@ -44,7 +45,7 @@ namespace m3D { { private: - vector m_bandwidth; + std::vector m_bandwidth; size_t m_index_of_z; T m_convective_threshold; T m_critical_delta_z; From a5c2f8b4c0be59707b9cecb0302c22e0228af5ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Wed, 8 Jul 2020 15:18:11 +0100 Subject: [PATCH 61/65] Corrected header file naming. --- CMakeLists.txt | 2 +- .../meanie3D/weights/{inv_erfc_weights.h => inv_erfc_weight.h} | 0 include/meanie3D/weights/weight_function_factory_impl.h | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename include/meanie3D/weights/{inv_erfc_weights.h => inv_erfc_weight.h} (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 169d18b..2f00b90 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -579,7 +579,7 @@ ADD_LIBRARY(meanie3D SHARED include/meanie3D/weights/exp10_weight.h include/meanie3D/weights/inverse_default.h include/meanie3D/weights/oase_weights.h - include/meanie3D/weights/inv_erfc_weights.h + include/meanie3D/weights/inv_erfc_weight.h include/meanie3D/weights/weight_function.h include/meanie3D/weights/weight_function_factory.h include/meanie3D/weights/weight_function_factory_impl.h diff --git a/include/meanie3D/weights/inv_erfc_weights.h b/include/meanie3D/weights/inv_erfc_weight.h similarity index 100% rename from include/meanie3D/weights/inv_erfc_weights.h rename to include/meanie3D/weights/inv_erfc_weight.h diff --git a/include/meanie3D/weights/weight_function_factory_impl.h b/include/meanie3D/weights/weight_function_factory_impl.h index 0e18fcb..2bd51e3 100644 --- a/include/meanie3D/weights/weight_function_factory_impl.h +++ b/include/meanie3D/weights/weight_function_factory_impl.h @@ -30,7 +30,7 @@ #include "default_weights.h" #include "inverse_default.h" #include "exp10_weight.h" -#include "inv_erfc_weights.h" +#include "inv_erfc_weight.h" #include "weight_function_factory.h" From ccbdc554508adda4694e6f40da275094765000c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Wed, 8 Jul 2020 15:18:52 +0100 Subject: [PATCH 62/65] Formatting. --- .../meanie3D/clustering/cluster_utils_impl.h | 178 +++++++++--------- include/meanie3D/clustering/detection_impl.h | 3 +- 2 files changed, 95 insertions(+), 86 deletions(-) diff --git a/include/meanie3D/clustering/cluster_utils_impl.h b/include/meanie3D/clustering/cluster_utils_impl.h index 7c0ef5f..9d4be90 100644 --- a/include/meanie3D/clustering/cluster_utils_impl.h +++ b/include/meanie3D/clustering/cluster_utils_impl.h @@ -21,7 +21,6 @@ * SOFTWARE. */ - #ifndef M3D_CLUSTERUTILS_IMPL_H #define M3D_CLUSTERUTILS_IMPL_H @@ -31,31 +30,37 @@ #include -namespace m3D { +namespace m3D +{ - template + template ClusterUtils::ClusterUtils(float merge_threshold) - : m_merge_threshold(merge_threshold) { + : m_merge_threshold(merge_threshold) + { } - template + template void ClusterUtils::filter_with_previous_clusters(typename ClusterList::ptr previous, typename ClusterList::ptr current, - CoordinateSystem *coord_system, - WeightFunction *weight_function, - const Verbosity verbosity) { + CoordinateSystem *coord_system, + WeightFunction *weight_function, + const Verbosity verbosity) + { using utils::SimpleMatrix; using namespace utils::vectors; - if (verbosity >= VerbosityNormal) { + if (verbosity >= VerbosityNormal) + { cout << endl << "Filtering with previous results ..." << endl; start_timer(); } // sanity check - if (current->clusters.size() == 0 || previous->clusters.size() == 0) { - if (verbosity >= VerbosityNormal) { + if (current->clusters.size() == 0 || previous->clusters.size() == 0) + { + if (verbosity >= VerbosityNormal) + { cout << "Nothing to do." << endl; return; } @@ -71,44 +76,39 @@ namespace m3D { typename SimpleMatrix::matrix_t coverNewByOld = SimpleMatrix::create_matrix(new_count, old_count); // compute overlap - size_t n, m; - boost::progress_display *progress = NULL; - - if (verbosity >= VerbosityNormal) { + if (verbosity >= VerbosityNormal) + { progress = new boost::progress_display(2 * old_count); } - - for (m = 0; m < old_count; m++) { + size_t n, m; + for (m = 0; m < old_count; m++) + { typename Cluster::ptr oldCluster = previous->clusters[m]; - - if (verbosity >= VerbosityNormal) { + if (verbosity >= VerbosityNormal) + { progress->operator++(); } - - for (n = 0; n < new_count; n++) { + for (n = 0; n < new_count; n++) + { typename Cluster::ptr newCluster = current->clusters[n]; - // Calculate overlap - T overlap = index_of_previous.occupation_ratio(newCluster, oldCluster); - - if (overlap > 0 && verbosity >= VerbosityDetails) { + if (overlap > 0 && verbosity >= VerbosityDetails) + { printf("old #%4lu with new #%4lu overlap = %3.2f\n", oldCluster->id, newCluster->id, overlap); } - coverNewByOld[n][m] = overlap; } } // for each new blob check coverage of old blobs - typedef set::ptr> cset_t; - cset_t erased, merged; - - for (m = 0; m < previous->clusters.size(); m++) { - if (verbosity >= VerbosityNormal) { + for (m = 0; m < previous->clusters.size(); m++) + { + if (verbosity >= VerbosityNormal) + { progress->operator++(); } @@ -117,38 +117,39 @@ namespace m3D { // figure out the largest candidate // TODO: do we still need this here? - for (n = 0; n < current->clusters.size(); n++) { + for (n = 0; n < current->clusters.size(); n++) + { T overlap = coverNewByOld[n][m]; - - if (overlap >= 0.33) { + if (overlap >= 0.33) + { candidates.push_back(n); } } - if (candidates.size() > 1) { - if (verbosity >= VerbosityDetails) { + if (candidates.size() > 1) + { + if (verbosity >= VerbosityDetails) + { printf("Old cluster ID#%4lu seems to have split into new clusters IDs ", old_cluster->id); - for (int i = 0; i < candidates.size(); i++) { + for (int i = 0; i < candidates.size(); i++) + { typename Cluster::ptr c = current->clusters[candidates[i]]; printf("#%lu ", c->id); } cout << endl; } - - typename Cluster::ptr merged_cluster - = new Cluster(old_cluster->mode, old_cluster->spatial_rank()); + typename Cluster::ptr merged_cluster = new Cluster(old_cluster->mode, old_cluster->spatial_rank()); merged_cluster->id = ++current_id; vector mode(previous->rank(), 0.0); - // merge with those candidates, that are direct neighbours - // neglect those, that have no direct boundary with any of - // the other candidates - + // merge with those candidates, that are direct neighbours. Neglect those that have + // no direct boundary with any of the other candidates size_t num_picked_candidates = 0; - - for (int i = 0; i < candidates.size(); i++) { + for (int i = 0; i < candidates.size(); i++) + { bool have_boundary = true; - if (have_boundary) { + if (have_boundary) + { typename Cluster::ptr c = current->clusters[candidates[i]]; merged_cluster->add_points(c->get_points()); mode += c->mode; @@ -156,54 +157,53 @@ namespace m3D { num_picked_candidates++; } } - - if (num_picked_candidates > 1) { - merged_cluster->mode = mode / ((T) num_picked_candidates); + if (num_picked_candidates > 1) + { + merged_cluster->mode = mode / ((T)num_picked_candidates); merged.insert(merged_cluster); - } else { + } + else + { delete merged_cluster; } } } // remove the erased ones and add the merged ones - typename cset_t::iterator ci; - - for (ci = erased.begin(); ci != erased.end(); ++ci) { + for (ci = erased.begin(); ci != erased.end(); ++ci) + { typename Cluster::ptr c = *ci; - typename Cluster::list::iterator di = find(current->clusters.begin(), current->clusters.end(), c); - current->clusters.erase(di); - delete c; } - - for (ci = merged.begin(); ci != merged.end(); ci++) { + for (ci = merged.begin(); ci != merged.end(); ci++) + { typename Cluster::ptr c = *ci; - current->clusters.push_back(c); } - - if (verbosity >= VerbosityNormal) { - cout << " done. (Found " << current->clusters.size() << " clusters in " << stop_timer() << " seconds)" << - endl; - + if (verbosity >= VerbosityNormal) + { + cout << " done. (Found " << current->clusters.size() << " clusters in " << stop_timer() << " seconds)" << endl; delete progress; } } - template + template void ClusterUtils::replace_points_from_datastore(typename ClusterList::ptr list, - typename DataStore::ptr dataStore) { - for (size_t ci = 0; ci < list->size(); ci++) { + typename DataStore::ptr dataStore) + { + for (size_t ci = 0; ci < list->size(); ci++) + { typename Cluster::ptr c = list->clusters.at(ci); typename Point::list::iterator pi; - for (pi = c->get_points().begin(); pi != c->get_points().end(); ++pi) { + for (pi = c->get_points().begin(); pi != c->get_points().end(); ++pi) + { typename Point::ptr p = *pi; - for (int vi = 0; vi < dataStore->rank(); vi++) { + for (int vi = 0; vi < dataStore->rank(); vi++) + { bool isInRange = false; bool isValid = false; T value = dataStore->get(vi, p->gridpoint, isInRange, isValid); @@ -213,25 +213,31 @@ namespace m3D { } } - template + template void ClusterUtils::obtain_margin_flag(typename ClusterList::ptr list, - typename FeatureSpace::ptr fs) { + typename FeatureSpace::ptr fs) + { vector dims = fs->coordinate_system->get_dimension_sizes(); ArrayIndex index(dims, fs->points, false); typename Cluster::list::iterator ci; - for (ci = list->clusters.begin(); ci != list->clusters.end(); ++ci) { + for (ci = list->clusters.begin(); ci != list->clusters.end(); ++ci) + { typename Cluster::ptr c = *ci; typename Point::list::iterator pi; - for (pi = c->get_points().begin(); pi != c->get_points().end() && !c->has_margin_points(); ++pi) { + for (pi = c->get_points().begin(); pi != c->get_points().end() && !c->has_margin_points(); ++pi) + { typename Point::ptr p = *pi; - if (!p->isOriginalPoint) continue; + if (!p->isOriginalPoint) + continue; typename Point::list neighbors; neighbors = index.find_neighbours(p->gridpoint, 1); typename Point::list::iterator ni; - for (ni = neighbors.begin(); ni != neighbors.end(); ++ni) { + for (ni = neighbors.begin(); ni != neighbors.end(); ++ni) + { typename Point::ptr n = *ni; - if (fs->off_limits()->get(n->gridpoint)) { + if (fs->off_limits()->get(n->gridpoint)) + { c->set_has_margin_points(true); break; } @@ -240,27 +246,31 @@ namespace m3D { } } - template + template void - ClusterUtils::provideUuids(typename ClusterList::ptr list, uuid_t &uuid) { + ClusterUtils::provideUuids(typename ClusterList::ptr list, uuid_t &uuid) + { typename Cluster::list::iterator ci; - for (ci = list->clusters.begin(); ci != list->clusters.end(); ++ci) { + for (ci = list->clusters.begin(); ci != list->clusters.end(); ++ci) + { typename Cluster::ptr c = *ci; uuid = m3D::nextUuid(uuid); c->uuid = uuid; } } - template + template void - ClusterUtils::provideIds(typename ClusterList::ptr list, id_t &id) { + ClusterUtils::provideIds(typename ClusterList::ptr list, id_t &id) + { typename Cluster::list::iterator ci; - for (ci = list->clusters.begin(); ci != list->clusters.end(); ++ci) { + for (ci = list->clusters.begin(); ci != list->clusters.end(); ++ci) + { typename Cluster::ptr c = *ci; id = m3D::nextId(id); c->id = id; } } -} +} // namespace m3D #endif diff --git a/include/meanie3D/clustering/detection_impl.h b/include/meanie3D/clustering/detection_impl.h index 93cc092..5040f20 100644 --- a/include/meanie3D/clustering/detection_impl.h +++ b/include/meanie3D/clustering/detection_impl.h @@ -407,8 +407,7 @@ namespace m3D { ctx.clusters->highest_uuid = uuid; // Collate with previous clusters, if provided - if (params.previous_clusters_filename != NULL - && params.postprocess_with_previous_output) { + if (params.postprocess_with_previous_output && params.previous_clusters_filename != NULL) { cout << endl << "Collating with previous results:" << endl; if (params.verbosity >= VerbosityDetails) ctx.clusters->print(); From a6c9fe358d302b3c89fee13bdacb379ed9eeee60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Wed, 8 Jul 2020 17:42:18 +0100 Subject: [PATCH 63/65] Added 'No weight' weight function. --- .../clustering/detection_commandline.h | 23 +++++--- include/meanie3D/weights/no_weight.h | 53 +++++++++++++++++++ include/meanie3D/weights/weight_function.h | 14 ++--- .../weights/weight_function_factory_impl.h | 5 ++ 4 files changed, 83 insertions(+), 12 deletions(-) create mode 100644 include/meanie3D/weights/no_weight.h diff --git a/include/meanie3D/clustering/detection_commandline.h b/include/meanie3D/clustering/detection_commandline.h index b2e0e10..c2e47a6 100644 --- a/include/meanie3D/clustering/detection_commandline.h +++ b/include/meanie3D/clustering/detection_commandline.h @@ -23,6 +23,7 @@ #include #include #include +#include #include "detection.h" @@ -51,7 +52,11 @@ namespace m3D ("replacement-values", program_options::value(),"Comma-separated list var1=val,var2=val,... of values to replace missing values with in feature space construction. If no replacement value is specified while even one variable is out of valid range at one point, the whole point is discarded") ("replacement-filter", program_options::value(), "Comma-separated list varname--[percentage], to replace values with average of lowest/highest percent or median of neighbours") ("kernel-name,k", program_options::value()->default_value(params.kernel_name), "uniform,gauss,epanechnikov or none") - ("weight-function-name,w", program_options::value()->default_value(params.weight_function_name), "default,inverse,pow10,inverfc,oase or oaseci") + #if WITH_SATELLITE + ("weight-function-name,w", program_options::value()->default_value(params.weight_function_name), "none, default, inverse, pow10, inverfc, oase or oaseci") + #else + ("weight-function-name,w", program_options::value()->default_value(params.weight_function_name), "none, default, inverse, pow10 or inverfc") + #endif ("wwf-lower-threshold",program_options::value()->default_value(params.wwf_lower_threshold), "Lower threshold for weight function filter.") ("wwf-upper-threshold", program_options::value()->default_value(params.wwf_upper_threshold), "Upper threshold for weight function filter.") ("scale,s", program_options::value()->default_value(params.scale), "Scale parameter to pre-smooth the data with. Filter size is calculated from this automatically.") @@ -411,18 +416,24 @@ namespace m3D // Kernel params.kernel_name = vm["kernel-name"].as(); - - if (!(params.kernel_name == "uniform" || params.kernel_name == "epanechnikov" || params.kernel_name == "gauss" || params.kernel_name == "none")) + std::set validKernels = {"uniform", "epanechnikov", "gauss", "none"}; + if (validKernels.find(params.kernel_name) == validKernels.end()) { - cerr << "Illegal kernel name " << params.kernel_name << ". Only 'none','uniform','gauss' or 'epanechnikov' are accepted." << endl; + cerr << "Illegal kernel name " << params.kernel_name << ". Use one of " << validKernels << endl; exit(EXIT_FAILURE); } // Weight Function params.weight_function_name = vm["weight-function-name"].as(); - if (!(params.weight_function_name == "default" || params.weight_function_name == "inverse" || params.weight_function_name == "pow10" || params.weight_function_name == "inverfc" || params.weight_function_name == "oase" || params.weight_function_name == "oase-ci")) + std:set validWeights = {"none", "default", "inverse", "pow10", "inverfc"}; + #if WITH_SATELLITE + validWeights.insert("oase"); + validWeights.insert("oase-ci"); + #endif + + if (validWeights.find(params.weight_function_name) == validWeights.end()) { - cerr << "Illegal weight function name " << params.weight_function_name << ". Only 'default','inverse','pow10' or 'oase' are known." << endl; + cerr << "Illegal weight function name " << params.weight_function_name << ". Use one of " << validWeights << endl; exit(EXIT_FAILURE); } params.wwf_lower_threshold = vm["wwf-lower-threshold"].as(); diff --git a/include/meanie3D/weights/no_weight.h b/include/meanie3D/weights/no_weight.h new file mode 100644 index 0000000..5352165 --- /dev/null +++ b/include/meanie3D/weights/no_weight.h @@ -0,0 +1,53 @@ +/* The MIT License (MIT) + * + * (c) Jürgen Simon 2014 (juergen.simon@uni-bonn.de) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef M3D_NOWEIGHTFUNCTION_H +#define M3D_NOWEIGHTFUNCTION_H + +#include +#include +#include + +namespace m3D { + + /** + * This weight function always returns 1: + * + * f(x) = 1 + * + * Use this weight function to switch off weights. + */ + template + class NoWeightFunction : public WeightFunction + { + public: + NoWeightFunction() {} + ~NoWeightFunction() {} + public: + T operator()(const typename Point::ptr p) const { + return (T) 1.0; + } + }; +} + +#endif diff --git a/include/meanie3D/weights/weight_function.h b/include/meanie3D/weights/weight_function.h index 3041626..2c09e9e 100644 --- a/include/meanie3D/weights/weight_function.h +++ b/include/meanie3D/weights/weight_function.h @@ -29,27 +29,29 @@ #include -namespace m3D { +namespace m3D +{ /** Weight function interface. The weight function plays an important * role in the mean-shift algorithm. It replaces density as criterion * in places, where the actual spatial density is homogenous, as is * often the case in scientific, gridded data sets. */ - template + template class WeightFunction { public: - - /** Weight at given point in feature-space + /** + * Weight at given point in feature-space * @param point object * @return weight */ virtual T operator()(const typename Point::ptr p) const = 0; - virtual ~WeightFunction() { + virtual ~WeightFunction() + { } }; -} +} // namespace m3D #endif diff --git a/include/meanie3D/weights/weight_function_factory_impl.h b/include/meanie3D/weights/weight_function_factory_impl.h index 2bd51e3..32ddd4d 100644 --- a/include/meanie3D/weights/weight_function_factory_impl.h +++ b/include/meanie3D/weights/weight_function_factory_impl.h @@ -31,6 +31,7 @@ #include "inverse_default.h" #include "exp10_weight.h" #include "inv_erfc_weight.h" +#include "no_weight.h" #include "weight_function_factory.h" @@ -68,6 +69,10 @@ namespace m3D { { weight_function = new InvErfcWeightFunction(params, ctx); } + else if (params.weight_function_name == "none") + { + weight_function = new NoWeightFunction(); + } else { weight_function = new DefaultWeightFunction(params, ctx); From 3085b3e961928ce69298fcf35b75d3a767cccafa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Wed, 8 Jul 2020 18:16:42 +0100 Subject: [PATCH 64/65] Comment out some defunct code (work in progress) --- include/meanie3D/utils/visit_impl.h | 39 +++++++++++++---------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/include/meanie3D/utils/visit_impl.h b/include/meanie3D/utils/visit_impl.h index 2ea58c5..f0beb32 100644 --- a/include/meanie3D/utils/visit_impl.h +++ b/include/meanie3D/utils/visit_impl.h @@ -544,8 +544,8 @@ namespace m3D cellColors->SetName("cell_color"); cellColors->SetLookupTable(cluster_lookup_table()); - vtkSmartPointer clusterIds = vtkSmartPointer::New(); - clusterIds->SetName("geometrical_center"); + // vtkSmartPointer clusterIds = vtkSmartPointer::New(); + // clusterIds->SetName("geometrical_center"); for (size_t ci = 0; ci < list->clusters.size(); ci++) { @@ -555,13 +555,13 @@ namespace m3D // Write ID T x, y, z; - VisitUtils::get_vtk_coords(cluster->geometrical_center(), x, y, z); - vtkIdType center; - center = centerPoints->InsertNextPoint(x, y, point_dim == 2 ? 0 : z); - vtkSmartPointer centerPixel = vtkSmartPointer::New(); - centerPixel->GetPointIds()->SetId(0, center); - centerCellArray->InsertNextCell(centerPixel); - clusterIds->InsertNextValue(id); + // VisitUtils::get_vtk_coords(cluster->geometrical_center(), x, y, z); + // vtkIdType center; + // center = centerPoints->InsertNextPoint(x, y, point_dim == 2 ? 0 : z); + // vtkSmartPointer centerPixel = vtkSmartPointer::New(); + // centerPixel->GetPointIds()->SetId(0, center); + // centerCellArray->InsertNextCell(centerPixel); + // clusterIds->InsertNextValue(id); // Write point_color and value information for (vtkIdType pi = 0; pi < cluster->size(); pi++) @@ -660,16 +660,14 @@ namespace m3D // 2D is a square, 3D a hexahedron VTKCellType type = point_dim == 2 ? VTK_QUAD : VTK_HEXAHEDRON; - #ifdef FOOBAR // TODO: research and finish this part. The problem seems to be to have two unstructured // grids with different numbers of points in one file. // Create an unstructured grid for center labels - vtkSmartPointer centerGrid = vtkSmartPointer::New(); - centerGrid->SetPoints(centerPoints); - centerGrid->SetCells(type, centerCellArray); - centerGrid->GetPointData()->AddArray(clusterIds); - centerGrid->GetCellData()->AddArray(clusterIds); - #endif + // vtkSmartPointer centerGrid = vtkSmartPointer::New(); + // centerGrid->SetPoints(centerPoints); + // centerGrid->SetCells(type, centerCellArray); + // centerGrid->GetPointData()->AddArray(clusterIds); + // centerGrid->GetCellData()->AddArray(clusterIds); // Create an unstructured grid and write it off vtkSmartPointer unstructuredGrid = vtkSmartPointer::New(); @@ -679,7 +677,7 @@ namespace m3D unstructuredGrid->GetPointData()->AddArray(pointColors); unstructuredGrid->GetCellData()->AddArray(cellData); unstructuredGrid->GetCellData()->AddArray(cellColors); - unstructuredGrid->GetCellData()->AddArray(clusterIds); + //unstructuredGrid->GetCellData()->AddArray(clusterIds); // Extract the envelope vtkSmartPointer enveloper; @@ -715,7 +713,7 @@ namespace m3D // from the same base class. Without this construct, the code would // have to be duplicated ... -#define WRITE_CLUSTERS(writer, centerGrid, unstructuredGrid, include_boundary, poly_filename, enveloper) \ +#define WRITE_CLUSTERS(writer, unstructuredGrid, include_boundary, poly_filename, enveloper) \ writer->SetFileName(mesh_filename.c_str()); \ /* TODO: comment back in when the problem with the labels on the grid is solved */ \ /* writer->SetInputData(centerGrid); */ \ @@ -728,13 +726,12 @@ namespace m3D writer->SetInputConnection(enveloper->GetOutputPort()); \ writer->Write(); \ } - if (write_xml) { vtkSmartPointer writer = vtkSmartPointer::New(); - WRITE_CLUSTERS(writer, centerGrid, unstructuredGrid, include_boundary, poly_filename, enveloper); + WRITE_CLUSTERS(writer, /*centerGrid,*/ unstructuredGrid, include_boundary, poly_filename, enveloper); } else { vtkSmartPointer writer = vtkSmartPointer::New(); - WRITE_CLUSTERS(writer, centerGrid, unstructuredGrid, include_boundary, poly_filename, enveloper); + WRITE_CLUSTERS(writer, /*centerGrid,*/ unstructuredGrid, include_boundary, poly_filename, enveloper); } } From d1068d1699af0bd25d868aa0f3117ce7bc6bff00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ju=CC=88rgen=20Simon?= Date: Tue, 20 Oct 2020 15:40:05 +0100 Subject: [PATCH 65/65] Bugfixes weight functions. --- include/meanie3D/weights/brightband_evidence.h | 2 -- include/meanie3D/weights/default_weights.h | 14 +++++++++----- include/meanie3D/weights/exp10_weight.h | 6 ------ include/meanie3D/weights/inverse_default.h | 14 +++++++++++++- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/include/meanie3D/weights/brightband_evidence.h b/include/meanie3D/weights/brightband_evidence.h index 0daf104..fbc8c98 100644 --- a/include/meanie3D/weights/brightband_evidence.h +++ b/include/meanie3D/weights/brightband_evidence.h @@ -55,9 +55,7 @@ namespace m3D { calculate_weight_function(FeatureSpace *fs) { for (size_t i = 0; i < fs->points.size(); i++) { Point *p = fs->points[i]; - T saliency = this->compute_weight(p); - m_weight->set(p->gridpoint, saliency); } }; diff --git a/include/meanie3D/weights/default_weights.h b/include/meanie3D/weights/default_weights.h index ac7286b..acb98d6 100644 --- a/include/meanie3D/weights/default_weights.h +++ b/include/meanie3D/weights/default_weights.h @@ -35,11 +35,15 @@ namespace m3D { - /** This weight function scales from [0..1]. The weight is calculated - * by iterating over all variables. For each variable a weight is - * generated that varies from [0..1] as the variable goes from it's - * valid_min to valid_max (linear). The values are summed up and - * divided by the number of variables. + /** + * Linear weight function: + * + * f(x) = x + * + * For each value range, the range of x is projected from 0..1 as + * the variable's value moves from it's min to it's max value. + * The actual value is the arithmetic mean of all variable weights. + * */ template class DefaultWeightFunction : public WeightFunction diff --git a/include/meanie3D/weights/exp10_weight.h b/include/meanie3D/weights/exp10_weight.h index eced8f9..c936560 100644 --- a/include/meanie3D/weights/exp10_weight.h +++ b/include/meanie3D/weights/exp10_weight.h @@ -53,9 +53,7 @@ namespace m3D { calculate_weight_function(FeatureSpace *fs) { for (size_t i = 0; i < fs->points.size(); i++) { Point *p = fs->points[i]; - T saliency = this->compute_weight(p); - m_weight->set(p->gridpoint, saliency); } }; @@ -85,15 +83,11 @@ namespace m3D { */ T compute_weight(Point *p) { T sum = 0.0; - size_t num_vars = p->values.size() - p->coordinate.size(); - for (size_t var_index = 0; var_index < num_vars; var_index++) { T value = p->values[p->coordinate.size() + var_index]; - sum += pow(boost::numeric_cast(value), 10.0); } - return sum; } diff --git a/include/meanie3D/weights/inverse_default.h b/include/meanie3D/weights/inverse_default.h index 6c3d489..825b3da 100644 --- a/include/meanie3D/weights/inverse_default.h +++ b/include/meanie3D/weights/inverse_default.h @@ -40,6 +40,18 @@ namespace m3D { using std::vector; using std::map; + /** + * This weight function is the inverse function to the default weight + * function: + * + * f(x) = 1 - x + * + * It varies from 1 to 0 as each variable in the value range goes from + * it's minimum to it's maximum. It can be useful if the goal is to track + * 'holes' rather than 'mountains'. The overall weight is again the arithmetic + * mean of all weights in the value range. + * + */ template class InverseDefaultWeightFunction : public WeightFunction { @@ -108,7 +120,7 @@ namespace m3D { sum += var_weight; } } - return sum; + return sum / num_vars; } T operator()(const typename Point::ptr p) const {