Skip to content

Commit

Permalink
Merge pull request #2985 from edwardhartnett/ejh_z2
Browse files Browse the repository at this point in the history
Add zstd test and fix plugin build for CMake/Cygwin
  • Loading branch information
WardF authored Aug 19, 2024
2 parents 1db9ffb + 5530997 commit 0cced87
Show file tree
Hide file tree
Showing 10 changed files with 180 additions and 41 deletions.
45 changes: 26 additions & 19 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -423,15 +423,15 @@ if(NC_EXTRA_DEPS)
foreach(_DEP ${DEPS_LIST})
string(REGEX REPLACE "^-l" "" _LIB ${_DEP})
FIND_LIBRARY("${_LIB}_DEP" NAMES "${_LIB}" "lib${_LIB}")
message(${${_LIB}_DEP})
message(STATUS ${${_LIB}_DEP})
if("${${_LIB}_DEP}" STREQUAL "${_LIB}_DEP-NOTFOUND")
message(FATAL_ERROR "Error finding ${_LIB}.")
else()
message(STATUS "Found ${_LIB}: ${${_LIB}_DEP}")
endif()
set(EXTRA_DEPS ${EXTRA_DEPS} "${${_LIB}_DEP}")
endforeach()
message("Extra deps: ${EXTRA_DEPS}")
message(STATUS "Extra deps: ${EXTRA_DEPS}")
list(REMOVE_DUPLICATES EXTRA_DEPS)
set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${EXTRA_DEPS})
endif()
Expand Down Expand Up @@ -618,22 +618,26 @@ set(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers t
set(REMOTETESTSERVERS "remotetest.unidata.ucar.edu" CACHE STRING "test servers to use for remote test")

# Locate some compressors
option(NETCDF_ENABLE_PLUGINS "Enable netCDF Plugins." ON)
option(NETCDF_ENABLE_FILTER_SZIP "Enable use of Szip compression library if it is available. Required if NETCDF_ENABLE_NCZARR is true." ON)
option(NETCDF_ENABLE_FILTER_BZ2 "Enable use of Bz2 compression library if it is available." ON)
option(NETCDF_ENABLE_FILTER_BLOSC "Enable use of blosc compression library if it is available." ON)
option(NETCDF_ENABLE_FILTER_ZSTD "Enable use of Zstd compression library if it is available." ON)

# If user wants, then install selected plugins (default on)
set(PLUGIN_INSTALL_DIR "NO" CACHE STRING "Whether and where we should install plugins; defaults to yes")
set(NETCDF_PLUGIN_INSTALL_DIR "YES" CACHE STRING "Whether and where we should install plugins; defaults to yes")
if(NOT NETCDF_ENABLE_PLUGINS)
unset(NETCDF_PLUGIN_INSTALL_DIR CACHE)
endif()

# This is ugly, but seems necessary because of CMake's boolean structure
set(boolval FALSE)
if(DEFINED PLUGIN_INSTALL_DIR)
booleanize(${PLUGIN_INSTALL_DIR} boolval)
if(DEFINED NETCDF_PLUGIN_INSTALL_DIR)
booleanize(${NETCDF_PLUGIN_INSTALL_DIR} boolval)
if(boolval)
set(ENABLE_PLUGIN_INSTALL YES)
# No actual value was specified
unset(PLUGIN_INSTALL_DIR CACHE)
unset(NETCDF_PLUGIN_INSTALL_DIR CACHE)
else()
if(boolval STREQUAL "NOTFOUND")
# Must be an actual value
Expand All @@ -648,43 +652,45 @@ endif()

# Ensure no defined plugin dir if not enabled
if(NOT ENABLE_PLUGIN_INSTALL)
unset(PLUGIN_INSTALL_DIR CACHE)
unset(NETCDF_PLUGIN_INSTALL_DIR CACHE)
endif()

if(ENABLE_PLUGIN_INSTALL)
if(NOT DEFINED PLUGIN_INSTALL_DIR)
if(NOT DEFINED NETCDF_PLUGIN_INSTALL_DIR)
# Default to HDF5_PLUGIN_PATH or its default directories
if(DEFINED ENV{HDF5_PLUGIN_PATH})
set(PLUGIN_INSTALL_DIR "$ENV{HDF5_PLUGIN_PATH}")
set(NETCDF_PLUGIN_INSTALL_DIR "$ENV{HDF5_PLUGIN_PATH}")
else()
if(ISMSVC OR ISMINGW)
set(PLUGIN_INSTALL_DIR "$ENV{ALLUSERSPROFILE}\\hdf5\\lib\\plugin")
set(NETCDF_PLUGIN_INSTALL_DIR "$ENV{ALLUSERSPROFILE}\\hdf5\\lib\\plugin")
else()
set(PLUGIN_INSTALL_DIR "/usr/local/hdf5/lib/plugin")
set(NETCDF_PLUGIN_INSTALL_DIR "/usr/local/hdf5/lib/plugin")
endif()
endif()
message("Defaulting to -DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}")
message(STATUS "Defaulting to -DPLUGIN_INSTALL_DIR=${NETCDF_PLUGIN_INSTALL_DIR}")
endif()
endif()

if(ENABLE_PLUGIN_INSTALL)
# Use the lowest priority dir in the path
if(NOT ISMSVC AND NOT ISMINGW)
string(REPLACE ":" ";" PATH_LIST ${PLUGIN_INSTALL_DIR})
string(REPLACE ":" ";" PATH_LIST ${NETCDF_PLUGIN_INSTALL_DIR})
else()
set(PATH_LIST ${PLUGIN_INSTALL_DIR})
set(PATH_LIST ${NETCDF_PLUGIN_INSTALL_DIR})
endif()

# Get last element
list(GET PATH_LIST -1 PLUGIN_INSTALL_DIR)
set(PLUGIN_INSTALL_DIR_SETTING "${PLUGIN_INSTALL_DIR}")
message("Final value of-DPLUGIN_INSTALL_DIR=${PLUGIN_INSTALL_DIR}")
list(GET PATH_LIST -1 NETCDF_PLUGIN_INSTALL_DIR)
set(PLUGIN_INSTALL_DIR_SETTING "${NETCDF_PLUGIN_INSTALL_DIR}")
message(STATUS "Final value of-DPLUGIN_INSTALL_DIR=${NETCDF_PLUGIN_INSTALL_DIR}")
else() # No option specified
unset(PLUGIN_INSTALL_DIR)
unset(PLUGIN_INSTALL_DIR CACHE)
unset(NETCDF_PLUGIN_INSTALL_DIR)
unset(NETCDF_PLUGIN_INSTALL_DIR CACHE)
set(PLUGIN_INSTALL_DIR_SETTING "N.A.")
endif()

message(STATUS "ENABLE_PLUGIN_INSTALL=${ENABLE_PLUGIN_INSTALL} PLUGIN_INSTALL_DIR=${NETCDF_PLUGIN_INSTALL_DIR}")

# Try to enable NCZarr zip support
option(NETCDF_ENABLE_NCZARR_ZIP "Enable NCZarr ZIP support." ${NETCDF_ENABLE_NCZARR})

Expand Down Expand Up @@ -1696,6 +1702,7 @@ is_enabled(NETCDF_ENABLE_S3_INTERNAL HAS_S3_INTERNAL)
is_enabled(HAS_HDF5_ROS3 HAS_HDF5_ROS3)
is_enabled(NETCDF_ENABLE_NCZARR HAS_NCZARR)
is_enabled(NETCDF_ENABLE_NCZARR_ZIP HAS_NCZARR_ZIP)
is_enabled(NETCDF_ENABLE_PLUGINS HAS_PLUGINS)
is_enabled(NETCDF_ENABLE_QUANTIZE HAS_QUANTIZE)
is_enabled(NETCDF_ENABLE_LOGGING HAS_LOGGING)
is_enabled(NETCDF_ENABLE_FILTER_TESTING DO_FILTER_TESTS)
Expand Down
5 changes: 3 additions & 2 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -2035,6 +2035,7 @@ AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3])
AC_SUBST(HAS_NCZARR,[$enable_nczarr])
AC_SUBST(NETCDF_ENABLE_S3_TESTING,[$with_s3_testing])
AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip])
AC_SUBST(HAS_PLUGINS, [$enable_plugins])
AC_SUBST(HAS_QUANTIZE,[$enable_quantize])
AC_SUBST(HAS_LOGGING,[$enable_logging])
AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing])
Expand Down Expand Up @@ -2102,7 +2103,7 @@ AC_ARG_WITH([plugin-dir], [AS_HELP_STRING([--with-plugin-dir=<absolute directory
[Install selected standard filters in specified or default directory])],
[],[with_plugin_dir=no])
if test "x$have_zstd" = xyes; then
if test "x$with_plugin_dir" = xno; then
if test "x$enable_plugins" = xyes && test "x$with_plugin_dir" = xno; then
with_plugin_dir=yes
fi
fi
Expand Down Expand Up @@ -2142,7 +2143,7 @@ AM_CONDITIONAL([ENABLE_PLUGIN_DIR], [test "x$enable_plugin_dir" = xyes])
AC_SUBST([PLUGIN_INSTALL_DIR], [$with_plugin_dir])
# Better value for libnetcdf.settings
AC_SUBST([PLUGIN_INSTALL_DIR_SETTING], [$with_plugin_dir_setting])

AC_SUBST(NETCDF_PLUGIN_INSTALL_DIR, [$PLUGIN_INSTALL_DIR])
# Access netcdf specific version of config.h
AH_BOTTOM([#include "ncconfigure.h"])

Expand Down
3 changes: 3 additions & 0 deletions libnetcdf.settings.in
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ MMap Support: @HAS_MMAP@
ERANGE Fill Support: @HAS_ERANGE_FILL@
Relaxed Boundary Check: @RELAX_COORD_BOUND@

Plugins Enabled: @HAS_PLUGINS@
Plugin Install Dir: @NETCDF_PLUGIN_INSTALL_DIR@

Quantization: @HAS_QUANTIZE@
Logging: @HAS_LOGGING@
SZIP Write Support: @HAS_SZLIB_WRITE@
Expand Down
2 changes: 1 addition & 1 deletion nc-config.cmake.in
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ prefix="@CMAKE_INSTALL_PREFIX@"
exec_prefix="@CMAKE_INSTALL_PREFIX@"
libdir="@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@"
includedir="@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@"
plugindir="@PLUGIN_INSTALL_DIR@"
plugindir="@NETCDF_PLUGIN_INSTALL_DIR@"

cc="@CMAKE_C_COMPILER@"
cflags="-I@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@"
Expand Down
6 changes: 6 additions & 0 deletions nc_test4/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ IF(HAVE_H5Z_SZIP)
ENDIF()
ENDIF()

IF(USE_HDF5 AND NETCDF_ENABLE_FILTER_TESTING)
IF(HAVE_ZSTD)
BUILD_BIN_TEST(tst_zstd)
add_sh_test(nc_test4 run_zstd_test)
ENDIF()
ENDIF()

# Copy some test files from current source dir to out-of-tree build dir.
FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.nc ${CMAKE_CURRENT_SOURCE_DIR}/ref_bzip2.c ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/*.h5 ${CMAKE_CURRENT_SOURCE_DIR}/*.cdl)
Expand Down
35 changes: 21 additions & 14 deletions nc_test4/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
include $(top_srcdir)/lib_flags.am

# Un comment to use a more verbose test driver
#SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
#TESTS_ENVIRONMENT = export SETX=1;
# SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
# sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
# LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose
# TESTS_ENVIRONMENT = export SETX=1;

TEST_EXTENSIONS = .sh

Expand Down Expand Up @@ -113,6 +113,14 @@ endif # NETCDF_ENABLE_PLUGINS
endif # USE_HDF5
endif # NETCDF_BUILD_UTILITIES

# Zstandard tests.
if NETCDF_ENABLE_PLUGINS
if HAVE_ZSTD
check_PROGRAMS += tst_zstd
TESTS += run_zstd_test.sh
endif # HAVE_ZSTD
endif # NETCDF_ENABLE_PLUGINS

# This are extra tests that will only be run if netcdf-4 is configured
# with --enable-parallel-tests.
if TEST_PARALLEL4
Expand All @@ -129,16 +137,15 @@ ref_tst_interops4.nc CMakeLists.txt run_grp_rename.sh \
run_empty_vlen_test.sh ref_hdf5_compat1.nc ref_hdf5_compat2.nc \
ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5 \
ref_szip.cdl tst_filter.sh bzip2.cdl noop.cdl ref_filtered.cdl \
ref_unfiltered.cdl ref_bzip2.c ref_unfilteredvv.cdl \
ref_filteredvv.cdl ref_multi.cdl \
ref_ncgenF.cdl ref_nccopyF.cdl \
ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \
ref_filter_order_create.txt ref_filter_order_read.txt \
ref_any.cdl tst_specific_filters.sh tst_unknown.sh \
tst_virtual_datasets.c noop1.cdl unknown.cdl \
tst_broken_files.c ref_bloscx.cdl tst_bloscfail.sh \
tst_fixedstring.sh ref_fixedstring.h5 ref_fixedstring.cdl \
tst_filterinstall.sh tst_filter_vlen.sh tst_filter_misc.sh
ref_unfiltered.cdl ref_bzip2.c ref_unfilteredvv.cdl \
ref_filteredvv.cdl ref_multi.cdl ref_ncgenF.cdl ref_nccopyF.cdl \
ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \
ref_filter_order_create.txt ref_filter_order_read.txt ref_any.cdl \
tst_specific_filters.sh tst_unknown.sh tst_virtual_datasets.c \
noop1.cdl unknown.cdl tst_broken_files.c ref_bloscx.cdl \
tst_bloscfail.sh tst_fixedstring.sh ref_fixedstring.h5 \
ref_fixedstring.cdl tst_filterinstall.sh tst_filter_vlen.sh \
tst_filter_misc.sh run_zstd_test.sh

CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \
tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt \
Expand Down
33 changes: 33 additions & 0 deletions nc_test4/run_zstd_test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/sh

# This .in file is processed at build time into a shell that runs some
# zstd for netCDF.

# Ed Hartnett, 8/10/24

set -e

if test "x$srcdir" = x ; then srcdir=`pwd`; fi
. ../test_common.sh

# Load the findplugins function
. ${builddir}/findplugin.sh
echo "findplugin.sh loaded"
echo "${HDF5_PLUGIN_DIR}"

findplugin h5zstd
echo
echo "Testing Zstandard compression..."

export HDF5_PLUGIN_PATH="${HDF5_PLUGIN_DIR}"
echo "HDF5_PLUGIN_PATH=$HDF5_PLUGIN_PATH"
ls -l "${HDF5_PLUGIN_PATH}"

echo "set HDF5_PLUGIN_PATH: ${HDF5_PLUGIN_PATH}"

./tst_zstd

echo "SUCCESS!"



78 changes: 78 additions & 0 deletions nc_test4/tst_zstd.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
/* This is part of the netCDF package.
Copyright 2018 University Corporation for Atmospheric Research/Unidata
See COPYRIGHT file for conditions of use.
Test netcdf-4 variables.
Ed Hartnett, 8/7/24
*/

#include "nc_tests.h"
#include "err_macros.h"
#include "netcdf.h"

#define FILE_NAME "tst_zstd.nc"
#define VAR_FLOAT_NAME "GALE_data"
#define VAR_DOUBLE_NAME "ERICA_data"
#define NDIM2 2
#define DIM1_NAME "x"
#define DIM1_LEN 2
#define DIM2_NAME "y"
#define DIM2_LEN 3

int
main(int argc, char **argv)
{
int ncid, dimids[NDIM2];
int float_varid, double_varid;
float float_in[DIM1_LEN][DIM2_LEN], float_out[DIM1_LEN][DIM2_LEN] = {{-.1f, 9999.99f, 100.001f},{-.1f, 9999.99f, 100.001f}};
double double_in[DIM1_LEN][DIM2_LEN], double_out[DIM1_LEN][DIM2_LEN] = {{0.02, .1128, 1090.1},{0.02, .1128, 1090.1}};
int i, j, ret;


printf("\n*** Testing netcdf-4 zstd compression.\n");

printf("*** testing netcdf-4 zstd with float...");
{
int nvars_in, varids_in[2];

/* Create a netcdf file with one dim and two vars. */
if (nc_create(FILE_NAME, NC_NETCDF4, &ncid)) ERR;
if (nc_def_dim(ncid, DIM1_NAME, DIM1_LEN, &dimids[0])) ERR;
if (nc_def_dim(ncid, DIM2_NAME, DIM2_LEN, &dimids[1])) ERR;
if (nc_def_var(ncid, VAR_FLOAT_NAME, NC_FLOAT, 2, dimids, &float_varid)) ERR;
if (nc_def_var(ncid, VAR_DOUBLE_NAME, NC_DOUBLE, 2, dimids, &double_varid)) ERR;
if ((ret = nc_def_var_zstandard(ncid, float_varid, 4)))
{
printf("error: %s\n", nc_strerror(ret));
ERR;
}
if (nc_def_var_zstandard(ncid, double_varid, 4)) ERR;
if (nc_put_var_float(ncid, float_varid, (float *)float_out)) ERR;
if (nc_put_var_double(ncid, double_varid, (double *)double_out)) ERR;
if (nc_close(ncid)) ERR;

/* Open the file and make sure nc_inq_varids yields correct
* result. */
if (nc_open(FILE_NAME, NC_NOWRITE, &ncid)) ERR;
if (nc_inq_varids(ncid, &nvars_in, varids_in)) ERR;
if (nvars_in != 2 || varids_in[0] != 0 || varids_in[1] != 1) ERR;
if (nc_get_var_float(ncid, float_varid, (float *)float_in)) ERR;
if (nc_get_var_double(ncid, double_varid, (double *)double_in)) ERR;
for (i = 0; i < DIM1_LEN; i++)
{
for (j = 0; j < DIM2_LEN; j++)
{
if (float_in[i][j] != float_out[i][j]) ERR;
if (double_in[i][j] != double_out[i][j]) ERR;
}
}


if (nc_close(ncid)) ERR;
}

SUMMARIZE_ERR;
SUMMARIZE_ERR;

FINAL_RESULTS;
}
8 changes: 6 additions & 2 deletions ncdump/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,8 @@ endif()
endif()

set_tests_properties(ncdump_tst_nccopy3 PROPERTIES DEPENDS
"ncdump_tst_calendars;ncdump_run_utf8_tests;ncdump_tst_output;ncdump_tst_64bit;ncdump_run_tests;ncdump_tst_lengths")
"ncdump_tst_calendars;ncdump_run_utf8_tests;ncdump_tst_output;ncdump_tst_64bit;ncdump_run_tests;ncdump_tst_lengths")
set_tests_properties(ncdump_tst_nccopy3 PROPERTIES RUN_SERIAL TRUE)

endif()

Expand All @@ -271,9 +272,12 @@ endif()
add_sh_test(ncdump tst_ncgen4)
add_sh_test(ncdump tst_netcdf4_4)
add_sh_test(ncdump tst_nccopy4)
set_tests_properties(ncdump_tst_nccopy4 PROPERTIES DEPENDS
"ncdump_ncgen_tests;ncdump_tst_ncgen4;ncdump_tst_fillbug;ncdump_tst_netcdf4_4;ncdump_tst_h_scalar;ncdump_tst_netcdf4;ncdump_run_ncgen_tests;ncdump_tst_output;ncdump_tst_ncgen4;ncdump_sh_tst_fillbug;tst_comp;tst_comp2;tst_nans;tst_opaque_data;tst_create_files;tst_special_atts;ncdump_tst_nccopy3")
set_tests_properties(ncdump_tst_nccopy4 PROPERTIES RUN_SERIAL TRUE)
add_sh_test(ncdump tst_calendars_nc4)

set_tests_properties(ncdump_tst_nccopy4 PROPERTIES DEPENDS "ncdump_run_ncgen_tests;ncdump_tst_output;ncdump_tst_ncgen4;ncdump_sh_tst_fillbug;ncdump_tst_netcdf4_4;ncdump_tst_h_scalar;tst_comp;tst_comp2;tst_nans;tst_opaque_data;tst_create_files;tst_special_atts")
set_tests_properties(ncdump_tst_nccopy4 PROPERTIES DEPENDS "")
set_tests_properties(ncdump_tst_nccopy5 PROPERTIES DEPENDS "ncdump_tst_nccopy4")

endif(USE_HDF5)
Expand Down
6 changes: 3 additions & 3 deletions plugins/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ if(WIN32)
set(PLUGINPRE "__nc")
elseif(CYGWIN)
set(PLUGINEXT "dll")
set(PLUGINPRE "lib__nc")
set(PLUGINPRE "cyg__nc")
else()
set(PLUGINPRE "lib__nc")
if(APPLE)
Expand Down Expand Up @@ -124,8 +124,8 @@ if(ENABLE_PLUGIN_INSTALL)

macro(installplugin PLUG)
set(INSTALLED_PLUGIN_LIB "${PLUGINPRE}${PLUG}.${PLUGINEXT}")
message(STATUS "Installing: ${INSTALLED_PLUGIN_LIB} into ${PLUGIN_INSTALL_DIR}")
install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${INSTALLED_PLUGIN_LIB} DESTINATION ${PLUGIN_INSTALL_DIR})
message(STATUS "Installing: ${INSTALLED_PLUGIN_LIB} into ${NETCDF_PLUGIN_INSTALL_DIR}")
install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${INSTALLED_PLUGIN_LIB} DESTINATION ${NETCDF_PLUGIN_INSTALL_DIR})
endmacro()

install(DIRECTORY DESTINATION ${PLUGIN_INSTALL_DIR})
Expand Down

0 comments on commit 0cced87

Please sign in to comment.