diff --git a/docs/building_blocks.md b/docs/building_blocks.md index 5b815a5..273c5e7 100644 --- a/docs/building_blocks.md +++ b/docs/building_blocks.md @@ -1412,7 +1412,7 @@ non-default compilers or other toolchain options are needed. The default is empty. - __version__: The version of HDF5 source to download. This value is -ignored if `directory` is set. The default value is `1.12.0`. +ignored if `directory` is set. The default value is `1.14.5`. - __with_PACKAGE[=ARG]__: Flags to control optional packages when configuring. For instance, `with_foo=True` maps to `--with-foo` @@ -2988,10 +2988,10 @@ directory. The default value is False. - __ospackages__: List of OS packages to install prior to configuring and building. For Ubuntu, the default values are -`ca-certificates`, `file`, `libcurl4-openssl-dev`, `m4`, `make`, -`wget`, and `zlib1g-dev`. For RHEL-based Linux distributions the -default values are `ca-certificates`, `file`, `libcurl-devel` -`m4`, `make`, `wget`, and `zlib-devel`. +`ca-certificates`, `file`, `libcurl4-openssl-dev`, `libxml2-dev`, `m4`, +`make`, `wget`, and `zlib1g-dev`. For RHEL-based Linux distributions the +default values are `ca-certificates`, `file`, `libcurl-devel`, +`libxml2-devel`, `m4`, `make`, `wget`, and `zlib-devel`. - __prefix__: The top level install location. The default location is `/usr/local/netcdf`. diff --git a/hpccm/building_blocks/hdf5.py b/hpccm/building_blocks/hdf5.py index 75c1055..f20a0af 100644 --- a/hpccm/building_blocks/hdf5.py +++ b/hpccm/building_blocks/hdf5.py @@ -21,6 +21,8 @@ from __future__ import unicode_literals from __future__ import print_function +from packaging.version import Version + import posixpath import re from copy import copy as _copy @@ -92,7 +94,7 @@ class hdf5(bb_base, hpccm.templates.envvars, hpccm.templates.ldconfig): default is empty. version: The version of HDF5 source to download. This value is - ignored if `directory` is set. The default value is `1.12.0`. + ignored if `directory` is set. The default value is `1.14.5`. with_PACKAGE[=ARG]: Flags to control optional packages when configuring. For instance, `with_foo=True` maps to `--with-foo` @@ -132,7 +134,7 @@ def __init__(self, **kwargs): super(hdf5, self).__init__(**kwargs) - self.__baseurl = kwargs.pop('baseurl', 'https://support.hdfgroup.org/ftp/HDF5/releases') + self.__baseurl = kwargs.pop('baseurl', False) self.__check = kwargs.pop('check', False) self.__configure_opts = kwargs.pop('configure_opts', ['--enable-cxx', @@ -143,7 +145,14 @@ def __init__(self, **kwargs): # without impacting the original self.__toolchain = _copy(kwargs.pop('toolchain', toolchain())) self.__runtime_ospackages = [] # Filled in by __distro() - self.__version = kwargs.pop('version', '1.12.0') + self.__version = kwargs.pop('version', '1.14.5') + + if not self.__baseurl: + # Download path changed with version 1.14 + if Version(self.__version) >= Version('1.14'): + self.__baseurl = 'https://support.hdfgroup.org/releases/hdf5' + else: + self.__baseurl = 'https://support.hdfgroup.org/ftp/HDF5/releases' # Set the Linux distribution specific parameters self.__distro() @@ -214,11 +223,19 @@ def __download(self): # path and the tarball contains MAJOR.MINOR.REVISION, so pull # apart the full version to get the MAJOR and MINOR components. match = re.match(r'(?P\d+)\.(?P\d+)', self.__version) - major_minor = '{0}.{1}'.format(match.groupdict()['major'], - match.groupdict()['minor']) - tarball = 'hdf5-{}.tar.bz2'.format(self.__version) - self.__url = '{0}/hdf5-{1}/hdf5-{2}/src/{3}'.format( - self.__baseurl, major_minor, self.__version, tarball) + if Version(self.__version) >= Version('1.14'): + major_minor = 'v{0}_{1}'.format(match.groupdict()['major'], + match.groupdict()['minor']) + tarball = 'hdf5-{}.tar.gz'.format(self.__version) + self.__url = '{0}/{1}/v{2}/downloads/{3}'.format( + self.__baseurl, major_minor, self.__version.replace('.', '_'), + tarball) + else: + major_minor = '{0}.{1}'.format(match.groupdict()['major'], + match.groupdict()['minor']) + tarball = 'hdf5-{}.tar.bz2'.format(self.__version) + self.__url = '{0}/hdf5-{1}/hdf5-{2}/src/{3}'.format( + self.__baseurl, major_minor, self.__version, tarball) def runtime(self, _from='0'): """Generate the set of instructions to install the runtime specific diff --git a/hpccm/building_blocks/netcdf.py b/hpccm/building_blocks/netcdf.py index 23440cd..789ad6f 100644 --- a/hpccm/building_blocks/netcdf.py +++ b/hpccm/building_blocks/netcdf.py @@ -23,6 +23,8 @@ from packaging.version import Version import posixpath +import re +from copy import copy as _copy import hpccm.config import hpccm.templates.envvars @@ -33,6 +35,7 @@ from hpccm.building_blocks.packages import packages from hpccm.common import linux_distro from hpccm.primitives.comment import comment +from hpccm.toolchain import toolchain class netcdf(bb_base, hpccm.templates.envvars, hpccm.templates.ldconfig): """The `netcdf` building block downloads, configures, builds, and @@ -80,10 +83,10 @@ class netcdf(bb_base, hpccm.templates.envvars, hpccm.templates.ldconfig): ospackages: List of OS packages to install prior to configuring and building. For Ubuntu, the default values are - `ca-certificates`, `file`, `libcurl4-openssl-dev`, `m4`, `make`, - `wget`, and `zlib1g-dev`. For RHEL-based Linux distributions the - default values are `ca-certificates`, `file`, `libcurl-devel` - `m4`, `make`, `wget`, and `zlib-devel`. + `ca-certificates`, `file`, `libcurl4-openssl-dev`, `libxml2-dev`, `m4`, + `make`, `wget`, and `zlib1g-dev`. For RHEL-based Linux distributions the + default values are `ca-certificates`, `file`, `libcurl-devel`, + `libxml2-devel`, `m4`, `make`, `wget`, and `zlib-devel`. prefix: The top level install location. The default location is `/usr/local/netcdf`. @@ -134,6 +137,9 @@ def __init__(self, **kwargs): self.__ospackages = kwargs.pop('ospackages', []) self.__prefix = kwargs.pop('prefix', '/usr/local/netcdf') self.__runtime_ospackages = [] # Filled in by __distro() + # Create a copy of the toolchain so that it can be modified + # without impacting the original + self.__toolchain = _copy(kwargs.pop('toolchain', toolchain())) self.__version = kwargs.pop('version', '4.7.4') self.__version_cxx = kwargs.pop('version_cxx', '4.3.1') self.__version_fortran = kwargs.pop('version_fortran', '4.5.3') @@ -165,6 +171,7 @@ def __init__(self, **kwargs): directory=self.__directory_c, prefix=self.__prefix, runtime_environment=self.environment_variables, + toolchain=self.__toolchain, url=self.__url_c, **kwargs)] @@ -180,12 +187,19 @@ def __init__(self, **kwargs): # Checks fail when using parallel make. Disable it. parallel=1 if self.__check else '$(nproc)', prefix=self.__prefix, + toolchain=self.__toolchain, url='{0}/v{1}.tar.gz'.format(self.__baseurl_cxx, self.__version_cxx), **kwargs)) # Setup optional Fortran build configuration if self.__fortran: + # PIC workaround when using the NVIDIA compilers + if self.__toolchain.FC and re.match('.*nvfortran', + self.__toolchain.FC): + if not self.__toolchain.FCFLAGS: + self.__toolchain.FCFLAGS = '-fPIC' + comments.append('NetCDF Fortran version {}'.format(self.__version_fortran)) self.__bb.append(generic_autotools( annotations={'version': self.__version_fortran}, @@ -196,6 +210,7 @@ def __init__(self, **kwargs): # Checks fail when using parallel make. Disable it. parallel=1 if self.__check else '$(nproc)', prefix=self.__prefix, + toolchain=self.__toolchain, url='{0}/v{1}.tar.gz'.format(self.__baseurl_fortran, self.__version_fortran), **kwargs)) @@ -212,14 +227,14 @@ def __distro(self): if hpccm.config.g_linux_distro == linux_distro.UBUNTU: if not self.__ospackages: self.__ospackages = ['ca-certificates', 'file', - 'libcurl4-openssl-dev', 'm4', 'make', - 'wget', 'zlib1g-dev'] + 'libcurl4-openssl-dev', 'libxml2-dev', + 'm4', 'make', 'wget', 'zlib1g-dev'] self.__runtime_ospackages = ['zlib1g'] elif hpccm.config.g_linux_distro == linux_distro.CENTOS: if not self.__ospackages: self.__ospackages = ['ca-certificates', 'file', - 'libcurl-devel', 'm4', 'make', - 'wget', 'zlib-devel'] + 'libcurl-devel', 'libxml2-devel', 'm4', + 'make', 'wget', 'zlib-devel'] if self.__check: self.__ospackages.append('diffutils') self.__runtime_ospackages = ['zlib'] diff --git a/test/test_hdf5.py b/test/test_hdf5.py index 39ce28c..72113e1 100644 --- a/test/test_hdf5.py +++ b/test/test_hdf5.py @@ -36,8 +36,8 @@ def setUp(self): def test_defaults_ubuntu(self): """Default hdf5 building block""" h = hdf5() - self.assertEqual(str(h), -r'''# HDF5 version 1.12.0 + self.assertMultiLineEqual(str(h), +r'''# HDF5 version 1.14.5 RUN apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ bzip2 \ @@ -46,12 +46,12 @@ def test_defaults_ubuntu(self): wget \ zlib1g-dev && \ rm -rf /var/lib/apt/lists/* -RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.0/src/hdf5-1.12.0.tar.bz2 && \ - mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.12.0.tar.bz2 -C /var/tmp -j && \ - cd /var/tmp/hdf5-1.12.0 && ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \ +RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/releases/hdf5/v1_14/v1_14_5/downloads/hdf5-1.14.5.tar.gz && \ + mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.14.5.tar.gz -C /var/tmp -z && \ + cd /var/tmp/hdf5-1.14.5 && ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \ make -j$(nproc) && \ make -j$(nproc) install && \ - rm -rf /var/tmp/hdf5-1.12.0 /var/tmp/hdf5-1.12.0.tar.bz2 + rm -rf /var/tmp/hdf5-1.14.5 /var/tmp/hdf5-1.14.5.tar.gz ENV CPATH=/usr/local/hdf5/include:$CPATH \ HDF5_DIR=/usr/local/hdf5 \ LD_LIBRARY_PATH=/usr/local/hdf5/lib:$LD_LIBRARY_PATH \ @@ -63,8 +63,8 @@ def test_defaults_ubuntu(self): def test_defaults_centos(self): """Default hdf5 building block""" h = hdf5() - self.assertEqual(str(h), -r'''# HDF5 version 1.12.0 + self.assertMultiLineEqual(str(h), +r'''# HDF5 version 1.14.5 RUN yum install -y \ bzip2 \ file \ @@ -72,12 +72,12 @@ def test_defaults_centos(self): wget \ zlib-devel && \ rm -rf /var/cache/yum/* -RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.0/src/hdf5-1.12.0.tar.bz2 && \ - mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.12.0.tar.bz2 -C /var/tmp -j && \ - cd /var/tmp/hdf5-1.12.0 && ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \ +RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://support.hdfgroup.org/releases/hdf5/v1_14/v1_14_5/downloads/hdf5-1.14.5.tar.gz && \ + mkdir -p /var/tmp && tar -x -f /var/tmp/hdf5-1.14.5.tar.gz -C /var/tmp -z && \ + cd /var/tmp/hdf5-1.14.5 && ./configure --prefix=/usr/local/hdf5 --enable-cxx --enable-fortran && \ make -j$(nproc) && \ make -j$(nproc) install && \ - rm -rf /var/tmp/hdf5-1.12.0 /var/tmp/hdf5-1.12.0.tar.bz2 + rm -rf /var/tmp/hdf5-1.14.5 /var/tmp/hdf5-1.14.5.tar.gz ENV CPATH=/usr/local/hdf5/include:$CPATH \ HDF5_DIR=/usr/local/hdf5 \ LD_LIBRARY_PATH=/usr/local/hdf5/lib:$LD_LIBRARY_PATH \ @@ -89,7 +89,7 @@ def test_defaults_centos(self): def test_ldconfig(self): """ldconfig option""" h = hdf5(ldconfig=True, version='1.10.4') - self.assertEqual(str(h), + self.assertMultiLineEqual(str(h), r'''# HDF5 version 1.10.4 RUN apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ @@ -117,7 +117,7 @@ def test_runtime(self): """Runtime""" h = hdf5() r = h.runtime() - self.assertEqual(r, + self.assertMultiLineEqual(r, r'''# HDF5 RUN apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ diff --git a/test/test_netcdf.py b/test/test_netcdf.py index 0042150..e683311 100644 --- a/test/test_netcdf.py +++ b/test/test_netcdf.py @@ -36,7 +36,7 @@ def setUp(self): def test_defaults_ubuntu(self): """Default netcdf building block""" n = netcdf() - self.assertEqual(str(n), + self.assertMultiLineEqual(str(n), r'''# NetCDF version 4.7.4, NetCDF C++ version 4.3.1, NetCDF Fortran # version 4.5.3 RUN apt-get update -y && \ @@ -44,6 +44,7 @@ def test_defaults_ubuntu(self): ca-certificates \ file \ libcurl4-openssl-dev \ + libxml2-dev \ m4 \ make \ wget \ @@ -77,13 +78,14 @@ def test_defaults_ubuntu(self): def test_defaults_centos(self): """Default netcdf building block""" n = netcdf() - self.assertEqual(str(n), + self.assertMultiLineEqual(str(n), r'''# NetCDF version 4.7.4, NetCDF C++ version 4.3.1, NetCDF Fortran # version 4.5.3 RUN yum install -y \ ca-certificates \ file \ libcurl-devel \ + libxml2-devel \ m4 \ make \ wget \ @@ -118,7 +120,7 @@ def test_ldconfig(self): """ldconfig option""" n = netcdf(ldconfig=True, version='4.6.1', version_cxx='4.3.0', version_fortran='4.4.4') - self.assertEqual(str(n), + self.assertMultiLineEqual(str(n), r'''# NetCDF version 4.6.1, NetCDF C++ version 4.3.0, NetCDF Fortran # version 4.4.4 RUN apt-get update -y && \ @@ -126,6 +128,7 @@ def test_ldconfig(self): ca-certificates \ file \ libcurl4-openssl-dev \ + libxml2-dev \ m4 \ make \ wget \ @@ -162,7 +165,7 @@ def test_runtime(self): """Runtime""" n = netcdf() r = n.runtime() - self.assertEqual(r, + self.assertMultiLineEqual(r, r'''# NetCDF RUN apt-get update -y && \ DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \