From d99fdd4c6f8b2518618b4a463d3262f53f124656 Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Mon, 5 Aug 2024 22:28:51 -0600 Subject: [PATCH 01/19] add HDF5: database, forward and kernel output --- DATA/Par_file | 8 +- Makefile.in | 14 + configure | 108 +- configure.ac | 50 +- src/gindex3D/rules.mk | 2 + src/meshfem3D/create_regions_mesh.F90 | 11 +- src/meshfem3D/get_absorb.f90 | 13 +- src/meshfem3D/initialize_mesher.f90 | 6 + src/meshfem3D/rules.mk | 11 + src/meshfem3D/save_arrays_solver.f90 | 48 + src/meshfem3D/save_arrays_solver_hdf5.F90 | 1107 ++++ src/meshfem3D/save_model_meshfiles_hdf5.F90 | 282 + src/shared/broadcast_computed_parameters.f90 | 13 +- src/shared/hdf5_manager.F90 | 6065 ++++++++++++++++++ src/shared/parallel.f90 | 45 + src/shared/read_parameter_file.F90 | 26 + src/shared/rules.mk | 17 + src/shared/shared_par.f90 | 12 + src/specfem3D/SIEM_compute_kernels.F90 | 6 + src/specfem3D/get_attenuation.f90 | 6 +- src/specfem3D/noise_tomography.f90 | 2 + src/specfem3D/prepare_attenuation.f90 | 2 + src/specfem3D/read_arrays_solver_hdf5.F90 | 1322 ++++ src/specfem3D/read_forward_arrays.F90 | 6 + src/specfem3D/read_forward_arrays_hdf5.F90 | 440 ++ src/specfem3D/read_mesh_databases.F90 | 135 +- src/specfem3D/rules.mk | 14 + src/specfem3D/save_forward_arrays.F90 | 6 + src/specfem3D/save_forward_arrays_hdf5.F90 | 877 +++ src/specfem3D/save_kernels.F90 | 22 +- src/specfem3D/save_kernels_hdf5.F90 | 675 ++ src/specfem3D/save_regular_kernels.f90 | 3 + src/specfem3D/write_movie_output.f90 | 9 + src/specfem3D/write_seismograms.f90 | 3 + 34 files changed, 11350 insertions(+), 16 deletions(-) create mode 100644 src/meshfem3D/save_arrays_solver_hdf5.F90 create mode 100644 src/meshfem3D/save_model_meshfiles_hdf5.F90 create mode 100644 src/shared/hdf5_manager.F90 create mode 100644 src/specfem3D/read_arrays_solver_hdf5.F90 create mode 100644 src/specfem3D/read_forward_arrays_hdf5.F90 create mode 100644 src/specfem3D/save_forward_arrays_hdf5.F90 create mode 100644 src/specfem3D/save_kernels_hdf5.F90 diff --git a/DATA/Par_file b/DATA/Par_file index a027f0d99..fb53c6442 100644 --- a/DATA/Par_file +++ b/DATA/Par_file @@ -227,7 +227,7 @@ MOVIE_START = 0 MOVIE_STOP = 40000 # save mesh files to check the mesh -SAVE_MESH_FILES = .false. +SAVE_MESH_FILES = .true. # restart files (number of runs can be 1 or higher, choose 1 for no restart files) NUMBER_OF_RUNS = 1 @@ -430,3 +430,9 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .true. +HDF5_FOR_MOVIES = .false. +HDF5_IO_NODES = 0 # HDF5 IO server with number of IO dedicated procs + diff --git a/Makefile.in b/Makefile.in index 40e3a3cd2..cf1b3e7ee 100644 --- a/Makefile.in +++ b/Makefile.in @@ -496,6 +496,20 @@ endif @COND_PETSC_TRUE@LDFLAGS += @PETSC_LDFLAGS@ @COND_PETSC_TRUE@MPILIBS += @HIP_LDFLAGS@ @PETSC_LIBS@ +####################################### +#### +#### HDF5 (parallel) +#### with configure: ./configure --with-hdf5 HDF5_LIBS=.. HDF5_FCFLAGS=.. HDF5_INC=.. +#### +####################################### + +@COND_HDF5_TRUE@HDF5 = yes +@COND_HDF5_FALSE@HDF5 = no + +# adds compiler flag +@COND_HDF5_TRUE@FCFLAGS += @HDF5_FCFLAGS@ @HDF5_INCLUDES@ $(FC_DEFINE)USE_HDF5 +@COND_HDF5_TRUE@LDFLAGS += @HDF5_LIBS@ -lhdf5_fortran -lhdf5hl_fortran # -lhdf5_hl -lhdf5 -lstdc++ + ####################################### ## static compilation ####################################### diff --git a/configure b/configure index c8c926281..ec9134a1c 100755 --- a/configure +++ b/configure @@ -645,6 +645,10 @@ LIBOBJS GIT_DATE_VERSION GIT_COMMIT_VERSION GIT_PACKAGE_VERSION +HDF5_INCLUDES +HDF5_LIBS +HDF5_FCFLAGS +HDF5_INC PETSC_LIBS PETSC_LDFLAGS PETSC_LIB @@ -740,6 +744,8 @@ ac_ct_FC LDFLAGS FCFLAGS FC +COND_HDF5_FALSE +COND_HDF5_TRUE COND_PETSC_FALSE COND_PETSC_TRUE COND_NETCDF_FALSE @@ -899,6 +905,7 @@ with_emc with_netcdf enable_netcdf with_petsc +with_hdf5 with_vtk with_vtk_suffix with_vtk_version @@ -954,7 +961,10 @@ EMC_FCFLAGS EMC_LIBS PETSC_INC PETSC_FCFLAGS -PETSC_LIB' +PETSC_LIB +HDF5_INC +HDF5_FCFLAGS +HDF5_LIBS' # Initialize some variables set by options. @@ -1613,6 +1623,7 @@ Optional Packages: --with-netcdf build NetCDF enabled version (IRIS EMC & CEM models) [default=no] --with-petsc build PETSc enabled version [default=no] + --with-hdf5 build HDF5 enabled version [default=no] --with-vtk The prefix where VTK is installed [default=/usr] --with-vtk-suffix Suffix to append to VTK's include directory, e.g., for vtk-5.2/ the suffix is "-5.2" @@ -1681,6 +1692,10 @@ Some influential environment variables: PETSC_FCFLAGS Fortran compiler flags (PETSC specific flags) PETSC_LIB extra library directory for linking programs with PETSC + HDF5_INC HDF5 include directory + HDF5_FCFLAGS + Fortran compiler flags + HDF5_LIBS extra libraries for linking programs Use these variables to override the choices made by 'configure' or to help it to find libraries and programs with nonstandard names/locations. @@ -4092,6 +4107,28 @@ else fi +### +### HDF5 +### + + +# Check whether --with-hdf5 was given. +if test ${with_hdf5+y} +then : + withval=$with_hdf5; want_hdf5="$withval" +else case e in #( + e) want_hdf5=no ;; +esac +fi + + if test x$"$want_hdf5" != xno; then + COND_HDF5_TRUE= + COND_HDF5_FALSE='#' +else + COND_HDF5_TRUE='#' + COND_HDF5_FALSE= +fi + ############################################################ @@ -11881,6 +11918,71 @@ ac_compiler_gnu=$ac_cv_fc_compiler_gnu +fi + +## +## HDF5 +## + +if test x"$want_hdf5" != xno +then : + + printf "%s\n" "## ---- ## +## HDF5 ## +## ---- ##" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: HDF5 is enabled" >&5 +printf "%s\n" "$as_me: HDF5 is enabled" >&6;} + + + + + + # HDF5 checking for Fortran + ac_ext=${ac_fc_srcext-f} +ac_compile='$FC -c $FCFLAGS $ac_fcflags_srcext conftest.$ac_ext >&5' +ac_link='$FC -o conftest$ac_exeext $FCFLAGS $LDFLAGS $ac_fcflags_srcext conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_fc_compiler_gnu + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HDF5 modules" >&5 +printf %s "checking for HDF5 modules... " >&6; } + FCFLAGS="$HDF5_FCFLAGS $FCFLAGS" + # adds include directory + if test "x$HDF5_INC" != "x"; then + HDF5_INCLUDES="-I$HDF5_INC" + FCFLAGS+=" $HDF5_INCLUDES" + fi + # compilation test + cat > conftest.$ac_ext <<_ACEOF + + program main + + use hdf5 + + end + +_ACEOF +if ac_fn_fc_try_compile "$LINENO" +then : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +else case e in #( + e) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + as_fn_error $? "HDF5 module not found; is HDF5 built with Fortran support for this compiler?" "$LINENO" 5 + ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ac_ext=${ac_fc_srcext-f} +ac_compile='$FC -c $FCFLAGS $ac_fcflags_srcext conftest.$ac_ext >&5' +ac_link='$FC -o conftest$ac_exeext $FCFLAGS $LDFLAGS $ac_fcflags_srcext conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_fc_compiler_gnu + + + fi ## @@ -12275,6 +12377,10 @@ if test -z "${COND_PETSC_TRUE}" && test -z "${COND_PETSC_FALSE}"; then as_fn_error $? "conditional \"COND_PETSC\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${COND_HDF5_TRUE}" && test -z "${COND_HDF5_FALSE}"; then + as_fn_error $? "conditional \"COND_HDF5\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi case $FC_MODINC in #( *\ ) FC_MODINC=$FC_MODINC'${ac_empty}' ;; esac diff --git a/configure.ac b/configure.ac index 49cc402d8..392bd4362 100644 --- a/configure.ac +++ b/configure.ac @@ -8,8 +8,8 @@ dnl You must have recent versions of Autoconf and Automake installed. ############################################################ -AC_PREREQ(2.61) -AC_INIT([Specfem3D_Globe], m4_normalize(m4_include([VERSION])), [see the wiki], [Specfem3DGlobe]) +AC_PREREQ([2.72]) +AC_INIT([Specfem3D_Globe],[m4_normalize(m4_include(VERSION))],[see the wiki],[Specfem3DGlobe]) AC_CONFIG_SRCDIR([src/specfem3D/specfem3D.F90]) AC_CONFIG_HEADERS([setup/config.h]) @@ -326,6 +326,16 @@ AC_ARG_WITH([petsc], [want_petsc=no]) AM_CONDITIONAL([COND_PETSC], [test x$"$want_petsc" != xno]) +### +### HDF5 +### + +AC_ARG_WITH([hdf5], + [AS_HELP_STRING([--with-hdf5], + [build HDF5 enabled version @<:@default=no@:>@])], + [want_hdf5="$withval"], + [want_hdf5=no]) +AM_CONDITIONAL([COND_HDF5], [test x$"$want_hdf5" != xno]) ############################################################ @@ -944,6 +954,42 @@ AS_IF([test x"$want_petsc" != xno], [ AC_SUBST([PETSC_LIBS]) ]) +## +## HDF5 +## + +AS_IF([test x"$want_hdf5" != xno], [ + AS_BOX([HDF5]) + AC_MSG_NOTICE([HDF5 is enabled]) + + AC_ARG_VAR(HDF5_INC, [HDF5 include directory]) + AC_ARG_VAR(HDF5_FCFLAGS, [Fortran compiler flags]) + AC_ARG_VAR(HDF5_LIBS, [extra libraries for linking programs]) + + # HDF5 checking for Fortran + AC_LANG_PUSH(Fortran) + AC_MSG_CHECKING([for HDF5 modules]) + FCFLAGS="$HDF5_FCFLAGS $FCFLAGS" + # adds include directory + if test "x$HDF5_INC" != "x"; then + HDF5_INCLUDES="-I$HDF5_INC" + FCFLAGS+=" $HDF5_INCLUDES" + fi + # compilation test + AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM([], [[ + use hdf5 + ]]) + ], [ + AC_MSG_RESULT(yes) + ], [ + AC_MSG_RESULT(no) + AC_MSG_ERROR([HDF5 module not found; is HDF5 built with Fortran support for this compiler?]) + ]) + AC_LANG_POP([Fortran]) + AC_SUBST([HDF5_INCLUDES]) +]) + ## ## Git version info ## diff --git a/src/gindex3D/rules.mk b/src/gindex3D/rules.mk index 778ae6f35..a5607dfd9 100644 --- a/src/gindex3D/rules.mk +++ b/src/gindex3D/rules.mk @@ -48,6 +48,7 @@ gindex3D_SHARED_OBJECTS = \ $O/bcast_mesh_databases.solverstatic.o \ $O/locate_regular_points.solverstatic.o \ $O/read_arrays_solver.solverstatic.o \ + $O/read_arrays_solver_hdf5.solverstatic.o \ $O/read_mesh_parameters.solverstatic.o \ $O/read_mesh_databases.solverstatic.o \ $O/specfem3D_par.solverstatic_module.o \ @@ -68,6 +69,7 @@ gindex3D_SHARED_OBJECTS += \ $O/get_model_parameters.shared.o \ $O/get_timestep_and_layers.shared.o \ $O/hex_nodes.shared.o \ + $O/hdf5_manager.shared_hdf5_module.o \ $O/memory_eval.shared.o \ $O/parallel.sharedmpi.o \ $O/param_reader.cc.o \ diff --git a/src/meshfem3D/create_regions_mesh.F90 b/src/meshfem3D/create_regions_mesh.F90 index 8c1bad88a..676fa75ec 100644 --- a/src/meshfem3D/create_regions_mesh.F90 +++ b/src/meshfem3D/create_regions_mesh.F90 @@ -51,7 +51,8 @@ subroutine create_regions_mesh(npointot, & SAVE_BOUNDARY_MESH,SAVE_MESHFILES_AVS_DX_FORMAT use shared_parameters, only: & - R_CENTRAL_CUBE,RICB,RCMB,RINF + R_CENTRAL_CUBE,RICB,RCMB,RINF, & + HDF5_ENABLED use meshfem_par, only: & myrank,nspec,nglob,iregion_code, & @@ -460,6 +461,10 @@ subroutine create_regions_mesh(npointot, & call save_arrays_solver_adios(idoubling,ibool,xstore,ystore,zstore, & NSPEC2DMAX_XMIN_XMAX, NSPEC2DMAX_YMIN_YMAX, & NSPEC2D_TOP,NSPEC2D_BOTTOM) + else if (HDF5_ENABLED) then + call save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & + NSPEC2DMAX_XMIN_XMAX, NSPEC2DMAX_YMIN_YMAX, & + NSPEC2D_TOP,NSPEC2D_BOTTOM) else call save_arrays_solver(idoubling,ibool,xstore,ystore,zstore, & NSPEC2D_TOP,NSPEC2D_BOTTOM) @@ -480,6 +485,8 @@ subroutine create_regions_mesh(npointot, & ! saves boundary file if (ADIOS_FOR_ARRAYS_SOLVER) then call save_arrays_boundary_adios() + else if (HDF5_ENABLED) then + call save_arrays_boundary_hdf5() else call save_arrays_boundary() endif @@ -501,6 +508,8 @@ subroutine create_regions_mesh(npointot, & if (ADIOS_FOR_SOLVER_MESHFILES) then ! adios file output call save_model_meshfiles_adios() + else if (HDF5_ENABLED) then + call save_model_meshfiles_hdf5() else ! outputs model files in binary format call save_model_meshfiles() diff --git a/src/meshfem3D/get_absorb.f90 b/src/meshfem3D/get_absorb.f90 index 28d6e3672..177f60c3f 100644 --- a/src/meshfem3D/get_absorb.f90 +++ b/src/meshfem3D/get_absorb.f90 @@ -34,6 +34,8 @@ subroutine get_absorb(prname,iregion,iboun, & use constants use meshfem_par, only: ADIOS_FOR_ARRAYS_SOLVER,nspec + use shared_parameters, only: HDF5_ENABLED + implicit none integer,intent(in) :: iregion @@ -146,6 +148,9 @@ subroutine get_absorb(prname,iregion,iboun, & call get_absorb_adios(iregion, & nimin, nimax, njmin, njmax, nkmin_xi, nkmin_eta, & NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX) + else if (HDF5_ENABLED) then + ! no HDF5 support yet + call exit_MPI(myrank,'HDF5 not supported yet for old format Stacey boundary condition output') else open(unit=IOUT,file=prname(1:len_trim(prname))//'stacey.old_format.bin', & status='unknown',form='unformatted',action='write',iostat=ier) @@ -196,8 +201,8 @@ subroutine get_absorb_create_Stacey_boundary_arrays(iregion,NSPEC2D_BOTTOM) abs_boundary_ijk,abs_boundary_normal,abs_boundary_jacobian2Dw ! input parameters - use shared_parameters, only: REGIONAL_MESH_CUTOFF,ADIOS_FOR_ARRAYS_SOLVER - + use shared_parameters, only: REGIONAL_MESH_CUTOFF,ADIOS_FOR_ARRAYS_SOLVER, & + HDF5_ENABLED ! debug use meshfem_par, only: nspec,nglob,ibool,xstore_glob,ystore_glob,zstore_glob @@ -511,6 +516,10 @@ subroutine get_absorb_create_Stacey_boundary_arrays(iregion,NSPEC2D_BOTTOM) call get_absorb_stacey_boundary_adios(iregion, num_abs_boundary_faces, & abs_boundary_ispec,abs_boundary_npoin, & abs_boundary_ijk,abs_boundary_normal,abs_boundary_jacobian2Dw) + else if (HDF5_ENABLED) then + call get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & + abs_boundary_ispec,abs_boundary_npoin, & + abs_boundary_ijk,abs_boundary_normal,abs_boundary_jacobian2Dw) else ! binary format open(unit=IOUT,file=prname(1:len_trim(prname))//'stacey.bin', & diff --git a/src/meshfem3D/initialize_mesher.f90 b/src/meshfem3D/initialize_mesher.f90 index 264528242..c84f775a4 100644 --- a/src/meshfem3D/initialize_mesher.f90 +++ b/src/meshfem3D/initialize_mesher.f90 @@ -119,6 +119,7 @@ subroutine im_initialize_system() use meshfem_models_par use manager_adios + use manager_hdf5 implicit none @@ -135,6 +136,11 @@ subroutine im_initialize_system() call initialize_adios() endif + ! HDF5 + if (HDF5_ENABLED) then + call h5_initialize() + endif + ! gravity integrals if (GRAVITY_INTEGRALS) then call gravity_initialize_integrals() diff --git a/src/meshfem3D/rules.mk b/src/meshfem3D/rules.mk index 7d6ff939e..e7eef11b0 100644 --- a/src/meshfem3D/rules.mk +++ b/src/meshfem3D/rules.mk @@ -119,7 +119,9 @@ meshfem3D_MESHER_OBJECTS = \ $O/model_spiral.check.o \ $O/moho_stretching.check.o \ $O/save_arrays_solver.check.o \ + $O/save_arrays_solver_hdf5.check.o \ $O/save_model_meshfiles.check.o \ + $O/save_model_meshfiles_hdf5.check.o \ $O/setup_color_perm.check.o \ $O/setup_counters.check.o \ $O/setup_inner_outer.check.o \ @@ -209,6 +211,7 @@ meshfem3D_SHARED_OBJECTS = \ $O/get_model_parameters.shared.o \ $O/get_timestep_and_layers.shared.o \ $O/gll_library.shared.o \ + $O/hdf5_manager.shared_hdf5_module.o \ $O/heap_sort.shared.o \ $O/hex_nodes.shared.o \ $O/init_openmp.shared.o \ @@ -398,3 +401,11 @@ $O/%.check_adios.o: $S/%.F90 $O/shared_par.shared_module.o $O/meshfem3D_par.chec $O/%.checknetcdf.o: $S/%.f90 $O/shared_par.shared_module.o $O/meshfem3D_par.check_module.o ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(NETCDF_INCLUDE) -c -o $@ $< + +## HDF5 file i/o + +#$O/%.mesh_hdf5.o: $S/%.F90 $O/shared_par.shared_module.o $O/meshfem3D_par.check_module.o $O/hdf5_manager.shared_hdf5_module.o +# ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< +# +#$O/%.mesh_hdf5.o: $S/%.f90 $O/shared_par.shared_module.o $O/meshfem3D_par.check_module.o $O/hdf5_manager.shared_hdf5_module.o +# ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< diff --git a/src/meshfem3D/save_arrays_solver.f90 b/src/meshfem3D/save_arrays_solver.f90 index c91368e26..772baf932 100644 --- a/src/meshfem3D/save_arrays_solver.f90 +++ b/src/meshfem3D/save_arrays_solver.f90 @@ -332,6 +332,8 @@ subroutine save_arrays_solver_MPI() IREGION_TRINFINITE,IREGION_INFINITE, & ADIOS_FOR_MPI_ARRAYS + use shared_parameters, only: HDF5_ENABLED + ! use MPI_interfaces_par use MPI_crust_mantle_par @@ -355,6 +357,15 @@ subroutine save_arrays_solver_MPI() num_phase_ispec_crust_mantle,phase_ispec_inner_crust_mantle, & num_colors_outer_crust_mantle,num_colors_inner_crust_mantle, & num_elem_colors_crust_mantle) + else if (HDF5_ENABLED) then + call save_MPI_arrays_hdf5(IREGION_CRUST_MANTLE,LOCAL_PATH, & + num_interfaces_crust_mantle,max_nibool_interfaces_cm, & + my_neighbors_crust_mantle,nibool_interfaces_crust_mantle, & + ibool_interfaces_crust_mantle, & + nspec_inner_crust_mantle,nspec_outer_crust_mantle, & + num_phase_ispec_crust_mantle,phase_ispec_inner_crust_mantle, & + num_colors_outer_crust_mantle,num_colors_inner_crust_mantle, & + num_elem_colors_crust_mantle) else call save_MPI_arrays(IREGION_CRUST_MANTLE,LOCAL_PATH, & num_interfaces_crust_mantle,max_nibool_interfaces_cm, & @@ -377,6 +388,15 @@ subroutine save_arrays_solver_MPI() num_phase_ispec_outer_core,phase_ispec_inner_outer_core, & num_colors_outer_outer_core,num_colors_inner_outer_core, & num_elem_colors_outer_core) + else if (HDF5_ENABLED) then + call save_MPI_arrays_hdf5(IREGION_OUTER_CORE,LOCAL_PATH, & + num_interfaces_outer_core,max_nibool_interfaces_oc, & + my_neighbors_outer_core,nibool_interfaces_outer_core, & + ibool_interfaces_outer_core, & + nspec_inner_outer_core,nspec_outer_outer_core, & + num_phase_ispec_outer_core,phase_ispec_inner_outer_core, & + num_colors_outer_outer_core,num_colors_inner_outer_core, & + num_elem_colors_outer_core) else call save_MPI_arrays(IREGION_OUTER_CORE,LOCAL_PATH, & num_interfaces_outer_core,max_nibool_interfaces_oc, & @@ -399,6 +419,16 @@ subroutine save_arrays_solver_MPI() num_phase_ispec_inner_core,phase_ispec_inner_inner_core, & num_colors_outer_inner_core,num_colors_inner_inner_core, & num_elem_colors_inner_core) + else if (HDF5_ENABLED) then + call save_MPI_arrays_hdf5(IREGION_INNER_CORE,LOCAL_PATH, & + num_interfaces_inner_core,max_nibool_interfaces_ic, & + my_neighbors_inner_core,nibool_interfaces_inner_core, & + ibool_interfaces_inner_core, & + nspec_inner_inner_core,nspec_outer_inner_core, & + num_phase_ispec_inner_core,phase_ispec_inner_inner_core, & + num_colors_outer_inner_core,num_colors_inner_inner_core, & + num_elem_colors_inner_core) + else call save_MPI_arrays(IREGION_INNER_CORE,LOCAL_PATH, & num_interfaces_inner_core,max_nibool_interfaces_ic, & @@ -421,6 +451,15 @@ subroutine save_arrays_solver_MPI() num_phase_ispec_trinfinite,phase_ispec_inner_trinfinite, & num_colors_outer_trinfinite,num_colors_inner_trinfinite, & num_elem_colors_trinfinite) + else if (HDF5_ENABLED) then + call save_MPI_arrays_hdf5(IREGION_TRINFINITE,LOCAL_PATH, & + num_interfaces_trinfinite,max_nibool_interfaces_trinfinite, & + my_neighbors_trinfinite,nibool_interfaces_trinfinite, & + ibool_interfaces_trinfinite, & + nspec_inner_trinfinite,nspec_outer_trinfinite, & + num_phase_ispec_trinfinite,phase_ispec_inner_trinfinite, & + num_colors_outer_trinfinite,num_colors_inner_trinfinite, & + num_elem_colors_trinfinite) else call save_MPI_arrays(IREGION_TRINFINITE,LOCAL_PATH, & num_interfaces_trinfinite,max_nibool_interfaces_trinfinite, & @@ -443,6 +482,15 @@ subroutine save_arrays_solver_MPI() num_phase_ispec_infinite,phase_ispec_inner_infinite, & num_colors_outer_infinite,num_colors_inner_infinite, & num_elem_colors_infinite) + else if (HDF5_ENABLED) then + call save_MPI_arrays_hdf5(IREGION_INFINITE,LOCAL_PATH, & + num_interfaces_infinite,max_nibool_interfaces_infinite, & + my_neighbors_infinite,nibool_interfaces_infinite, & + ibool_interfaces_infinite, & + nspec_inner_infinite,nspec_outer_infinite, & + num_phase_ispec_infinite,phase_ispec_inner_infinite, & + num_colors_outer_infinite,num_colors_inner_infinite, & + num_elem_colors_infinite) else call save_MPI_arrays(IREGION_INFINITE,LOCAL_PATH, & num_interfaces_infinite,max_nibool_interfaces_infinite, & diff --git a/src/meshfem3D/save_arrays_solver_hdf5.F90 b/src/meshfem3D/save_arrays_solver_hdf5.F90 new file mode 100644 index 000000000..7e2fbb0f4 --- /dev/null +++ b/src/meshfem3D/save_arrays_solver_hdf5.F90 @@ -0,0 +1,1107 @@ + subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & + NSPEC2D_TOP,NSPEC2D_BOTTOM) + + use constants + + use meshfem_par, only: & + nspec + +#ifdef USE_HDF5 + use shared_parameters, only: LOCAL_PATH, ATT_F_C_SOURCE + use meshfem_models_par, only: & + OCEANS,TRANSVERSE_ISOTROPY,ANISOTROPIC_3D_MANTLE, & + ANISOTROPIC_INNER_CORE,ATTENUATION + use meshfem_par, only: & + nglob, iregion_code, & + NCHUNKS,ABSORBING_CONDITIONS, & + ROTATION,EXACT_MASS_MATRIX_FOR_ROTATION, & + OUTPUT_FILES,xstore_glob,ystore_glob,zstore_glob, & + NPROCTOT, & + ATT1,ATT2,ATT3 + use regions_mesh_par2, only: & + xixstore,xiystore,xizstore,etaxstore,etaystore,etazstore, & + gammaxstore,gammaystore,gammazstore, & + rhostore,kappavstore,kappahstore,muvstore,muhstore,eta_anisostore, & + c11store,c12store,c13store,c14store,c15store,c16store,c22store, & + c23store,c24store,c25store,c26store,c33store,c34store,c35store, & + c36store,c44store,c45store,c46store,c55store,c56store,c66store, & + mu0store, & + rmassx,rmassy,rmassz,rmass_ocean_load, & + b_rmassx,b_rmassy, & + ibelm_xmin,ibelm_xmax,ibelm_ymin,ibelm_ymax,ibelm_bottom,ibelm_top, & + normal_xmin,normal_xmax,normal_ymin,normal_ymax,normal_bottom,normal_top, & + jacobian2D_xmin,jacobian2D_xmax,jacobian2D_ymin,jacobian2D_ymax, & + jacobian2D_bottom,jacobian2D_top, & + rho_vp,rho_vs, & + nspec2D_xmin,nspec2D_xmax,nspec2D_ymin,nspec2D_ymax, & + ispec_is_tiso, & + tau_s_store,tau_e_store,Qmu_store, & + prname, nglob_oceans, nglob_xy + + + + use manager_hdf5 +#endif + + implicit none + + ! doubling mesh flag + integer,dimension(nspec),intent(in) :: idoubling + integer,dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(in) :: ibool + + ! arrays with the mesh in double precision + double precision,dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(in) :: xstore,ystore,zstore + + ! boundary parameters locator + integer,intent(in) :: NSPEC2D_TOP,NSPEC2D_BOTTOM + +#ifdef USE_HDF5 + ! local parameters + integer :: i,j,k,ispec,iglob,ier + real(kind=CUSTOM_REAL),dimension(:),allocatable :: tmp_array + + ! MPI variables + integer :: info, comm + + logical :: if_collective = .true. + + ! processor dependent group names + character(len=64) :: gname_region, fname + + ! offset arrays + integer, dimension(0:NPROCTOT-1) :: offset_nnodes + integer, dimension(0:NPROCTOT-1) :: offset_nnodes_xy + integer, dimension(0:NPROCTOT-1) :: offset_nnodes_oceans + integer, dimension(0:NPROCTOT-1) :: offset_nelems + + ! nspec2d_* arrays (for storing actual number of elements) + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_xmin + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_xmax + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_ymin + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_ymax + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_bottom + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_top + + ! sub array for nspec2d_* arrays (for storing the padded arrays) + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_xmin + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_xmax + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_ymin + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_ymax + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_bottom + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_top + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT) + + ! share the offset info + ! + ! offset_nnodes + ! offset_nelems + ! offset_n_elms_bounds + call gather_all_all_singlei(nglob, offset_nnodes, NPROCTOT) + call gather_all_all_singlei(nglob_xy, offset_nnodes_xy, NPROCTOT) + call gather_all_all_singlei(nglob_oceans, offset_nnodes_oceans, NPROCTOT) + call gather_all_all_singlei(nspec, offset_nelems, NPROCTOT) + + ! arr_nspec2D_* arrays + call gather_all_all_singlei(nspec2D_xmin, act_arr_nspec2D_xmin, NPROCTOT) + call gather_all_all_singlei(nspec2D_xmax, act_arr_nspec2D_xmax, NPROCTOT) + call gather_all_all_singlei(nspec2D_ymin, act_arr_nspec2D_ymin, NPROCTOT) + call gather_all_all_singlei(nspec2D_ymax, act_arr_nspec2D_ymax, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_BOTTOM, act_arr_nspec2D_bottom, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_TOP, act_arr_nspec2D_top, NPROCTOT) + call gather_all_all_singlei(size(ibelm_xmin), arr_nspec2D_xmin, NPROCTOT) + call gather_all_all_singlei(size(ibelm_xmax), arr_nspec2D_xmax, NPROCTOT) + call gather_all_all_singlei(size(ibelm_ymin), arr_nspec2D_ymin, NPROCTOT) + call gather_all_all_singlei(size(ibelm_ymax), arr_nspec2D_ymax, NPROCTOT) + call gather_all_all_singlei(size(ibelm_bottom), arr_nspec2D_bottom, NPROCTOT) + call gather_all_all_singlei(size(ibelm_top), arr_nspec2D_top, NPROCTOT) + + ! + ! prepare file, group and dataset by myrank == 0 + ! + if (myrank == 0) then + ! create and open solver_data.h5 + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'solver_data.h5' + if (iregion_code == 1) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + + ! create group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + call h5_create_group(gname_region) + + ! open group for iregion_code + call h5_open_group(gname_region) + + ! create dataset for offset arrays and store them by myrank == 0 + + call h5_create_dataset_gen_in_group('offset_nnodes', (/NPROCTOT/), 1, 1) ! nnode = nglob + call h5_create_dataset_gen_in_group('offset_nnodes_xy', (/NPROCTOT/), 1, 1) ! nnode = nglob_xy + call h5_create_dataset_gen_in_group('offset_nnodes_oceans', (/NPROCTOT/), 1, 1) ! nnode = nglob_oceans + call h5_create_dataset_gen_in_group('offset_nelems' , (/NPROCTOT/), 1, 1) ! nspec = nelem + + ! the other dataset need to be written by all ranks + ! so here we just create the dataset + call h5_create_dataset_gen_in_group('xstore', (/sum(offset_nnodes(:))/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('ystore', (/sum(offset_nnodes(:))/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('zstore', (/sum(offset_nnodes(:))/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('ibool', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, 1) + call h5_create_dataset_gen_in_group('idoubling', (/sum(offset_nelems(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ispec_is_tiso', (/sum(offset_nelems(:))/), 1, 0) + call h5_create_dataset_gen_in_group('xixstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('xiystore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('xizstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('etaxstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('etaystore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('etazstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('gammaxstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('gammaystore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('gammazstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('rhostore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('kappavstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + if (iregion_code /= IREGION_OUTER_CORE) then + call h5_create_dataset_gen_in_group('muvstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + + select case(iregion_code) + case (IREGION_CRUST_MANTLE) + ! crusl/mantle region mesh + ! save anisotropy in the mantle only + if (ANISOTROPIC_3D_MANTLE) then + call h5_create_dataset_gen_in_group('c11store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c12store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c13store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c14store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c15store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c16store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c22store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c23store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c24store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c25store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c26store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c33store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c34store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c35store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c36store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c44store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c45store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c46store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c55store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c56store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c66store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + else + if (TRANSVERSE_ISOTROPY) then + call h5_create_dataset_gen_in_group('kappahstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('muhstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('eta_anisostore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + endif + ! for azimutahl aniso kernels + call h5_create_dataset_gen_in_group('mu0store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + + case(IREGION_INNER_CORE) + ! inner core region mesh + if (ANISOTROPIC_INNER_CORE) then + call h5_create_dataset_gen_in_group('c11store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c12store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c13store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c33store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c44store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + end select + + ! Stacy + if (ABSORBING_CONDITIONS) then + if (iregion_code == IREGION_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group('rho_vp', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('rho_vs', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + else if (iregion_code == IREGION_OUTER_CORE) then + call h5_create_dataset_gen_in_group('rho_vp', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + endif + + ! mass matrices + if (((NCHUNKS /= 6 .and. ABSORBING_CONDITIONS) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) .and. iregion_code == IREGION_INNER_CORE)) then + call h5_create_dataset_gen_in_group('rmassx', (/sum(offset_nnodes_xy(:))/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('rmassy', (/sum(offset_nnodes_xy(:))/), 1, CUSTOM_REAL) + endif + + call h5_create_dataset_gen_in_group('rmassz', (/sum(offset_nnodes(:))/), 1, CUSTOM_REAL) + + ! maxx matrices for backward simulation when ROTATION is .true. + if (ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) then + if (iregion_code == IREGION_CRUST_MANTLE .or. iregion_code == IREGION_INNER_CORE) then + call h5_create_dataset_gen_in_group('b_rmassx', (/sum(offset_nnodes_xy(:))/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('b_rmassy', (/sum(offset_nnodes_xy(:))/), 1, CUSTOM_REAL) + endif + endif + + ! ocean load mass matrix + if (OCEANS .and. iregion_code == IREGION_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group('rmass_ocean_load', (/sum(offset_nnodes_oceans(:))/), 1, CUSTOM_REAL) + endif + + ! close group for iregion_code + call h5_close_group() + ! close solver_data.h5 + call h5_close_file() + + ! create and open bounday.h5 + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'boundary.h5' + if (iregion_code == 1) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + + ! create group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + call h5_create_group(gname_region) + ! open group for iregion_code + call h5_open_group(gname_region) + + ! create node/element number dumps + call h5_create_dataset_gen_in_group('nspec2D_xmin', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('nspec2D_xmax', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('nspec2D_ymin', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('nspec2D_ymax', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('nspec2D_bottom', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('nspec2D_top', (/NPROCTOT/), 1, 1) + + call h5_create_dataset_gen_in_group('sub_nspec2D_xmin', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_nspec2D_xmax', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_nspec2D_ymin', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_nspec2D_ymax', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_nspec2D_bottom', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_nspec2D_top', (/NPROCTOT/), 1, 1) + + call h5_create_dataset_gen_in_group('ibelm_xmin', (/sum(arr_nspec2d_xmin(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_xmax', (/sum(arr_nspec2d_xmax(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_ymin', (/sum(arr_nspec2d_ymin(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_ymax', (/sum(arr_nspec2d_ymax(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_bottom', (/sum(arr_nspec2d_bottom(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_top', (/sum(arr_nspec2d_top(:))/), 1, 1) + + call h5_create_dataset_gen_in_group('normal_xmin', (/NDIM,NGLLY,NGLLZ,sum(arr_nspec2d_xmin(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_xmax', (/NDIM,NGLLY,NGLLZ,sum(arr_nspec2d_xmax(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_ymin', (/NDIM,NGLLX,NGLLZ,sum(arr_nspec2d_ymin(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_ymax', (/NDIM,NGLLX,NGLLZ,sum(arr_nspec2d_ymax(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_bottom', (/NDIM,NGLLX,NGLLY,sum(arr_nspec2d_bottom(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_top', (/NDIM,NGLLX,NGLLY,sum(arr_nspec2d_top(:))/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen_in_group('jacobian2D_xmin', (/NGLLY,NGLLZ,sum(arr_nspec2d_xmin(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('jacobian2D_xmax', (/NGLLY,NGLLZ,sum(arr_nspec2d_xmin(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('jacobian2D_ymin', (/NGLLX,NGLLZ,sum(arr_nspec2d_ymin(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('jacobian2D_ymax', (/NGLLX,NGLLZ,sum(arr_nspec2d_ymax(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('jacobian2D_bottom', (/NGLLX,NGLLY,sum(arr_nspec2d_bottom(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('jacobian2D_top', (/NGLLX,NGLLY,sum(arr_nspec2d_top(:))/), 3, CUSTOM_REAL) + + ! close group for iregion_code + call h5_close_group() + ! close boundary.h5 + call h5_close_file() + + ! create and open attenuation.h5 + if (ATTENUATION) then + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'attenuation.h5' + if (iregion_code == 1) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + + ! create group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + call h5_create_group(gname_region) + ! open group for iregion_code + call h5_open_group(gname_region) + + ! create dataset for attenuation + call h5_create_dataset_gen_in_group('tau_s_store', (/N_SLS*NPROCTOT/), 1, 8) + call h5_create_dataset_gen_in_group('tau_e_store', (/ATT1, ATT2, ATT3, N_SLS, sum(offset_nelems(:))/), 5, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('Qmu_store', (/ATT1, ATT2, ATT3, sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('att_f_c_source', (/NPROCTOT/), 1, 8) + + call h5_close_group() + call h5_close_file() + endif + + endif ! myrank == 0 + + ! + ! write the data to the HDF5 file by all ranks + ! (except the number of nodes/elements, which is written by myrank == 0) + ! + ! create and open solver_data.h5 + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'solver_data.h5' + call h5_open_file_p_collect(name_database_hdf5) + + ! open group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + call h5_open_group(gname_region) + + call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes", (/offset_nnodes(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes_xy", (/offset_nnodes_xy(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes_oceans", (/offset_nnodes_oceans(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("offset_nelems", (/offset_nelems(myrank)/), (/myrank/), if_collective) + + allocate(tmp_array(nglob),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array for mesh topology') + + !--- x coordinate + tmp_array(:) = 0._CUSTOM_REAL + do ispec = 1,nspec + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + iglob = ibool(i,j,k,ispec) + ! distinguish between single and double precision for reals + tmp_array(iglob) = real(xstore(i,j,k,ispec), kind=CUSTOM_REAL) + enddo + enddo + enddo + enddo + + call h5_write_dataset_collect_hyperslab_in_group('xstore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), if_collective) + !--- y coordinate + tmp_array(:) = 0._CUSTOM_REAL + do ispec = 1,nspec + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + iglob = ibool(i,j,k,ispec) + ! distinguish between single and double precision for reals + tmp_array(iglob) = real(ystore(i,j,k,ispec), kind=CUSTOM_REAL) + enddo + enddo + enddo + enddo + call h5_write_dataset_collect_hyperslab_in_group('ystore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), if_collective) + !--- z coordinate + tmp_array(:) = 0._CUSTOM_REAL + do ispec = 1,nspec + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + iglob = ibool(i,j,k,ispec) + ! distinguish between single and double precision for reals + tmp_array(iglob) = real(zstore(i,j,k,ispec), kind=CUSTOM_REAL) + enddo + enddo + enddo + enddo + call h5_write_dataset_collect_hyperslab_in_group('zstore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), if_collective) + deallocate(tmp_array) + + call h5_write_dataset_collect_hyperslab_in_group('ibool', ibool, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('idoubling', idoubling, (/sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ispec_is_tiso', ispec_is_tiso, (/sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('xixstore', xixstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('xiystore', xiystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('xizstore', xizstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('etaxstore', etaxstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('etaystore', etaystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('etazstore', etazstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('gammaxstore', gammaxstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('gammaystore', gammaystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('gammazstore', gammazstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('rhostore', rhostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('kappavstore', kappavstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + if (iregion_code /= IREGION_OUTER_CORE) then + call h5_write_dataset_collect_hyperslab_in_group('muvstore', muvstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + endif + + select case(iregion_code) + case (IREGION_CRUST_MANTLE) + ! crusl/mantle region mesh + ! save anisotropy in the mantle only + if (ANISOTROPIC_3D_MANTLE) then + call h5_write_dataset_collect_hyperslab_in_group('c11store', c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c12store', c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c13store', c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c14store', c14store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c15store', c15store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c16store', c16store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c22store', c22store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c23store', c23store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c24store', c24store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c25store', c25store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c26store', c26store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c33store', c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c34store', c34store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c35store', c35store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c36store', c36store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c44store', c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c45store', c45store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c46store', c46store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c55store', c55store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c56store', c56store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c66store', c66store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + else + if (TRANSVERSE_ISOTROPY) then + call h5_write_dataset_collect_hyperslab_in_group('kappahstore', kappahstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('muhstore', muhstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('eta_anisostore', eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + endif + endif + ! for azimutahl aniso kernels + call h5_write_dataset_collect_hyperslab_in_group('mu0store', mu0store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) +!! + case(IREGION_INNER_CORE) + ! inner core region mesh + if (ANISOTROPIC_INNER_CORE) then + call h5_write_dataset_collect_hyperslab_in_group('c11store', c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c12store', c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c13store', c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c33store', c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c44store', c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + endif + end select +!! + ! Stacy + if (ABSORBING_CONDITIONS) then + if (iregion_code == IREGION_CRUST_MANTLE) then + call h5_write_dataset_collect_hyperslab_in_group('rho_vp', rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('rho_vs', rho_vs, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + else if (iregion_code == IREGION_OUTER_CORE) then + call h5_write_dataset_collect_hyperslab_in_group('rho_vp', rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + endif + endif +!!! + ! mass matrices + if (((NCHUNKS /= 6 .and. ABSORBING_CONDITIONS) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) .and. iregion_code == IREGION_INNER_CORE)) then + call h5_write_dataset_collect_hyperslab_in_group('rmassx', rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('rmassy', rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), if_collective) + endif + + call h5_write_dataset_collect_hyperslab_in_group('rmassz', rmassz, (/sum(offset_nnodes(0:myrank-1))/), if_collective) + + ! maxx matrices for backward simulation when ROTATION is .true. + if (ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) then + if (iregion_code == IREGION_CRUST_MANTLE .or. iregion_code == IREGION_INNER_CORE) then + call h5_write_dataset_collect_hyperslab_in_group('b_rmassx', b_rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('b_rmassy', b_rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), if_collective) + endif + endif + + ! ocean load mass matrix + if (OCEANS .and. iregion_code == IREGION_CRUST_MANTLE) then + call h5_write_dataset_collect_hyperslab_in_group('rmass_ocean_load', rmass_ocean_load, (/sum(offset_nnodes_oceans(0:myrank-1))/), if_collective) + endif + + ! close group for iregion_code + call h5_close_group() + ! close solver_data.h5 + call h5_close_file_p() + + ! create and open bounday.h5 + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'boundary.h5' + call h5_open_file_p_collect(name_database_hdf5) + + ! create group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + ! open group for iregion_code + call h5_open_group(gname_region) + + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_xmin', (/act_arr_nspec2d_xmin(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_xmax', (/act_arr_nspec2d_xmax(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_ymin', (/act_arr_nspec2d_ymin(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_ymax', (/act_arr_nspec2d_ymax(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_bottom', (/act_arr_nspec2d_bottom(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_top', (/act_arr_nspec2d_top(myrank)/), (/myrank/), if_collective) + + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_xmin', (/arr_nspec2d_xmin(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_xmax', (/arr_nspec2d_xmax(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_ymin', (/arr_nspec2d_ymin(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_ymax', (/arr_nspec2d_ymax(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_bottom', (/arr_nspec2d_bottom(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_top', (/arr_nspec2d_top(myrank)/), (/myrank/), if_collective) + + if (arr_nspec2d_xmin(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmin', ibelm_xmin, (/sum(arr_nspec2d_xmin(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('normal_xmin', normal_xmin, (/0,0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmin', jacobian2D_xmin, (/0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), if_collective) + endif + if (arr_nspec2d_xmax(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmax', ibelm_xmax, (/sum(arr_nspec2d_xmax(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('normal_xmax', normal_xmax, (/0,0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmax', jacobian2D_xmax, (/0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), if_collective) + endif + if (arr_nspec2d_ymin(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymin', ibelm_ymin, (/sum(arr_nspec2d_ymin(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('normal_ymin', normal_ymin, (/0,0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymin', jacobian2D_ymin, (/0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), if_collective) + endif + if (arr_nspec2d_ymax(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymax', ibelm_ymax, (/sum(arr_nspec2d_ymax(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('normal_ymax', normal_ymax, (/0,0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymax', jacobian2D_ymax, (/0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), if_collective) + endif + if (arr_nspec2d_bottom(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_bottom', ibelm_bottom, (/sum(arr_nspec2d_bottom(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('normal_bottom', normal_bottom, (/0,0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_bottom', jacobian2D_bottom, (/0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), if_collective) + endif + if (arr_nspec2d_top(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_top', ibelm_top, (/sum(arr_nspec2d_top(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('normal_top', normal_top, (/0,0,0,sum(arr_nspec2d_top(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_top', jacobian2D_top, (/0,0,sum(arr_nspec2d_top(0:myrank-1))/), if_collective) + endif + + ! close group for iregion_code + call h5_close_group() + ! close boundary.h5 + call h5_close_file_p() + + ! create and open attenuation.h5 + if (ATTENUATION) then + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'attenuation.h5' + call h5_open_file_p_collect(name_database_hdf5) + ! create group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + ! open group for iregion_code + call h5_open_group(gname_region) + + ! create dataset for attenuation + call h5_write_dataset_collect_hyperslab_in_group('tau_s_store', tau_s_store, (/myrank*N_SLS/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('tau_e_store', tau_e_store, (/0,0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('Qmu_store', Qmu_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('att_f_c_source', (/ATT_F_C_SOURCE/), (/myrank/), if_collective) + + call h5_close_group() + call h5_close_file_p() + endif + +#else + ! no HDF5 compilation + + ! user output + print * + print *, "Error: HDF5 routine save_databases_hdf5() called without HDF5 Support." + print *, "To enable HDF5 support, reconfigure with --with-hdf5 flag." + print * + stop 'Error HDF5 save_databases_hdf5(): called without compilation support' + +#endif + +end subroutine save_arrays_solver_hdf5 + +! +!----------------------------------------------------------------------- +! + + subroutine save_arrays_boundary_hdf5() + +! saves arrays for boundaries such as MOHO, 400 and 670 discontinuities + +#ifdef USE_HDF5 + use shared_parameters, only: LOCAL_PATH + + use constants, only: myrank,SUPPRESS_CRUSTAL_MESH,CUSTOM_REAL, & + NDIM, NGLLX, NGLLY + + use meshfem_models_par, only: & + HONOR_1D_SPHERICAL_MOHO + + use meshfem_par, only: & + iregion_code, NPROCTOT + +! boundary kernels + use regions_mesh_par2, only: & + NSPEC2D_MOHO, NSPEC2D_400, NSPEC2D_670, & + ibelm_moho_top,ibelm_moho_bot,ibelm_400_top,ibelm_400_bot, & + ibelm_670_top,ibelm_670_bot,normal_moho,normal_400,normal_670, & + ispec2D_moho_top,ispec2D_moho_bot,ispec2D_400_top,ispec2D_400_bot, & + ispec2D_670_top,ispec2D_670_bot, & + prname + + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + + ! local parameters + integer :: ier + logical :: if_collective = .true. + + + ! arrays for Moho, 400, 670 + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_moho_top + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_moho_bot + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_400_top + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_400_bot + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_670_top + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_670_bot + + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_moho_top + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_moho_bot + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_400_top + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_400_bot + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_670_top + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_670_bot + + ! processor dependent group names + character(len=64) :: gname_region + + ! process rank + myrank = myrank + + ! first check the number of surface elements are the same for Moho, 400, 670 + if (.not. SUPPRESS_CRUSTAL_MESH .and. HONOR_1D_SPHERICAL_MOHO) then + if (ispec2D_moho_top /= NSPEC2D_MOHO .or. ispec2D_moho_bot /= NSPEC2D_MOHO) & + call exit_mpi(myrank, 'Not the same number of Moho surface elements') + endif + if (ispec2D_400_top /= NSPEC2D_400 .or. ispec2D_400_bot /= NSPEC2D_400) & + call exit_mpi(myrank,'Not the same number of 400 surface elements') + if (ispec2D_670_top /= NSPEC2D_670 .or. ispec2D_670_bot /= NSPEC2D_670) & + call exit_mpi(myrank,'Not the same number of 670 surface elements') + + ! gather the number of surface elements for Moho, 400, 670 + call gather_all_all_singlei(size(ibelm_moho_top), arr_nspec2d_moho_top, NPROCTOT) + call gather_all_all_singlei(size(ibelm_moho_bot), arr_nspec2d_moho_bot, NPROCTOT) + call gather_all_all_singlei(size(ibelm_400_top), arr_nspec2d_400_top, NPROCTOT) + call gather_all_all_singlei(size(ibelm_400_bot), arr_nspec2d_400_bot, NPROCTOT) + call gather_all_all_singlei(size(ibelm_670_top), arr_nspec2d_670_top, NPROCTOT) + call gather_all_all_singlei(size(ibelm_670_bot), arr_nspec2d_670_bot, NPROCTOT) + + call gather_all_all_singlei(NSPEC2D_MOHO, act_arr_nspec2d_moho_top, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_MOHO, act_arr_nspec2d_moho_bot, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_400, act_arr_nspec2d_400_top, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_400, act_arr_nspec2d_400_bot, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_670, act_arr_nspec2d_670_top, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_670, act_arr_nspec2d_670_bot, NPROCTOT) + + ! file name + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'boundary_disc.h5' + ! group name + write(gname_region, "('reg',i1)") iregion_code + + ! create dataset + if (myrank == 0) then + ! create and open boundary.h5 + if (iregion_code == 1) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + + ! create group for iregion_code + call h5_create_group(gname_region) + + ! open group for iregion_code + call h5_open_group(gname_region) + + ! write the number of surface elements for Moho, 400, 670 + call h5_create_dataset_gen_in_group('NSPEC2D_MOHO', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('NSPEC2D_400', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('NSPEC2D_670', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_MOHO_top', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_MOHO_bot', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_400_top', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_400_bot', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_670_top', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_670_bot', (/NPROCTOT/), 1, 1) + + ! create dataset for Moho, 400, 670 + call h5_create_dataset_gen_in_group('ibelm_moho_top', (/sum(arr_nspec2d_moho_top(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_moho_bot', (/sum(arr_nspec2d_moho_bot(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_400_top', (/sum(arr_nspec2d_400_top(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_400_bot', (/sum(arr_nspec2d_400_bot(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_670_top', (/sum(arr_nspec2d_670_top(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_670_bot', (/sum(arr_nspec2d_670_bot(:))/), 1, 1) + call h5_create_dataset_gen_in_group('normal_moho', (/NDIM,NGLLX,NGLLY,sum(arr_nspec2d_moho_top(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_400', (/NDIM,NGLLX,NGLLY,sum(arr_nspec2d_400_top(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_670', (/NDIM,NGLLX,NGLLY,sum(arr_nspec2d_670_top(:))/), 4, CUSTOM_REAL) + + call h5_close_group() + call h5_close_file() + + endif + + ! + ! write datasets by all processors + ! + ! open file and group + call h5_open_file_p_collect(name_database_hdf5) + call h5_open_group(gname_region) + + ! write datasets + call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_MOHO', (/act_arr_nspec2d_moho_top(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_400', (/act_arr_nspec2d_400_top(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_670', (/act_arr_nspec2d_670_top(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_MOHO_top', (/act_arr_nspec2d_moho_top(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_MOHO_bot', (/act_arr_nspec2d_moho_bot(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_400_top', (/act_arr_nspec2d_400_top(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_400_bot', (/act_arr_nspec2d_400_bot(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_670_top', (/act_arr_nspec2d_670_top(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_670_bot', (/act_arr_nspec2d_670_bot(myrank)/), (/myrank/), if_collective) + + call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_top', ibelm_moho_top, (/sum(arr_nspec2d_moho_top(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_bot', ibelm_moho_bot, (/sum(arr_nspec2d_moho_bot(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_400_top', ibelm_400_top, (/sum(arr_nspec2d_400_top(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_400_bot', ibelm_400_bot, (/sum(arr_nspec2d_400_bot(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_670_top', ibelm_670_top, (/sum(arr_nspec2d_670_top(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_670_bot', ibelm_670_bot, (/sum(arr_nspec2d_670_bot(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('normal_moho', normal_moho, (/0,0,0,sum(arr_nspec2d_moho_top(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('normal_400', normal_400, (/0,0,0,sum(arr_nspec2d_400_top(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('normal_670', normal_670, (/0,0,0,sum(arr_nspec2d_670_top(0:myrank-1))/), if_collective) + + ! close group for iregion_code + call h5_close_group() + ! close boundary_disc.h5 + call h5_close_file_p() + +#else + ! no HDF5 compilation + + ! user output + print * + print *, "Error: HDF5 routine save_databases_hdf5() called without HDF5 Support." + print *, "To enable HDF5 support, reconfigure with --with-hdf5 flag." + print * + stop 'Error HDF5 save_databases_hdf5(): called without compilation support' +#endif + +end subroutine save_arrays_boundary_hdf5 + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & + num_interfaces,max_nibool_interfaces, & + my_neighbors,nibool_interfaces, & + ibool_interfaces, & + nspec_inner,nspec_outer, & + num_phase_ispec,phase_ispec_inner, & + num_colors_outer,num_colors_inner, & + num_elem_colors) + use constants + +#ifdef USE_HDF5 + use meshfem_par, only: & + NPROCTOT + use manager_hdf5 +#endif + + implicit none + + integer,intent(in) :: iregion_code + + character(len=MAX_STRING_LEN),intent(in) :: LOCAL_PATH + + ! MPI interfaces + integer,intent(in) :: num_interfaces,max_nibool_interfaces + integer, dimension(num_interfaces),intent(in) :: my_neighbors + integer, dimension(num_interfaces),intent(in) :: nibool_interfaces + integer, dimension(max_nibool_interfaces,num_interfaces),intent(in) :: & + ibool_interfaces + + ! inner/outer elements + integer,intent(in) :: nspec_inner,nspec_outer + integer,intent(in) :: num_phase_ispec + integer,dimension(num_phase_ispec,2),intent(in) :: phase_ispec_inner + + ! mesh coloring + integer,intent(in) :: num_colors_outer,num_colors_inner + integer, dimension(num_colors_outer + num_colors_inner),intent(in) :: & + num_elem_colors + +#ifdef USE_HDF5 + ! local parameters + character(len=64) :: gname_region + integer :: ier + logical :: if_collective = .true. + + ! offset arrays + integer, dimension(0:NPROCTOT-1) :: offset_num_interfaces + integer, dimension(0:NPROCTOT-1) :: offset_max_nibool_interfaces + integer, dimension(0:NPROCTOT-1) :: offset_nspec_inner + integer, dimension(0:NPROCTOT-1) :: offset_nspec_outer + integer, dimension(0:NPROCTOT-1) :: offset_num_phase_ispec + integer, dimension(0:NPROCTOT-1) :: offset_num_colors_outer + integer, dimension(0:NPROCTOT-1) :: offset_num_colors_inner + + myrank = myrank + + ! gather the offsets + call gather_all_all_singlei(num_interfaces, offset_num_interfaces, NPROCTOT) + call gather_all_all_singlei(max_nibool_interfaces, offset_max_nibool_interfaces, NPROCTOT) + call gather_all_all_singlei(nspec_inner, offset_nspec_inner, NPROCTOT) + call gather_all_all_singlei(nspec_outer, offset_nspec_outer, NPROCTOT) + call gather_all_all_singlei(num_phase_ispec, offset_num_phase_ispec, NPROCTOT) + call gather_all_all_singlei(num_colors_outer, offset_num_colors_outer, NPROCTOT) + call gather_all_all_singlei(num_colors_inner, offset_num_colors_inner, NPROCTOT) + + ! file name + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'solver_data_mpi.h5' + ! group name + write(gname_region, "('reg',i1)") iregion_code + + if (myrank == 0) then + ! create and open solver_data_mpi.h5 + if (iregion_code == IREGION_CRUST_MANTLE) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + ! create group for iregion_code + call h5_create_group(gname_region) + ! open group for iregion_code + call h5_open_group(gname_region) + + ! create datasets + call h5_create_dataset_gen_in_group('offset_num_interfaces', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_max_nibool_interfaces', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_nspec_inner', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_nspec_outer', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_num_phase_ispec', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_num_colors_outer', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_num_colors_inner', (/NPROCTOT/), 1, 1) + + if (sum(offset_num_interfaces) > 0) then + call h5_create_dataset_gen_in_group("max_nibool_interfaces", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("my_neighbors", (/sum(offset_num_interfaces(:))/), 1, 1) + call h5_create_dataset_gen_in_group("nibool_interfaces", (/sum(offset_num_interfaces(:))/), 1, 1) + call h5_create_dataset_gen_in_group("ibool_interfaces", (/maxval(offset_max_nibool_interfaces),sum(offset_num_interfaces(:))/), 2, 1) + else + ! dummy + call h5_create_dataset_gen_in_group("max_nibool_interfaces", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("my_neighbors", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("nibool_interfaces", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("ibool_interfaces", (/1,NPROCTOT/), 2, 1) + endif + + ! num_phase_ispec + if (sum(offset_num_phase_ispec) > 0) then + call h5_create_dataset_gen_in_group("phase_ispec_inner", (/sum(offset_num_phase_ispec(:)),2/), 2, 1) + else + ! dummy + call h5_create_dataset_gen_in_group("phase_ispec_inner", (/NPROCTOT,2/), 2, 1) + endif + + ! mesh coloring + if (sum(offset_num_colors_outer) + sum(offset_num_colors_inner) > 0) then + call h5_create_dataset_gen_in_group("num_elem_colors", (/sum(offset_num_colors_outer(:)) + sum(offset_num_colors_inner(:))/), 1, 1) + else + ! dummy + call h5_create_dataset_gen_in_group("num_elem_colors", (/NPROCTOT/), 1, 1) + endif + + ! close group for iregion_code + call h5_close_group() + ! close solver_data_mpi.h5 + call h5_close_file() + + endif + + ! + ! write datasets by all processors + ! + + ! open file and group + call h5_open_file_p_collect(name_database_hdf5) + call h5_open_group(gname_region) + + ! write datasets + call h5_write_dataset_collect_hyperslab_in_group('offset_num_interfaces', (/offset_num_interfaces(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('offset_max_nibool_interfaces', (/offset_max_nibool_interfaces(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('offset_nspec_inner', (/offset_nspec_inner(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('offset_nspec_outer', (/offset_nspec_outer(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_phase_ispec', (/offset_num_phase_ispec(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_outer', (/offset_num_colors_outer(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_inner', (/offset_num_colors_inner(myrank)/), (/myrank/), if_collective) + + if (sum(offset_num_interfaces) > 0) then + call h5_write_dataset_collect_hyperslab_in_group("max_nibool_interfaces", (/max_nibool_interfaces/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("my_neighbors", my_neighbors, (/sum(offset_num_interfaces(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("nibool_interfaces", nibool_interfaces, (/sum(offset_num_interfaces(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("ibool_interfaces", ibool_interfaces, (/0,sum(offset_num_interfaces(0:myrank-1))/), if_collective) + else + ! dummy + call h5_write_dataset_collect_hyperslab_in_group("max_nibool_interfaces", (/max_nibool_interfaces/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("my_neighbors", my_neighbors, (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("nibool_interfaces", nibool_interfaces, (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("ibool_interfaces", ibool_interfaces, (/0,myrank/), if_collective) + endif + + ! num_phase_ispec + if (sum(offset_num_phase_ispec) > 0) then + call h5_write_dataset_collect_hyperslab_in_group("phase_ispec_inner", phase_ispec_inner, (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_collective) + else + ! dummy + call h5_write_dataset_collect_hyperslab_in_group("phase_ispec_inner", phase_ispec_inner, (/myrank,0/), if_collective) + endif + + ! mesh coloring + if (sum(offset_num_colors_outer) + sum(offset_num_colors_inner) > 0) then + call h5_write_dataset_collect_hyperslab_in_group("num_elem_colors", num_elem_colors, (/sum(offset_num_colors_outer(0:myrank-1)) + sum(offset_num_colors_inner(0:myrank-1))/), if_collective) + else + ! dummy + call h5_write_dataset_collect_hyperslab_in_group("num_elem_colors", num_elem_colors, (/myrank/), if_collective) + endif + + ! close group for iregion_code + call h5_close_group() + ! close solver_data_mpi.h5 + call h5_close_file_p() + +#else + ! no HDF5 compilation + print* , "Error: HDF5 routine save_MPI_arrays_hdf5() called without HDF5 Support." + print* , "To enable HDF5 support, reconfigure with --with-hdf5 flag." + stop 'Error HDF5 save_MPI_arrays_hdf5(): called without compilation support' +#endif + + end subroutine save_MPI_arrays_hdf5 + + + subroutine get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & + abs_boundary_ispec,abs_boundary_npoin, & + abs_boundary_ijk,abs_boundary_normal,abs_boundary_jacobian2Dw) + + use constants, only: NDIM,NGLLX,NGLLY,NGLLSQUARE,CUSTOM_REAL,MAX_STRING_LEN,IREGION_CRUST_MANTLE + +#ifdef USE_HDF5 + use constants, only: myrank + use meshfem_par, only: LOCAL_PATH, NPROCTOT + use manager_hdf5 +#endif + + implicit none + + integer,intent(in) :: iregion + + ! absorbing boundary arrays + integer,intent(in) :: num_abs_boundary_faces + integer, dimension(num_abs_boundary_faces), intent(in) :: abs_boundary_ispec + integer, dimension(num_abs_boundary_faces), intent(in) :: abs_boundary_npoin + integer, dimension(3,NGLLSQUARE,num_abs_boundary_faces), intent(in) :: abs_boundary_ijk + real(kind=CUSTOM_REAL), dimension(NDIM,NGLLSQUARE,num_abs_boundary_faces), intent(in) :: abs_boundary_normal + real(kind=CUSTOM_REAL), dimension(NGLLSQUARE,num_abs_boundary_faces), intent(in) :: abs_boundary_jacobian2Dw + +#ifdef USE_HDF5 + ! local parameters + integer, dimension(0:NPROCTOT-1) :: offset_num_abs_boundary_faces + + ! dummy arrays + integer, dimension(1,1), parameter :: i2d_dummy = reshape((/0/),(/1,1/)) + integer, dimension(1,1,1), parameter :: i3d_dummy = reshape((/0/),(/1,1,1/)) + real(kind=CUSTOM_REAL), dimension(1,1), parameter :: r2d_dummy = reshape((/0.0/),(/1,1/)) + real(kind=CUSTOM_REAL), dimension(1,1,1), parameter :: r3d_dummy = reshape((/0.0/),(/1,1,1/)) + + ! variables for HDF5 + character(len=64) :: gname_region + integer :: ier + logical :: if_col = .true. ! collective write + + ! file name + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'stacey.h5' + ! group name + write(gname_region, "('reg',i1)") iregion + + ! get offset arrays + call gather_all_all_singlei(num_abs_boundary_faces, offset_num_abs_boundary_faces, NPROCTOT) + + ! create datasets by myrank=0 + if (myrank == 0) then + ! create and open stacey.h5 + if (iregion == IREGION_CRUST_MANTLE) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + + ! create group for iregion_code + call h5_create_group(gname_region) + ! open group for iregion_code + call h5_open_group(gname_region) + + call h5_create_dataset_gen_in_group("num_abs_boundary_faces", (/NPROCTOT/), 1, 1) + + if (sum(offset_num_abs_boundary_faces) > 0) then + call h5_create_dataset_gen_in_group("abs_boundary_ispec", (/sum(offset_num_abs_boundary_faces(:))/), 1, 1) + call h5_create_dataset_gen_in_group("abs_boundary_npoin", (/sum(offset_num_abs_boundary_faces(:))/), 1, 1) + call h5_create_dataset_gen_in_group("abs_boundary_ijk", (/NDIM,NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("abs_boundary_jacobian2Dw", (/NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 2, CUSTOM_REAL) + + if (iregion == IREGION_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group("abs_boundary_normal", (/NDIM,NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 3, CUSTOM_REAL) + endif + else + ! dummy + call h5_create_dataset_gen_in_group("abs_boundary_ispec", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("abs_boundary_npoin", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("abs_boundary_ijk", (/NDIM,NGLLSQUARE,NPROCTOT/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("abs_boundary_jacobian2Dw", (/NGLLSQUARE,NPROCTOT/), 2, CUSTOM_REAL) + + if (iregion == IREGION_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group("abs_boundary_normal", (/NDIM,NGLLSQUARE,NPROCTOT/), 3, CUSTOM_REAL) + endif + endif ! iregion == IREGION_CRUST_MANTEL + + ! close group for iregion_code + call h5_close_group() + ! close stacey.h5 + call h5_close_file() + endif ! myrank == 0 + + ! write datasets by all processors + call h5_open_file_p_collect(name_database_hdf5) + call h5_open_group(gname_region) + + ! write datasets + call h5_write_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", (/offset_num_abs_boundary_faces(myrank)/), (/myrank/), .true.) + + if (sum(offset_num_abs_boundary_faces) > 0) then + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + if (iregion == IREGION_CRUST_MANTLE) then + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + endif + else + ! dummy + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ispec", (/0/), (/myrank/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_npoin", (/0/), (/myrank/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ijk", i3d_dummy, (/0,0,myrank/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", r2d_dummy, (/0,myrank/), if_col) + if (iregion == IREGION_CRUST_MANTLE) then + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_normal", r3d_dummy, (/0,0,myrank/), if_col) + endif + endif + + ! close group for iregion_code + call h5_close_group() + ! close stacey.h5 + call h5_close_file_p() + +#else + ! no HDF5 compilation + print* , "Error: HDF5 routine get_absorb_stacey_boundary_hdf5() called without HDF5 Support." + print* , "To enable HDF5 support, reconfigure with --with-hdf5 flag." + stop 'Error HDF5 get_absorb_stacey_boundary_hdf5(): called without compilation support' +#endif + + + + + + + end subroutine get_absorb_stacey_boundary_hdf5 diff --git a/src/meshfem3D/save_model_meshfiles_hdf5.F90 b/src/meshfem3D/save_model_meshfiles_hdf5.F90 new file mode 100644 index 000000000..2cef2f83b --- /dev/null +++ b/src/meshfem3D/save_model_meshfiles_hdf5.F90 @@ -0,0 +1,282 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + + subroutine save_model_meshfiles_hdf5() + +! outputs model files in binary format + + use constants + use shared_parameters, only: R_PLANET,RHOAV,LOCAL_PATH + + use meshfem_par, only: nspec,iregion_code,NPROCTOT + + use meshfem_models_par, only: & + TRANSVERSE_ISOTROPY,ATTENUATION,ATTENUATION_3D,ATTENUATION_1D_WITH_3D_STORAGE, & + HETEROGEN_3D_MANTLE,ANISOTROPIC_3D_MANTLE + + use regions_mesh_par2, only: & + rhostore,kappavstore,kappahstore,muvstore,muhstore,eta_anisostore, & + Qmu_store,Gc_prime_store,Gs_prime_store,mu0store, & + prname + + use model_heterogen_mantle_par + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + ! local parameters + integer :: i,j,k,ispec,ier + real(kind=CUSTOM_REAL) :: scaleval1,scaleval2,scaleval,scale_GPa + real(kind=CUSTOM_REAL),dimension(:,:,:,:),allocatable :: temp_store + +#ifdef USE_HDF5 + + ! dset_name and group_name + character(len=64) :: dset_name, gname_region + + ! offset for nspec + integer, dimension(0:NPROCTOT-1) :: offset_nelems + + ! if collective write + logical :: if_collective = .true. + + ! gather nspec + call gather_all_all_singlei(nspec, offset_nelems, NPROCTOT) + + ! file name + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'meshfile.h5' + ! group name + write(gname_region, "('reg',i1)") iregion_code + + if (myrank == 0) then + ! create the file + if (iregion_code == 1) then + call h5_create_file(name_database_hdf5) + endif + + ! open the file + call h5_open_file(name_database_hdf5) + ! create the group + call h5_create_group(gname_region) + ! open the group + call h5_open_group(gname_region) + + if (TRANSVERSE_ISOTROPY) then + ! vpv + dset_name = 'vpv' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! vph + dset_name = 'vph' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! vsv + dset_name = 'vsv' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! vsh + dset_name = 'vsh' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! rho + dset_name = 'rho' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! eta + dset_name = 'eta' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + + else + + ! isotropic model + ! vp + dset_name = 'vp' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! vs + dset_name = 'vs' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! rho + dset_name = 'rho' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + + endif ! TRANSVERSE_ISOTROPY + + ! anisotropic values + if (ANISOTROPIC_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then + ! Gc_prime + dset_name = 'Gc_prime' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! Gs_prime + dset_name = 'Gs_prime' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! mu0 + dset_name = 'mu0' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + + if (ATTENUATION) then + ! Qmu + dset_name = 'Qmu' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + + if (HETEROGEN_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then + ! dvp + dset_name = 'dvp' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + + ! close the group + call h5_close_group() + ! close the file + call h5_close_file() + + endif ! myrank == 0 + + ! + ! write the data + ! + ! open the file + call h5_open_file_p(name_database_hdf5) + call h5_open_group(gname_region) + + ! scaling factors to re-dimensionalize units + scaleval1 = real(sqrt(PI*GRAV*RHOAV)*(R_PLANET/1000.0d0),kind=CUSTOM_REAL) + scaleval2 = real(RHOAV/1000.0d0,kind=CUSTOM_REAL) + + ! uses temporary array + allocate(temp_store(NGLLX,NGLLY,NGLLZ,nspec),stat=ier) + if (ier /= 0) stop 'Error allocating temp_store array' + temp_store(:,:,:,:) = 0._CUSTOM_REAL + + if (TRANSVERSE_ISOTROPY) then + + ! vpv + temp_store(:,:,:,:) = sqrt((kappavstore(:,:,:,:) + 4.0_CUSTOM_REAL * muvstore(:,:,:,:)/3.0_CUSTOM_REAL)/rhostore(:,:,:,:)) & + * scaleval1 + dset_name = 'vpv' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + ! vph + temp_store(:,:,:,:) = sqrt((kappahstore(:,:,:,:) + 4.0_CUSTOM_REAL * muhstore(:,:,:,:)/3.0_CUSTOM_REAL)/rhostore(:,:,:,:)) & + * scaleval1 + dset_name = 'vph' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + ! vsv + temp_store(:,:,:,:) = sqrt( muvstore(:,:,:,:)/rhostore(:,:,:,:) )*scaleval1 + dset_name = 'vsv' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + ! vsh + temp_store(:,:,:,:) = sqrt( muhstore(:,:,:,:)/rhostore(:,:,:,:) )*scaleval1 + dset_name = 'vsh' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + ! rho + temp_store(:,:,:,:) = rhostore(:,:,:,:) * scaleval2 + dset_name = 'rho' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + ! eta + dset_name = 'eta' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + + else + ! isotropic model + ! vp + temp_store(:,:,:,:) = sqrt((kappavstore(:,:,:,:) + 4.0_CUSTOM_REAL * muvstore(:,:,:,:)/3.0_CUSTOM_REAL)/rhostore(:,:,:,:)) & + * scaleval1 + dset_name = 'vp' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + ! vs + temp_store(:,:,:,:) = sqrt( muvstore(:,:,:,:)/rhostore(:,:,:,:) )*scaleval1 + dset_name = 'vs' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + ! rho + temp_store(:,:,:,:) = rhostore(:,:,:,:) * scaleval2 + dset_name = 'rho' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + + endif ! TRANSVERSE_ISOTROPY + + ! anisotropic values + if (ANISOTROPIC_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then + ! the scale of GPa--[g/cm^3][(km/s)^2] + scaleval = real(sqrt(PI*GRAV*RHOAV),kind=CUSTOM_REAL) + scale_GPa = real((RHOAV/1000.d0)*((R_PLANET*scaleval/1000.d0)**2),kind=CUSTOM_REAL) + + ! Gc_prime + dset_name = 'Gc_prime' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, Gc_prime_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + ! Gs_prime + dset_name = 'Gs_prime' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, Gs_prime_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + ! mu0 + temp_store(:,:,:,:) = mu0store(:,:,:,:) * scale_GPa + dset_name = 'mu0' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + + endif + + if (ATTENUATION) then + if (ATTENUATION_3D .or. ATTENUATION_1D_WITH_3D_STORAGE) then + temp_store(:,:,:,:) = Qmu_store(:,:,:,:) + else + do ispec = 1,nspec + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + temp_store(i,j,k,ispec) = Qmu_store(1,1,1,ispec) + end do + end do + end do + end do + endif + + ! Qmu + dset_name = 'Qmu' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + + endif ! ATTENUATION + + deallocate(temp_store) + + if (HETEROGEN_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then + ! dvp + dset_name = 'dvp' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, dvpstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + endif + + ! close the group + call h5_close_group() + ! close the file + call h5_close_file() + +#else + + print *, 'Error: HDF5 support not enabled in this build' + stop + +#endif + + +end subroutine save_model_meshfiles_hdf5 \ No newline at end of file diff --git a/src/shared/broadcast_computed_parameters.f90 b/src/shared/broadcast_computed_parameters.f90 index a62f231d4..9c8c1cfd6 100644 --- a/src/shared/broadcast_computed_parameters.f90 +++ b/src/shared/broadcast_computed_parameters.f90 @@ -34,10 +34,10 @@ subroutine broadcast_computed_parameters() ! local parameters ! broadcast parameter arrays - integer, parameter :: nparam_i = 50 + integer, parameter :: nparam_i = 51 integer, dimension(nparam_i) :: bcast_integer - integer, parameter :: nparam_l = 74 + integer, parameter :: nparam_l = 76 logical, dimension(nparam_l) :: bcast_logical integer, parameter :: nparam_dp = 42 @@ -77,7 +77,8 @@ subroutine broadcast_computed_parameters() GPU_RUNTIME,NUMBER_OF_SIMULTANEOUS_RUNS, & MODEL_GLL_TYPE,USER_NSTEP, & NSTEP_STEADY_STATE,NTSTEP_BETWEEN_OUTPUT_SAMPLE, & - POISSON_SOLVER /) + POISSON_SOLVER, & + HDF5_IO_NODES /) bcast_logical = (/ & TRANSVERSE_ISOTROPY,ANISOTROPIC_3D_MANTLE,ANISOTROPIC_INNER_CORE, & @@ -111,7 +112,8 @@ subroutine broadcast_computed_parameters() OUTPUT_SEISMOS_3D_ARRAY, & REGIONAL_MESH_CUTOFF,REGIONAL_MESH_ADD_2ND_DOUBLING, & EMC_MODEL, & - FULL_GRAVITY /) + FULL_GRAVITY, & + HDF5_ENABLED, HDF5_FOR_MOVIES /) bcast_double_precision = (/ & DT, & @@ -274,6 +276,7 @@ subroutine broadcast_computed_parameters() NSTEP_STEADY_STATE = bcast_integer(48) NTSTEP_BETWEEN_OUTPUT_SAMPLE = bcast_integer(49) POISSON_SOLVER = bcast_integer(50) + HDF5_IO_NODES = bcast_integer(51) ! logicals TRANSVERSE_ISOTROPY = bcast_logical(1) @@ -350,6 +353,8 @@ subroutine broadcast_computed_parameters() REGIONAL_MESH_ADD_2ND_DOUBLING = bcast_logical(72) EMC_MODEL = bcast_logical(73) FULL_GRAVITY = bcast_logical(74) + HDF5_ENABLED = bcast_logical(75) + HDF5_FOR_MOVIES = bcast_logical(76) ! double precisions DT = bcast_double_precision(1) diff --git a/src/shared/hdf5_manager.F90 b/src/shared/hdf5_manager.F90 new file mode 100644 index 000000000..d53259f45 --- /dev/null +++ b/src/shared/hdf5_manager.F90 @@ -0,0 +1,6065 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +!------------------------------------------------------------------------------- +!> Tools for parallel HDF5 support, +!> contains wrapper subroutines for common hdf5 calls +! +! The idea for these HDF5 tools is to convert all database files to a HDF5-format +! for better scalability to very large simulations. +! It is complementary to ADIOS2 support in case those libraries are not installed/wanted. +! These HDF5 files should avoid bottlenecks on HPC clusters due to the possible higher number of file outputs +! in our simulations (database files, seismograms, visualization snapshots, etc.). + +! note: HDF5 library calls use a format like "h5**do_something***_f()" +! our own wrapper functions in this module will rather use something like "h5_***do_something_***()", +! and higher-level routines called in other Fortran routines like "***do_something_h5***()", +! to better distinguish between library functions and wrappers. +! +! version info: +! - initial version, April 2023: +! Masaru Nagaso (main developer, all initial hdf5 functionality) +! Daniel Peter (just adding Masaru-san's original implementation, sometimes renaming things to more specfem-coherent ways) +! +!------------------------------------------------------------------------------- + +! +! parallel HDF5 file I/O routines +! + +module manager_hdf5 + +! class-like module for HDF5 routines + + use constants, only: CUSTOM_REAL, MAX_STRING_LEN + +#if defined(USE_HDF5) + use hdf5 +#endif + + implicit none + + private + + ! public routines + ! also to act as empty stubs (for missing HDF5 compilation support) + public :: write_attenuation_file_hdf5, read_attenuation_file_hdf5 + public :: write_checkmesh_data_hdf5, write_checkmesh_xdmf_hdf5 + + ! functional interface + public :: h5_initialize + +#if defined(USE_HDF5) + ! only w/ HDF5 compilation + + public :: & + h5_finalize + + public :: & + h5_create_file, & + h5_create_or_open_file, & + h5_open_file, & + h5_close_file, & + h5_create_group, & + h5_open_group, & + h5_close_group, & + h5_create_subgroup, & + h5_open_subgroup, & + h5_open_or_create_group, & + h5_close_subgroup + + public :: & + h5_open_dataset, & + h5_open_dataset2, & + h5_close_dataset + + public :: & + h5_add_attribute_i, & + h5_set_mpi_info, & + h5_set_buffer_size, & + h5_set_sieve_buffer_size, & + h5_set_group_name, & + h5_gather_dsetsize + + public :: & + h5_create_file_p, & + h5_create_file_p_collect, & + h5_open_file_p, & + h5_open_file_p_collect, & + h5_close_file_p, & + h5_create_file_prop_list, & + h5_close_prop_list, & + h5_close_prop_list_nocheck, & + h5_open_group_prop_list, & + h5_create_group_prop_list, & + h5_close_group_prop_list, & + h5_create_dataset_prop_list, & + h5_create_dataset_gen, & + h5_create_dataset_gen_in_group, & + h5_check_dataset_exists, & + h5_create_group_p, & + h5_open_group_p + + public :: & + h5_read_attribute_p, & + h5_read_dataset_p, & + h5_read_dataset_p_scalar, & + h5_read_dataset_scalar_collect_hyperslab, & + h5_read_dataset_scalar_collect_hyperslab_in_group, & + h5_read_dataset_collect_hyperslab_in_group, & + h5_read_dataset_collect_hyperslab + + public :: & + i2c + + public :: & + h5_write_dataset, & + h5_write_dataset_no_group, & + h5_write_dataset_p, & + h5_write_dataset_p_a, & + h5_write_dataset_1d_to_2d_r_collect_hyperslab, & + h5_write_dataset_2d_to_3d_r_collect_hyperslab, & + h5_write_dataset_collect_hyperslab_in_group, & + h5_write_dataset_collect_hyperslab + + ! private routines + + private :: & + h5_check_arr_dim, & + h5_check_collective, & + h5_check_group + + private :: & + create_dataset_collect, & + bool_array2integer, & + int_array2bool + + ! generic interface to read dataset + interface h5_read_dataset_p + module procedure h5_read_dataset_p_1d_l ! logical + module procedure h5_read_dataset_p_1d_i ! integer + module procedure h5_read_dataset_p_2d_i + module procedure h5_read_dataset_p_3d_i + module procedure h5_read_dataset_p_4d_i + module procedure h5_read_dataset_p_1d_r ! real + module procedure h5_read_dataset_p_2d_r + module procedure h5_read_dataset_p_3d_r + module procedure h5_read_dataset_p_4d_r + module procedure h5_read_dataset_p_5d_r + module procedure h5_read_dataset_p_2d_d ! double + module procedure h5_read_dataset_p_2d_c ! char + end interface h5_read_dataset_p + + ! generic interface to read scalar dataset + interface h5_read_dataset_p_scalar + module procedure h5_read_dataset_p_scalar_i + module procedure h5_read_dataset_p_scalar_r + end interface h5_read_dataset_p_scalar + + ! generic interface to read dataset in collective mode + interface h5_read_dataset_scalar_collect_hyperslab + module procedure h5_read_dataset_scalar_i_collect_hyperslab + module procedure h5_read_dataset_scalar_r_collect_hyperslab + end interface h5_read_dataset_scalar_collect_hyperslab + + interface h5_read_dataset_scalar_collect_hyperslab_in_group + module procedure h5_read_dataset_scalar_i_collect_hyperslab_in_group + end interface h5_read_dataset_scalar_collect_hyperslab_in_group + + ! generic interface to read dataset in collective mode + interface h5_read_dataset_collect_hyperslab_in_group + module procedure h5_read_dataset_1d_l_collect_hyperslab_in_group ! logical + module procedure h5_read_dataset_1d_i_collect_hyperslab_in_group ! integer + module procedure h5_read_dataset_1d_r_collect_hyperslab_in_group + module procedure h5_read_dataset_1d_d_collect_hyperslab_in_group ! double + module procedure h5_read_dataset_2d_i_collect_hyperslab_in_group + module procedure h5_read_dataset_2d_r_collect_hyperslab_in_group + module procedure h5_read_dataset_3d_i_collect_hyperslab_in_group + module procedure h5_read_dataset_3d_r_collect_hyperslab_in_group + module procedure h5_read_dataset_4d_i_collect_hyperslab_in_group + module procedure h5_read_dataset_4d_r_collect_hyperslab_in_group + module procedure h5_read_dataset_5d_r_collect_hyperslab_in_group + end interface h5_read_dataset_collect_hyperslab_in_group + + ! generic interface to read dataset in collective mode + interface h5_read_dataset_collect_hyperslab + module procedure h5_read_dataset_1d_l_collect_hyperslab ! logical + module procedure h5_read_dataset_1d_i_collect_hyperslab ! integer + module procedure h5_read_dataset_2d_i_collect_hyperslab + module procedure h5_read_dataset_3d_i_collect_hyperslab + module procedure h5_read_dataset_4d_i_collect_hyperslab + module procedure h5_read_dataset_1d_r_collect_hyperslab ! real + module procedure h5_read_dataset_2d_r_collect_hyperslab + module procedure h5_read_dataset_3d_r_collect_hyperslab + module procedure h5_read_dataset_4d_r_collect_hyperslab + module procedure h5_read_dataset_5d_r_collect_hyperslab + module procedure h5_read_dataset_2d_d_collect_hyperslab ! double + end interface h5_read_dataset_collect_hyperslab + + ! generic interface to write dataset + interface h5_write_dataset + module procedure h5_write_dataset_1d_i ! integer + module procedure h5_write_dataset_2d_i + module procedure h5_write_dataset_1d_d ! double + module procedure h5_write_dataset_2d_d + module procedure h5_write_dataset_1d_c ! char + module procedure h5_write_dataset_2d_c + module procedure h5_write_dataset_2d_r ! real + module procedure h5_write_dataset_4d_r + end interface h5_write_dataset + + interface h5_write_dataset_no_group + module procedure h5_write_dataset_1d_i_no_group + module procedure h5_write_dataset_1d_d_no_group + module procedure h5_write_dataset_1d_c_no_group + module procedure h5_write_dataset_2d_r_no_group + end interface h5_write_dataset_no_group + + ! generic interface to write dataset + interface h5_write_dataset_p + module procedure h5_write_dataset_p_1d_l ! logical + module procedure h5_write_dataset_p_1d_i ! integer + module procedure h5_write_dataset_p_2d_i + module procedure h5_write_dataset_p_3d_i + module procedure h5_write_dataset_p_4d_i + module procedure h5_write_dataset_p_1d_r ! real + module procedure h5_write_dataset_p_2d_r + module procedure h5_write_dataset_p_3d_r + module procedure h5_write_dataset_p_4d_r + module procedure h5_write_dataset_p_5d_r + module procedure h5_write_dataset_p_2d_d ! double + end interface h5_write_dataset_p + + interface h5_write_dataset_p_a + module procedure h5_write_dataset_p_1d_ia + module procedure h5_write_dataset_p_2d_ia + end interface h5_write_dataset_p_a + + ! generic interface to write dataset in collective mode + interface h5_write_dataset_collect_hyperslab_in_group + module procedure h5_write_dataset_1d_l_collect_hyperslab_in_group ! logical + module procedure h5_write_dataset_1d_i_collect_hyperslab_in_group + module procedure h5_write_dataset_2d_i_collect_hyperslab_in_group + module procedure h5_write_dataset_3d_i_collect_hyperslab_in_group + module procedure h5_write_dataset_4d_i_collect_hyperslab_in_group + module procedure h5_write_dataset_1d_r_collect_hyperslab_in_group ! real + module procedure h5_write_dataset_2d_r_collect_hyperslab_in_group + module procedure h5_write_dataset_3d_r_collect_hyperslab_in_group + module procedure h5_write_dataset_4d_r_collect_hyperslab_in_group + module procedure h5_write_dataset_5d_r_collect_hyperslab_in_group + module procedure h5_write_dataset_1d_d_collect_hyperslab_in_group ! double + module procedure h5_write_dataset_2d_d_collect_hyperslab_in_group ! double + end interface h5_write_dataset_collect_hyperslab_in_group + + ! generic interface to write dataset in collective mode + interface h5_write_dataset_collect_hyperslab + module procedure h5_write_dataset_1d_l_collect_hyperslab ! logical + module procedure h5_write_dataset_1d_i_collect_hyperslab ! integer + module procedure h5_write_dataset_2d_i_collect_hyperslab + module procedure h5_write_dataset_3d_i_collect_hyperslab + module procedure h5_write_dataset_4d_i_collect_hyperslab + module procedure h5_write_dataset_1d_r_collect_hyperslab ! real + module procedure h5_write_dataset_2d_r_collect_hyperslab + module procedure h5_write_dataset_3d_r_collect_hyperslab + module procedure h5_write_dataset_4d_r_collect_hyperslab + module procedure h5_write_dataset_5d_r_collect_hyperslab + module procedure h5_write_dataset_2d_d_collect_hyperslab ! double + end interface h5_write_dataset_collect_hyperslab + + ! object-oriented interface + ! (Fortran 2003 standard style) + ! + ! usage example: + ! subroutine ** + ! .. + ! use manager_hdf5, only: h5io + ! type(h5io) :: h5 + ! .. + ! h5 = h5io() ! calls the object constructor, i.e. h5io_constructor() + ! call h5%open(fname) ! object function call + ! ! or in a single call: + ! ! h5 = h5io(fname) + ! call h5%write("x",store_val_x_all) + ! call h5%close() + ! .. + ! end subroutine ! h5 goes out of scope, will call the object destructor, i.e., h5io_destructor() + ! + type, public :: h5io + private + logical :: is_initialized = .false. + integer(HID_T) :: f_id = -1, g_id = -1, d_id = -1 + character(len=MAX_STRING_LEN) :: filename = "" + contains + procedure :: open => h5io_open_file + procedure :: close => h5io_close_file + generic :: write => h5io_write_dataset_i, h5io_write_dataset_r + procedure, private :: h5io_write_dataset_i + procedure, private :: h5io_write_dataset_r + final :: h5io_destructor + end type h5io + ! object constructor + ! called by: h5 = h5io() + interface h5io + module procedure :: h5io_constructor + module procedure :: h5io_constructor_from_file + end interface h5io + + ! module parameters + ! ids + integer(HID_T) :: file_id, group_id, parent_group_id, dataset_id + integer(HID_T) :: mem_dspace_id, file_dspace_id ! for collective IO + + ! parallel process + integer(HID_T) :: plist_id, fplist_id, gplist_id + + ! string array + ! initial string length for network/station names (to be determined later) + integer(SIZE_T) :: str_len = 16 + integer(HID_T) :: str_type + ! maximum string length for passing character array undef_mat_prop + integer(SIZE_T), parameter :: str_len_max = MAX_STRING_LEN + integer(HID_T) :: str_type_max + + type(c_ptr) :: f_ptr + + ! MPI info + integer :: this_rank, this_info, this_comm, total_proc + + ! io unit + integer, parameter :: xdmf_mesh = 190 + + ! mesh nodes + public :: xdmf_mesh_nnodes + integer :: xdmf_mesh_nnodes + + ! function call errors + integer :: error + + ! class-wide private variables + character(len=256) :: file_path + character(len=256) :: store_group_name ! groupname for debug + + public :: name_database_hdf5 + + ! store HDF5 output file name + character(len=MAX_STRING_LEN) :: name_database_hdf5 + +#endif + +contains + +!------------------------------------------------------------------------------- +! +! public HDF5 wrapper routines (also available without hdf5 compilation support) +! +!------------------------------------------------------------------------------- + + subroutine h5_initialize() + +#if defined(USE_HDF5) + use constants, only: MAX_LENGTH_NETWORK_NAME, MAX_LENGTH_STATION_NAME +#endif + + implicit none + +#if defined(USE_HDF5) + ! initialize Fortran interface + call h5open_f(error) + call check_error() + + ! prepare string array type + if (MAX_LENGTH_STATION_NAME >= MAX_LENGTH_NETWORK_NAME) then + str_len = MAX_LENGTH_STATION_NAME + else + str_len = MAX_LENGTH_NETWORK_NAME + endif + + call h5tcopy_f(H5T_Fortran_S1, str_type, error) + call check_error() + + call h5tset_size_f(str_type, str_len, error) + call check_error() + + ! string array type for undef_mat_prop array + call h5tcopy_f(H5T_Fortran_S1, str_type_max, error) + call check_error() + + call h5tset_size_f(str_type_max, str_len_max, error) + call check_error() +#else + ! no HDF5 compilation support + + ! compilation without HDF5 support + print * + print *, "Error: HDF5 routine h5_initialize() called without HDF5 Support." + print *, "To enable HDF5 support, reconfigure with --with-hdf5 flag." + print * + + ! safety stop + stop 'Error HDF5 manager: h5_initialize() intitialization called without compilation support' + +#endif + + end subroutine h5_initialize + +!------------------------------------------------------------------------------- +! +! higher level utilities +! +!------------------------------------------------------------------------------- + + subroutine write_attenuation_file_hdf5(factor_common, scale_factor, factor_common_kappa, scale_factor_kappa) + +#if defined(USE_HDF5) + use shared_parameters, only: NPROC, LOCAL_PATH + use constants, only: myrank,N_SLS,NGLLX,NGLLY,NGLLZ +#endif + + implicit none + + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:,:) :: factor_common + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:) :: scale_factor + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:,:) :: factor_common_kappa + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:) :: scale_factor_kappa + +#if defined(USE_HDF5) + integer, dimension(4) :: dims + integer :: nspec + + ! offset arrays + integer, dimension(0:NPROC-1) :: offset_nspec + + ! hdf5 valiables + character(len=64) :: filename, dset_name, tempstr + + dims = shape(scale_factor) + nspec = dims(4) + + ! prepare offset arrays + call gather_all_all_singlei(nspec,offset_nspec,NPROC) ! n spec in each proc + + tempstr = "/external_mesh.h5" + filename = LOCAL_PATH(1:len_trim(LOCAL_PATH))//trim(tempstr) + + ! initialize h5 object + call h5_initialize() + + ! prepare dataset + if (myrank == 0) then + call h5_open_file(filename) + dset_name = "scale_factor" + call h5_create_dataset_gen(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec)/), 4, CUSTOM_REAL) + dset_name = "scale_factor_kappa" + call h5_create_dataset_gen(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec)/), 4, CUSTOM_REAL) + dset_name = "factor_common" + call h5_create_dataset_gen(dset_name, (/N_SLS,NGLLX,NGLLY,NGLLZ,sum(offset_nspec)/), 5, CUSTOM_REAL) + dset_name = "factor_common_kappa" + call h5_create_dataset_gen(dset_name, (/N_SLS,NGLLX,NGLLY,NGLLZ,sum(offset_nspec)/), 5, CUSTOM_REAL) + call h5_close_file() + endif + + call synchronize_all() + + ! write + call h5_open_file_p_collect(filename) + dset_name = "scale_factor" + call h5_write_dataset_4d_r_collect_hyperslab(dset_name, & + scale_factor,(/0,0,0,sum(offset_nspec(0:myrank-1))/),.true.) + dset_name = "scale_factor_kappa" + call h5_write_dataset_4d_r_collect_hyperslab(dset_name, & + scale_factor_kappa,(/0,0,0,sum(offset_nspec(0:myrank-1))/),.true.) + dset_name = "factor_common" + call h5_write_dataset_5d_r_collect_hyperslab(dset_name, & + factor_common,(/0,0,0,0,sum(offset_nspec(0:myrank-1))/),.true.) + dset_name = "factor_common_kappa" + call h5_write_dataset_5d_r_collect_hyperslab(dset_name, & + factor_common_kappa,(/0,0,0,0,sum(offset_nspec(0:myrank-1))/),.true.) + + call h5_close_file_p() + call h5_finalize() +#else + ! no compilation support + ! to avoid compiler warnings + real(kind=CUSTOM_REAL) :: dummy + + dummy = factor_common(1,1,1,1,1) + dummy = factor_common_kappa(1,1,1,1,1) + dummy = scale_factor(1,1,1,1) + dummy = scale_factor_kappa(1,1,1,1) + + ! safety stop + stop 'Error HDF5 manager: write_attenuation_file_hdf5() called without compilation support' +#endif + + end subroutine write_attenuation_file_hdf5 + +! +!------------------------------------------------------------------------------- +! + + subroutine read_attenuation_file_hdf5(factor_common, scale_factor, factor_common_kappa, scale_factor_kappa) + +#if defined(USE_HDF5) + use shared_parameters, only: NPROC, LOCAL_PATH + use constants, only: myrank +#endif + + implicit none + + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:,:) :: factor_common + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:) :: scale_factor + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:,:) :: factor_common_kappa + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:) :: scale_factor_kappa + +#if defined(USE_HDF5) + ! offset array + integer, dimension(0:NPROC-1) :: offset_nspec + + ! hdf5 valiables + character(len=64) :: fname, tempstr + + tempstr = "/external_mesh.h5" + fname = LOCAL_PATH(1:len_trim(LOCAL_PATH))//trim(tempstr) + + ! initialize h5 object + call h5_initialize() + + ! open file + call h5_open_file_p_collect(fname) + ! read offset array + call h5_read_dataset_1d_i_collect_hyperslab("offset_nspec",offset_nspec, (/0/), .true.) + + call h5_read_dataset_4d_r_collect_hyperslab("scale_factor", scale_factor, & + (/0,0,0,sum(offset_nspec(0:myrank-1))/), .true.) + call h5_read_dataset_4d_r_collect_hyperslab("scale_factor_kappa", scale_factor_kappa, & + (/0,0,0,sum(offset_nspec(0:myrank-1))/), .true.) + call h5_read_dataset_5d_r_collect_hyperslab("factor_common", factor_common, & + (/0,0,0,0,sum(offset_nspec(0:myrank-1))/), .true.) + call h5_read_dataset_5d_r_collect_hyperslab("factor_common_kappa", factor_common_kappa, & + (/0,0,0,0,sum(offset_nspec(0:myrank-1))/), .true.) + + call h5_close_file_p() + call h5_finalize() +#else + ! no compilation support + ! to avoid compiler warnings + real(kind=CUSTOM_REAL) :: dummy + + dummy = factor_common(1,1,1,1,1) + dummy = factor_common_kappa(1,1,1,1,1) + dummy = scale_factor(1,1,1,1) + dummy = scale_factor_kappa(1,1,1,1) + + ! safety stop + stop 'Error HDF5 manager: read_attenuation_file_hdf5() called without compilation support' +#endif + + end subroutine read_attenuation_file_hdf5 + +! +!------------------------------------------------------------------------------- +! + + subroutine write_checkmesh_data_hdf5(dset_name,dump_array) + +#if defined(USE_HDF5) + use shared_parameters, only: LOCAL_PATH, NPROC + use constants, only: myrank +#endif + + implicit none + + real(kind=CUSTOM_REAL),dimension(:), intent(in) :: dump_array + character(len=MAX_STRING_LEN), intent(in) :: dset_name + +#if defined(USE_HDF5) + character(len=MAX_STRING_LEN) :: filename + integer, dimension(0:NPROC-1) :: offset + + ! MPI variables + integer :: info, comm + + ! hdf5 valiables + character(len=64) :: tempstr + + ! flag if dataset exists + logical :: exists = .false. + + ! saves mesh file external_mesh.h5 + tempstr = "/external_mesh.h5" + filename = LOCAL_PATH(1:len_trim(LOCAL_PATH))//trim(tempstr) + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize h5 object + call h5_initialize() + + call h5_set_mpi_info(comm, info, myrank, NPROC) + + ! get offset info + call gather_all_all_singlei(size(dump_array), offset, NPROC) + + ! make dataset + if (myrank == 0) then + call h5_open_file(filename) + ! check if dataset exists + call h5_check_dataset_exists(dset_name, exists) + if (.not. exists) then + call h5_create_dataset_gen(dset_name, (/sum(offset(:))/), 1, CUSTOM_REAL) + endif + call h5_close_file() + endif + call synchronize_all() + + ! open file + call h5_open_file_p_collect(filename) + call h5_write_dataset_1d_r_collect_hyperslab(dset_name, dump_array, (/sum(offset(0:myrank-1))/),.true.) + call h5_close_file_p() + + call h5_finalize() +#else + ! no compilation support + ! to avoid compiler warnings + real(kind=CUSTOM_REAL) :: dummy + character(len=1) :: c_dummy + + dummy = dump_array(1) + c_dummy = dset_name(1:1) + + ! safety stop + stop 'Error HDF5 manager: write_checkmesh_data_hdf5() called without compilation support' +#endif + + end subroutine write_checkmesh_data_hdf5 + +! +!------------------------------------------------------------------------------- +! + + subroutine write_checkmesh_xdmf_hdf5(NSPEC_AB) + +#if defined(USE_HDF5) + use constants, only: myrank + use shared_parameters +#endif + implicit none + + integer, intent(in) :: NSPEC_AB + +#if defined(USE_HDF5) + character(len=MAX_STRING_LEN) :: fname_xdmf_checkmesh,fname_h5_database_xdmf,fname_h5_extmesh_xdmf + integer, dimension(0:NPROC-1) :: nelms, nnodes + character(len=20) :: type_str,nelm_str,nnode_str + + ! gather number of elements in each proc + call gather_all_singlei(NSPEC_AB, nelms, NPROC) + + ! count and gather the number of control nodes in each proc + call gather_all_singlei(xdmf_mesh_nnodes, nnodes, NPROC) + + if (myrank == 0) then + ! writeout xdmf file for surface movie + fname_xdmf_checkmesh = trim(LOCAL_PATH)//"/checkmesh.xmf" + fname_h5_database_xdmf = "./Database.h5" ! relative to checkmesh.xmf file + fname_h5_extmesh_xdmf = "./external_mesh.h5" + + open(unit=xdmf_mesh, file=trim(fname_xdmf_checkmesh), recl=512) + + ! definition of topology and geometry + ! refer only control nodes (8 or 27) as a coarse output + ! data array need to be extracted from full data array on GLL points + write(xdmf_mesh,'(a)') '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + !write(xdmf_mesh,*) '' + + nelm_str = i2c(sum(nelms(:))) + nnode_str = i2c(sum(nnodes(:))) + + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) ' '//trim(fname_h5_database_xdmf)//':/elm_conn_xdmf' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) ' '//trim(fname_h5_database_xdmf)//':/nodes_coords' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + + type_str = "res_Courant_number" + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) ' '//trim(fname_h5_extmesh_xdmf)//':/'//trim(type_str) + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + + type_str = "res_minimum_period" + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) ' '//trim(fname_h5_extmesh_xdmf)//':/'//trim(type_str) + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + + !write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + + close(xdmf_mesh) + endif + +#else + ! no compilation support + ! to avoid compiler warnings + integer :: dummy + + dummy = NSPEC_AB + + ! safety stop + stop 'Error HDF5 manager: write_checkmesh_xdmf_hdf5() called without compilation support' +#endif + + end subroutine write_checkmesh_xdmf_hdf5 + + +!------------------------------------------------------------------------------- +! +! HDF5 wrapper routines (only available with HDF5 compilation support) +! +!------------------------------------------------------------------------------- + +#if defined(USE_HDF5) +! only available with HDF5 compilation support + + + function i2c(k) result(str) + ! "Convert an integer to string." + implicit none + integer, intent(in) :: k + character(len=20) str + write (str, "(i20)") k + str = adjustl(str) + end function i2c + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_finalize() + implicit none + ! close Fortran interface + call h5close_f(error) + call check_error() + end subroutine h5_finalize + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_check_collective(dataset_name) + implicit none + character(len=*), intent(in) :: dataset_name + integer :: io_mode + call h5pget_mpio_actual_io_mode_f(plist_id,io_mode,error) + if (error /= 0) write(*,*) 'hdf5 get_mpio_actual_io_mode failed for ',trim(dataset_name) + !if (io_mode == H5D_MPIO_NO_COLLECTIVE_F) print *, & + ! "collective read/write not possible for dataset: ", dataset_name + call check_error() + end subroutine h5_check_collective + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_check_arr_dim(dim) + implicit none + integer(kind=HSIZE_T), dimension(:), intent(in) :: dim + integer :: i + ! if one of the dimension is 0, cancel space_id and file_id + ! loop all elements in dim + do i = 1, size(dim) + if (dim(i) == 0) then + call h5sselect_none_f(mem_dspace_id, error) + call check_error() + call h5sselect_none_f(file_dspace_id, error) + call check_error() + endif + enddo + end subroutine h5_check_arr_dim + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_file(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5fcreate_f(trim(file_path), H5F_ACC_TRUNC_F, file_id, error) + if (error /= 0) then + print *,'Error file create: ', trim(file_path) + print * + print *,'check if path exists: ', trim(file_path) + stop 'Error file create h5' + endif + end subroutine h5_create_file + +! +!------------------------------------------------------------------------------- +! + subroutine h5_create_or_open_file(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + logical :: file_exists + + ! set the target file path + file_path = fpath_in + + ! check if file exists + inquire(file=trim(file_path), exist=file_exists) + if (file_exists) then + call h5fopen_f(trim(file_path), H5F_ACC_RDWR_F, file_id, error) + if (error /= 0) then + print *,'Error file open: ', trim(file_path) + stop 'Error file open h5' + endif + else + call h5fcreate_f(trim(file_path), H5F_ACC_TRUNC_F, file_id, error) + if (error /= 0) then + print *,'Error file create: ', trim(file_path) + print * + print *,'check if path exists: ', trim(file_path) + stop 'Error file create h5' + endif + endif + end subroutine h5_create_or_open_file + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_file(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5fopen_f(trim(file_path), H5F_ACC_RDWR_F, file_id, error) + if (error /= 0) then + print *, 'Error file open: ', trim(file_path) + stop 'Error file open h5' + endif + end subroutine h5_open_file + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_file() + implicit none + call h5fclose_f(file_id, error) + if (error /= 0) write(*,*) 'error while closing a file.' + call check_error() + end subroutine h5_close_file + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_file_p() + implicit none + call h5_close_file_prop_list() + call h5fclose_f(file_id, error) + if (error /= 0) write(*,*) 'error while closing a file.' + call check_error() + end subroutine h5_close_file_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_group(group_name) + implicit none + character(len=*), intent(in) :: group_name + call h5gcreate_f(file_id, trim(group_name), group_id, error) + if (error /= 0) write(*,*) 'error while creating a group, ', group_name + call check_error() + call h5gclose_f(group_id, error) + if (error /= 0) write(*,*) 'error while closing a group, ' , group_name + call check_error() + end subroutine h5_create_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_subgroup(group_name) + implicit none + character(len=*), intent(in) :: group_name + integer(HID_T) :: temp_group_id + + call h5gcreate_f(group_id, trim(group_name), temp_group_id, error) + if (error /= 0) write(*,*) 'error while creating a subgroup, ', group_name + call check_error() + call h5gclose_f(temp_group_id, error) + if (error /= 0) write(*,*) 'error while closing a subgroup, ' , group_name + call check_error() + end subroutine h5_create_subgroup + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_group(group_name) + implicit none + character(len=*), intent(in) :: group_name + ! open group + call h5gopen_f(file_id, trim(group_name), group_id, error) ! group open + if (error /= 0) write(*,*) 'hdf5 open group failed for ', trim(group_name) + call check_error() + ! stores name + store_group_name = group_name + end subroutine h5_open_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_subgroup(group_name) + implicit none + character(len=*), intent(in) :: group_name + integer(HID_T) :: temp_group_id + ! open subgroup + call h5gopen_f(group_id, trim(group_name), temp_group_id, error) ! group open + if (error /= 0) write(*,*) 'hdf5 open subgroup failed for ', trim(group_name) + call check_error() + ! put the group id of the first level becomes parent group + ! only while the group at second level exists + parent_group_id = group_id + group_id = temp_group_id + end subroutine h5_open_subgroup + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_or_create_group(group_name) + ! Check if group with the given name exists. Create it if it doesn't, + ! open it if it does. + implicit none + character(len=*), intent(in) :: group_name + ! Variable for checking if a group exists or not + logical :: group_exists + ! check group + call h5_check_group(group_name, group_exists) + ! open or create + if (group_exists) then + call h5gopen_f(file_id, group_name, group_id, error) + else + call h5gcreate_f(file_id, group_name, group_id, error) + endif + call check_error() + end subroutine h5_open_or_create_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_check_group(group_name,group_exists) + ! Check if group with the given name exists. Create it if it doesn't, + ! open it if it does. + implicit none + character(len=*), intent(in) :: group_name + logical, intent(out) :: group_exists + ! Variable for checking if a group exists or not + group_exists = .false. + ! check + call h5lexists_f(file_id, group_name, group_exists, error) + call check_error() + end subroutine h5_check_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_group() + implicit none + call h5gclose_f(group_id, error) ! group open + if (error /= 0) write(*,*) 'hdf5 close group failed' + call check_error() + end subroutine h5_close_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_subgroup() + implicit none + call h5gclose_f(group_id, error) ! group open + if (error /= 0) write(*,*) 'hdf5 close group failed' + call check_error() + ! stores group + group_id = parent_group_id + end subroutine h5_close_subgroup + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_set_group_name(group_name) + implicit none + character(len=*), intent(in) :: group_name + store_group_name = group_name + end subroutine h5_set_group_name + +! +!------------------------------------------------------------------------------- +! + + ! open dataset. group need to be opened before. + subroutine h5_open_dataset(dataset_name) + implicit none + character(len=*), intent(in) :: dataset_name + + call h5dopen_f(group_id, trim(dataset_name), dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 open dataset failed for ', dataset_name + call check_error() + end subroutine h5_open_dataset + +! +!------------------------------------------------------------------------------- +! + + ! open dataset without open group + subroutine h5_open_dataset2(dataset_name) + implicit none + character(len=*), intent(in) :: dataset_name + + call h5dopen_f(file_id, trim(dataset_name), dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 open dataset2 failed for ', dataset_name + call check_error() + end subroutine h5_open_dataset2 + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_dataset() + implicit none + call h5dclose_f(dataset_id, error) ! group open + if (error /= 0) write(*,*) 'hdf5 close dataset failed' + call check_error() + end subroutine h5_close_dataset + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_check_dataset_exists(dataset_name, exists) + implicit none + character(len=*), intent(in) :: dataset_name + logical, intent(out) :: exists + + call h5lexists_f(file_id, dataset_name, exists, error) + call check_error() + end subroutine h5_check_dataset_exists + + +!------------------------------------------------------------------------------- +! +! serial write routines +! +!------------------------------------------------------------------------------- + + ! dataset writer for 1d integer array + subroutine h5_write_dataset_1d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + dim = size(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_1d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_1d_i_no_group(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + dim = size(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + ! closes dataset + call h5_close_dataset() + end subroutine h5_write_dataset_1d_i_no_group + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 1d custom real array + subroutine h5_write_dataset_1d_d(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + dim = size(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + if (CUSTOM_REAL == 4) then + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + else + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + endif + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_1d_d + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 1d custom real array without grouping + subroutine h5_write_dataset_1d_d_no_group(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + dim = size(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + if (CUSTOM_REAL == 4) then + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + else + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + endif + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + ! closes dataset + call h5_close_dataset() + end subroutine h5_write_dataset_1d_d_no_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_1d_c(dataset_name, data_in) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), dimension(:), intent(in) :: data_in + integer(HID_T) :: dspace_id ! dataspace id is local. + character(len=str_len), dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer :: i + + dim = shape(data_in) + + allocate(data(dim(1)),stat=error) + ! fill blancs after each string to have the same length of strings + do i = 1, dim(1) + data(i) = data_in(i)//repeat(' ',str_len-len(data_in(i))) + enddo + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(group_id, trim(dataset_name), str_type, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, str_type, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + deallocate(data, stat=error) + + end subroutine h5_write_dataset_1d_c + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_1d_c_no_group(dataset_name, data_in) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), dimension(:), intent(in) :: data_in + integer(HID_T) :: dspace_id ! dataspace id is local. + character(len=str_len), dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer :: i + + dim = shape(data_in) + + allocate(data(dim(1)),stat=error) + ! fill blancs after each string to have the same length of strings + do i = 1, dim(1) + data(i) = data_in(i)//repeat(' ',str_len-len(data_in(i))) + enddo + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(file_id, trim(dataset_name), str_type, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, str_type, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + ! closes dataset + call h5_close_dataset() + deallocate(data, stat=error) + end subroutine h5_write_dataset_1d_c_no_group + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 2d integer array + subroutine h5_write_dataset_2d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_2d_i + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 2d double array + subroutine h5_write_dataset_2d_d(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_2d_d + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 2d double array + subroutine h5_write_dataset_2d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + if (CUSTOM_REAL == 4) then + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + else + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + endif + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_2d_r + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 2d double array + subroutine h5_write_dataset_2d_r_no_group(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + if (CUSTOM_REAL == 4) then + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + else + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + endif + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + ! closes dataset + call h5_close_dataset() + end subroutine h5_write_dataset_2d_r_no_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_4d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + if (CUSTOM_REAL == 4) then + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + else + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + endif + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_4d_r + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 2d character array + subroutine h5_write_dataset_2d_c(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), dimension(:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + ! write string array for undef_mat_prop + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(group_id, trim(dataset_name), str_type_max, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, str_type_max, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_2d_c + +! +!------------------------------------------------------------------------------- +! + + ! set attribute to a dataset + subroutine h5_add_attribute_i(attribute_name, data) + implicit none + integer(HID_T) :: aspace_id, atype_id, attr_id ! attribute id + integer :: rank = 1 ! attribute rank + integer, dimension(:), intent(in) :: data + integer(HSIZE_T) :: taglen + character(len=*), intent(in) :: attribute_name + integer(HSIZE_T), dimension(1) :: dim + + dim = shape(data) + taglen = len(trim(attribute_name)) + + call h5screate_simple_f(rank, dim, aspace_id, error) + if (error /= 0) write(*,*) 'hdf5 screate failed for attribute, ', attribute_name + call check_error() + call h5tcopy_f(H5T_NATIVE_INTEGER, atype_id, error) ! for a string tag of attribute value + if (error /= 0) write(*,*) 'hdf5 tcopy failed for attribute, ', attribute_name + call check_error() + call h5tset_size_f(atype_id, taglen, error) + if (error /= 0) write(*,*) 'hdf5 set_size failed for attribute, ', attribute_name + call check_error() + ! here the attribute is written on the current openning dataset + call h5acreate_f(dataset_id, trim(attribute_name), atype_id, aspace_id, attr_id, error) + if (error /= 0) write(*,*) 'hdf5 acreate failed for attribute, ', attribute_name + call check_error() + call h5awrite_f(attr_id, atype_id, data, dim, error) ! write + if (error /= 0) write(*,*) 'hdf5 awrite failed for attribute, ', attribute_name + call check_error() + call h5sclose_f(aspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', attribute_name + call check_error() + call h5aclose_f(attr_id, error) + if (error /= 0) write(*,*) 'hdf5 aclose failed for ', attribute_name + call check_error() + call h5tclose_f(atype_id, error) + if (error /= 0) write(*,*) 'hdf5 tclose failed for ', attribute_name + call check_error() + end subroutine h5_add_attribute_i + + +!------------------------------------------------------------------------------- +! +! parallel routines +! +!------------------------------------------------------------------------------- + + subroutine h5_set_mpi_info(comm, info, rank, nproc) + implicit none + integer, intent(in) :: comm, info, rank, nproc + + this_comm = comm + this_info = info + this_rank = rank + total_proc = nproc + end subroutine h5_set_mpi_info + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_set_sieve_buffer_size() + implicit none + integer(hsize_t) :: buf_size = 1*1024*1024 ! 1 MB + integer(hsize_t) :: alig_size = 1*1024*1024 ! 1 MB + call h5pset_sieve_buf_size_f(fplist_id, buf_size, error) ! buf_size may vary depending on machiens + call check_error() + call h5pset_alignment_f(fplist_id, buf_size, alig_size, error) + call check_error() + end subroutine h5_set_sieve_buffer_size + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_set_buffer_size() + implicit none + integer(hsize_t) :: buf_size = 1*1024*1024 ! 1 MB + call h5pset_buffer_f(plist_id, buf_size, error) + call check_error() + end subroutine h5_set_buffer_size + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_file_prop_list(if_collective) + implicit none + logical, intent(in) :: if_collective + + ! Setup file access property list with parallel I/O access. + call h5pcreate_f(H5P_FILE_ACCESS_F, fplist_id, error) + if (error /= 0) write(*,*) 'hdf5 create plist failed.' + call check_error() + call h5pset_all_coll_metadata_ops_f(fplist_id, if_collective, error) + call check_error() + call h5pset_coll_metadata_write_f(fplist_id, if_collective, error) + call check_error() + call h5pset_fapl_mpio_f(fplist_id, this_comm, this_info, error) + if (error /= 0) write(*,*) 'hdf5 set_fapl failed.' + call check_error() + end subroutine h5_create_file_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_group_prop_list() + implicit none + logical, parameter :: if_collective = .true. + ! Setup file access property list with parallel I/O access. + call h5pcreate_f(H5P_GROUP_ACCESS_F, gplist_id, error) + if (error /= 0) write(*,*) 'hdf5 create group plist failed.' + call check_error() + call h5pset_all_coll_metadata_ops_f(gplist_id, if_collective, error) + call check_error() + call h5pset_coll_metadata_write_f(gplist_id, if_collective, error) + if (error /= 0) write(*,*) 'hdf5 group set_all_coll_metadata_ops failed.' + call check_error() + end subroutine h5_open_group_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_group_prop_list() + implicit none + ! Setup file access property list with parallel I/O access. + call h5pcreate_f(H5P_GROUP_CREATE_F, gplist_id, error) + if (error /= 0) write(*,*) 'hdf5 create group plist failed.' + call check_error() + end subroutine h5_create_group_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_dataset_prop_list(if_collective) + implicit none + logical, intent(in) :: if_collective + ! Setup file access property list with parallel I/O access. + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + if (error /= 0) write(*,*) 'hdf5 create dataset plist failed.' + call check_error() + ! use larger buffer size from the default + call h5_set_buffer_size() + if (if_collective) then + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + else + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_INDEPENDENT_F, error) + endif + if (error /= 0) write(*,*) 'hdf5 dataset set_dxpl_mpio failed.' + call check_error() + end subroutine h5_create_dataset_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_prop_list(dataset_name) + implicit none + character(len=*), intent(in) :: dataset_name + !print *,"DEBUG closing dataset prop: ", dataset_name + ! check if the latest dataset transfer has been done in collective mode + call h5_check_collective(dataset_name) + call h5pclose_f(plist_id, error) ! property list can be closed soon + if (error /= 0) write(*,*) 'hdf5 close_prop_list failed for ',trim(dataset_name) + call check_error() + end subroutine h5_close_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_prop_list_nocheck(dataset_name) + implicit none + character(len=*), intent(in) :: dataset_name + !print *,"DEBUG closing dataset prop: ", dataset_name + ! check if the latest dataset transfer has been done in collective mode + call h5pclose_f(plist_id, error) ! property list can be closed soon + if (error /= 0) write(*,*) 'hdf5 close_prop_list failed for ',trim(dataset_name) + call check_error() + end subroutine h5_close_prop_list_nocheck + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_file_prop_list() + implicit none + call h5pclose_f(fplist_id, error) ! property list can be closed soon + if (error /= 0) write(*,*) 'hdf5 close_file_prop_list failed.' + call check_error() + end subroutine h5_close_file_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_group_prop_list() + implicit none + call h5pclose_f(gplist_id, error) ! property list can be closed soon + if (error /= 0) write(*,*) 'hdf5 close_prop_list failed.' + call check_error() + end subroutine h5_close_group_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_file_p(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5_create_file_prop_list(.false.) + call h5fcreate_f(trim(file_path), H5F_ACC_TRUNC_F, file_id, error, creation_prp=H5P_DEFAULT_F, access_prp=fplist_id) + if (error /= 0) write(*,*) 'hdf5 create file p failed.' + call check_error() + end subroutine h5_create_file_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_file_p_collect(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5_create_file_prop_list(.true.) + call h5fcreate_f(trim(file_path), H5F_ACC_TRUNC_F, file_id, error, creation_prp=H5P_DEFAULT_F, access_prp=fplist_id) + if (error /= 0) write(*,*) 'hdf5 create file p failed.' + call check_error() + end subroutine h5_create_file_p_collect + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_file_p(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5_create_file_prop_list(.false.) + call h5fopen_f(trim(file_path), H5F_ACC_RDWR_F, file_id, error, access_prp=fplist_id) + if (error /= 0) write(*,*) 'hdf5 open file p failed.' + call check_error() + end subroutine h5_open_file_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_file_p_collect(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5_create_file_prop_list(.true.) + call h5_set_sieve_buffer_size() + + call h5fopen_f(trim(file_path), H5F_ACC_RDWR_F, file_id, error, access_prp=fplist_id) + if (error /= 0) write(*,*) 'hdf5 open file p failed.' + call check_error() + end subroutine h5_open_file_p_collect + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_dataset_gen(dataset_name, dim_in, rank, dtype_id) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in) :: dim_in + + integer(HSIZE_T), dimension(size(dim_in)) :: dim + integer, intent(in) :: dtype_id ! 1:int, 4:real4, 8:real8, + integer, intent(in) :: rank + + integer(HID_T) :: dspace_id + !logical :: if_chunk = .true. + !integer :: i + + dim = dim_in ! convert data type + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5pcreate_f(H5P_DATASET_CREATE_F, plist_id, error) + call check_error() + + ! chunk size setting + !do i = 1, rank + ! if (dim(i) <= 0) then + ! if_chunk = .false. + ! print *, "dataset not chunk set: ", dataset_name + ! endif + !enddo + !if (if_chunk) call h5pset_chunk_f(plist_id,rank,dim,error) + + if (dtype_id == 0) then ! bool uses integer + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 1) then ! integer + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 2) then ! character + call h5dcreate_f(file_id, trim(dataset_name), str_type, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 4) then ! real + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 8) then ! double + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else + print *, "specified dtype_id is not implemented yet for hdf5 io. aborting..." + stop 'Invalid dtype_id, not implemented yet in h5_create_dataset_gen() routine' + endif + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + + call h5_close_prop_list_nocheck(dataset_name) + + call h5dclose_f(dataset_id,error) + if (error /= 0) write(*,*) 'hdf5 dataset close failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace close failed for ', dataset_name + call check_error() + + end subroutine h5_create_dataset_gen + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_dataset_gen_in_group(dataset_name, dim_in, rank, dtype_id) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in) :: dim_in + + integer(HSIZE_T), dimension(size(dim_in)) :: dim + integer, intent(in) :: dtype_id ! 1:int, 4:real4, 8:real8, + integer, intent(in) :: rank + + integer(HID_T) :: dspace_id + + dim = dim_in ! convert data type + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5pcreate_f(H5P_DATASET_CREATE_F, plist_id, error) + call check_error() + if (dtype_id == 0) then ! logial uses integer + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 1) then ! integer + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 2) then ! character + call h5dcreate_f(group_id, trim(dataset_name), str_type, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 4) then ! real + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 8) then ! double + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else + print *, "Error: specified dtype_id is not implemented yet for hdf5 io. aborting..." + stop 'Invalid dtype_id, not implemented yet in h5_create_dataset_gen_in_group()' + endif + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + + call h5_close_prop_list_nocheck(dataset_name) + + call h5dclose_f(dataset_id,error) + if (error /= 0) write(*,*) 'hdf5 dataset close failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace close failed for ', dataset_name + call check_error() + end subroutine h5_create_dataset_gen_in_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_group_p(group_name) + implicit none + character(len=*), intent(in) :: group_name + + call h5_create_group_prop_list() + call h5gcreate_f(file_id, trim(group_name), group_id, error, gcpl_id=gplist_id) + if (error /= 0) write(*,*) 'error while creating a group, ', group_name + call check_error() + call h5gclose_f(group_id, error) + if (error /= 0) write(*,*) 'error while closing a group, ' , group_name + call check_error() + call h5_close_group_prop_list() + end subroutine h5_create_group_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_group_p(group_name) + implicit none + character(len=*), intent(in) :: group_name + call h5_open_group_prop_list() + call h5gopen_f(file_id, trim(group_name), group_id, error, gapl_id=gplist_id) + if (error /= 0) write(*,*) 'hdf5 open group failed for ', group_name + call check_error() + call h5_close_group_prop_list() + store_group_name = group_name + end subroutine h5_open_group_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_scalar_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, intent(out) :: data + integer, dimension(1) :: rdata + integer(HSIZE_T), dimension(1) :: dim + dim = shape(rdata) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, rdata, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + data = rdata(1) + end subroutine h5_read_dataset_p_scalar_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_scalar_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), intent(out) :: data + real(kind=CUSTOM_REAL), dimension(1) :: rdata + integer(HSIZE_T), dimension(1) :: dim + dim = shape(rdata) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, rdata, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, rdata, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + data = rdata(1) + end subroutine h5_read_dataset_p_scalar_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_1d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(inout) :: data + integer(HSIZE_T), dimension(1) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_1d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_1d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(inout) :: data + integer(HSIZE_T), dimension(1) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_1d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_1d_l(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(inout) :: data + integer, dimension(:), allocatable :: ldata + integer(HSIZE_T), dimension(1) :: dim + integer :: lsize + dim = shape(data) + lsize = size(data) + allocate(ldata(lsize),stat=error) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, ldata, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + call int_array2bool(ldata, data) + + deallocate(ldata, stat=error) + end subroutine h5_read_dataset_p_1d_l + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_2d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(inout) :: data + integer(HSIZE_T), dimension(2) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_2d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_2d_d(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(inout) :: data + integer(HSIZE_T), dimension(2) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_2d_d + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_2d_c(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), dimension(:,:), intent(inout) :: data + integer(HSIZE_T), dimension(2) :: dim + + ! reads in string array for undef_mat_prop + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, str_type_max, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_2d_c + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_2d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(inout) :: data + integer(HSIZE_T), dimension(2) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_2d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_3d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(inout) :: data + integer(HSIZE_T), dimension(3) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_3d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_3d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(inout) :: data + integer(HSIZE_T), dimension(3) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_3d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_4d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(inout) :: data + integer(HSIZE_T), dimension(4) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_4d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_5d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(inout) :: data + integer(HSIZE_T), dimension(5) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_5d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_attribute_p(attribute_name, dataset_name, data) + implicit none + character(len=*), intent(in) :: attribute_name + character(len=*), intent(in) :: dataset_name + integer(HID_T) :: attr_id + integer, dimension(:), intent(inout) :: data + integer(HSIZE_T), dimension(1) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5aopen_f(dataset_id, attribute_name, attr_id, error) + call check_error() + call h5aread_f(attr_id, H5T_NATIVE_INTEGER, data, dim, error) + call check_error() + call h5aclose_f(attr_id,error) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_attribute_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_4d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(inout) :: data + integer(HSIZE_T), dimension(4) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_4d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_scalar_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + !dim = shape(data) + dim = (/1/) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_scalar_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_scalar_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + !dim = shape(data) + dim = (/1/) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_scalar_i_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_scalar_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + !dim = shape(data) + dim = (/1/) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_scalar_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_1d_l_collect_hyperslab(dataset_name, data_out, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(inout), target :: data_out + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer, dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data_out) + offset = offset_in ! convert data type + allocate(data(dim(1))) + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + f_ptr = c_loc(data(1)) + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + + call int_array2bool(data, data_out) + deallocate(data) + end subroutine h5_read_dataset_1d_l_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_1d_l_collect_hyperslab_in_group(dataset_name, data_out, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(inout), target :: data_out + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer, dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data_out) + offset = offset_in ! convert data type + allocate(data(dim(1))) + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + f_ptr = c_loc(data(1)) + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + + call int_array2bool(data, data_out) + deallocate(data) + end subroutine h5_read_dataset_1d_l_collect_hyperslab_in_group + +! +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_1d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + call h5_check_arr_dim(dim) + + ! write array using Fortran pointer + !call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! use F2003 API + f_ptr = c_loc(data(1)) + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_1d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_1d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + call h5_check_arr_dim(dim) + + ! write array using Fortran pointer + !call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! use F2003 API + f_ptr = c_loc(data(1)) + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_1d_i_collect_hyperslab_in_group + +! +! +!------------------------------------------------------------------------------- +! + + ! store local 1d array to global 1d array + subroutine h5_read_dataset_1d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_1d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 1d array to global 1d array + subroutine h5_read_dataset_1d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_1d_r_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + ! store local 1d array to global 1d array + subroutine h5_read_dataset_1d_d_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_1d_d_collect_hyperslab_in_group + +! +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_2d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_2d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_2d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_2d_i_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + ! store local 2d array to global 2d array + subroutine h5_read_dataset_2d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1)! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_2d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 2d array to global 2d array + subroutine h5_read_dataset_2d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1)! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_2d_r_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_2d_d_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1)! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_2d_d_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_3d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_3d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_3d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_3d_i_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_3d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_3d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_3d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_3d_r_collect_hyperslab_in_group + +! +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_4d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_4d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_4d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_4d_i_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_4d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_4d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_4d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_4d_r_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_5d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 5 + integer(HSIZE_T), dimension(5) :: dim + integer(HSIZE_T), dimension(5) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(5) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + count(5) = dim(5) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_5d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_5d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 5 + integer(HSIZE_T), dimension(5) :: dim + integer(HSIZE_T), dimension(5) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(5) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + count(5) = dim(5) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_5d_r_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! +! parallel write routines +! +!------------------------------------------------------------------------------- + +! +! collective writers +! + + subroutine h5_write_dataset_p_1d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in), target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + ! dummy array for generate 0 length dataset + integer, dimension(1) :: dummy_1d_array = (/0/) + + dim = shape(data) + + ! add dummy 0 for no_element array + if (size(data) == 0) then + dim = 1 + endif + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + if (size(data) == 0) then + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, dummy_1d_array, dim, error,xfer_prp=plist_id) + else + f_ptr = c_loc(data(1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_1d_i + +! +!------------------------------------------------------------------------------- +! + + ! write 1d integer and attribute + subroutine h5_write_dataset_p_1d_ia(dataset_name, data, attribute_name, attr_data) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), intent(in) :: attribute_name + integer, dimension(:), intent(in), target :: data + integer, intent(in) :: attr_data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + ! dummy array for generate 0 length dataset + integer, dimension(1) :: dummy_1d_array = (/0/) + + dim = shape(data) + + ! add dummy 0 for no_element array + if (size(data) == 0) then + dim = 1 + endif + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + if (size(data) == 0) then + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, dummy_1d_array, dim, error,xfer_prp=plist_id) + else + f_ptr = c_loc(data(1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + ! add attribute + call h5_add_attribute_i(attribute_name, (/attr_data/)) + + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_1d_ia + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_1d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in), target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, CUSTOM_REAL, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1)) + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_1d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_2d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(in), target :: data + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_2d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_2d_ia(dataset_name, data, attribute_name, attr_data) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), intent(in) :: attribute_name + integer, dimension(:,:), intent(in), target :: data + integer, intent(in) :: attr_data + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + ! add attribute + call h5_add_attribute_i(attribute_name, (/attr_data/)) + + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_2d_ia + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_2d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in), target :: data + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, CUSTOM_REAL, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1)) + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_2d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_2d_d(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(in), target :: data + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 8, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_2d_d + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_3d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(in), target :: data + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1,1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_3d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_3d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(in), target :: data + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, CUSTOM_REAL, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1,1)) + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_3d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_4d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(in), target :: data + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1,1,1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_4d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_4d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(in), target :: data + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, CUSTOM_REAL, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1,1,1)) + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_4d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_5d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(in), target :: data + integer :: rank = 5 + integer(HSIZE_T), dimension(5) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, CUSTOM_REAL, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1,1,1,1)) + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_5d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_1d_l(dataset_name, data_in) + ! writer for logical array + ! logical array will be converted to integer array before write + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(in) :: data_in + integer, dimension(:), allocatable, target :: data + + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data_in) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + allocate(data(size(data_in)), stat=error) + call bool_array2integer(data_in, data) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + deallocate(data) + end subroutine h5_write_dataset_p_1d_l + +! +!------------------------------------------------------------------------------- +! + + ! from 1d local array to 2d global array + subroutine h5_write_dataset_1d_to_2d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in), target :: data + integer, dimension(2), intent(in) :: offset_in ! the position where the datablock is inserted + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset + + dim = size(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! size of data array inserted. + count(1) = dim(1) + count(2) = 1 + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_to_2d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 2d array to global 3d array + subroutine h5_write_dataset_2d_to_3d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = 3! NDIM + count(2) = dim(2) ! NSTEP partial + count(3) = 1 ! nrec + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_to_3d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_1d_l_collect_hyperslab(dataset_name, data_in, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(in), target :: data_in + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer, dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data_in) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! if_collective == .false., this function gather all data from procs then write in a file at once + + ! select a place where data is inserted. + count(1) = dim(1) + allocate(data(dim(1))) + + ! convert logical array to integer array + call bool_array2integer(data_in, data) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + deallocate(data) + + end subroutine h5_write_dataset_1d_l_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_1d_l_collect_hyperslab_in_group(dataset_name, data_in, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(in), target :: data_in + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer, dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data_in) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! if_collective == .false., this function gather all data from procs then write in a file at once + + ! select a place where data is inserted. + count(1) = dim(1) + allocate(data(dim(1))) + + ! convert logical array to integer array + call bool_array2integer(data_in, data) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + deallocate(data) + + end subroutine h5_write_dataset_1d_l_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + + ! store local 1d array to global 1d array + subroutine h5_write_dataset_1d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + call h5_check_arr_dim(dim) + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! use F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 1d array to global 1d array + subroutine h5_write_dataset_1d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + call h5_check_arr_dim(dim) + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! use F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_i_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + + subroutine h5_write_dataset_1d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + !call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1)),error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 1d array to global 1d array + subroutine h5_write_dataset_1d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + !call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_r_collect_hyperslab_in_group +! +!------------------------------------------------------------------------------- +! + + + subroutine h5_write_dataset_1d_d_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_d_collect_hyperslab + + +! +!------------------------------------------------------------------------------- +! + subroutine h5_write_dataset_1d_d_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_d_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + + ! store local 2d array to global 2d array + subroutine h5_write_dataset_2d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_2d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_i_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + ! store local 2d array to global 2d array + subroutine h5_write_dataset_2d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + !call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_r_collect_hyperslab +! +!------------------------------------------------------------------------------- +! + + ! store local 2d array to global 2d array + subroutine h5_write_dataset_2d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + !call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_r_collect_hyperslab_in_group + +! +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_2d_d_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_d_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_2d_d_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_d_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_3d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_3d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_3d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_3d_i_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + ! store local 3d array to global 3d array + subroutine h5_write_dataset_3d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_3d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 3d array to global 3d array + subroutine h5_write_dataset_3d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_3d_r_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + + subroutine h5_write_dataset_4d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_4d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_4d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_4d_i_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + + subroutine h5_write_dataset_4d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_4d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_4d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_4d_r_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_5d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 5 + integer(HSIZE_T), dimension(5) :: dim + integer(HSIZE_T), dimension(5) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(5) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + count(5) = dim(5) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_5d_r_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_5d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 5 + integer(HSIZE_T), dimension(5) :: dim + integer(HSIZE_T), dimension(5) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(5) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + count(5) = dim(5) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_5d_r_collect_hyperslab + +!------------------------------------------------------------------------------- +! +! other utilities +! +!------------------------------------------------------------------------------- + + subroutine check_error() + implicit none + if (error /= 0) then + print *,'Error: HDF5 routine call got error ',error + stop 'HDF5 error' + endif + end subroutine check_error + +! +!------------------------------------------------------------------------------- +! + + subroutine bool_array2integer(boolarray, intarray) + implicit none + logical, dimension(:), intent(in) :: boolarray + integer, dimension(:), intent(out) :: intarray + integer :: array_size, i + + array_size = size(boolarray) + intarray(:) = 0 + do i = 1, array_size + if (boolarray(i)) then + intarray(i) = 1 + endif + enddo + end subroutine bool_array2integer + +! +!------------------------------------------------------------------------------- +! + + subroutine int_array2bool(intarray, boolarray) + implicit none + integer, dimension(:), intent(in) :: intarray + logical, dimension(:), intent(out) :: boolarray + integer :: array_size,i + + array_size = size(intarray) + boolarray(:) = .false. + + do i = 1, array_size + if (intarray(i) /= 0) then + boolarray(i) = .true. + endif + enddo + end subroutine int_array2bool + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_gather_dsetsize(dimin, data_rank, all_dim) + + implicit none + integer, intent(in) :: data_rank + integer(HSIZE_T), dimension(:), intent(in) :: dimin + integer, dimension(data_rank) :: dim + integer, dimension(data_rank,0:total_proc-1), intent(out) :: all_dim + + dim = dimin ! convert integer 8(HSIZE_T) to 4 + + if (data_rank == 1) then + call gather_all_all_singlei(dim(1),all_dim,total_proc) + !debug + !if (this_rank == 0) then + ! print *, "rank: ", this_rank & + ! ,"dimin", dimin, ", kind: ", kind(dimin) & + ! ,"dim", dim, ", kind: ", kind(dim) & + ! ,"shape dimin", shape(dimin) & + ! ,"data_rank", data_rank & + ! ,"all dim", all_dim & + ! ,"shape all dim", shape(all_dim) & + ! ,"kind all_dim", kind(all_dim(1,0)) & + ! ,"total proc " , total_proc & + ! ,"kind mpi_integer ", kind(MPI_INTEGER) + !endif + else + ! gather only on main rank + !call gather_all_i(dim, data_rank, all_dim, data_rank, total_proc) + ! gather on all processes + call gather_all_all_i(dim, data_rank, all_dim, data_rank, total_proc) + endif + + call synchronize_all() + end subroutine + +! +!------------------------------------------------------------------------------- +! + + subroutine create_dataset_collect(dataset_name, dim, data_rank, dtype_id, base_id) + implicit none + character(len=*), intent(in) :: dataset_name + integer(HSIZE_T), dimension(:), intent(in) :: dim + integer, intent(in) :: dtype_id ! 1:int, 4:real4, 8:real8, + integer, intent(in) :: data_rank + integer(HID_T), intent(in) :: base_id ! base (file_id or group_id) of dataset creation + + integer, dimension(data_rank,0:total_proc-1) :: all_dim + integer(HSIZE_T), dimension(data_rank) :: dim_h5 + integer :: iproc + character(len=128) :: group_and_dataset_name + integer(HID_T) :: dspace_id + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + ! gather dataset dimension of other processors + call h5_gather_dsetsize(dim, data_rank, all_dim) + ! create all datasets of all process by rank0 + + !print *, dataset_name, ": debug returned all dim", all_dim + do iproc = 0, total_proc -1 + write(tempstr, "(i6.6)") iproc + group_name = gname_proc_head // trim(tempstr) + group_and_dataset_name = trim(group_name) //"/"// dataset_name + + dim_h5 = all_dim(:,iproc) + + call h5screate_simple_f(data_rank, dim_h5, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5pcreate_f(H5P_DATASET_CREATE_F, plist_id, error) + call check_error() + ! This is required for this data pattern + call H5Pset_alloc_time_f(plist_id, H5D_ALLOC_TIME_EARLY_F, error) + call check_error() + if (dtype_id == 1) then + call h5dcreate_f(base_id, trim(group_and_dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 4) then + call h5dcreate_f(base_id, trim(group_and_dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else + call h5dcreate_f(base_id, trim(group_and_dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5dclose_f(dataset_id,error) + if (error /= 0) write(*,*) 'hdf5 dataset close failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace close failed for ', dataset_name + call check_error() + enddo + + call synchronize_all() + end subroutine create_dataset_collect + +!------------------------------------------------------------------------------- +! +! object-oriented interface +! +!------------------------------------------------------------------------------- +! Fortran 2003 standard +! +! this interface is mainly for testing purposes, and for checking if compilers by now support these features :( +! +! in the past, we had issues with compilers due to keywords like class(). +! here, we use strictly Fortran 2003 features, like polymorphism with 'class()', object destructor procedure by 'final ::', +! and type binding by a pass statement like 'procedure :: . => ..' +! +! for a status report on compiler support, see Fortran+2003+status on: +! https://Fortranwiki.org/ +! Cray, GNU, IBM and Intel compilers should well support these features. + + ! constructors + + function h5io_constructor() result(this) + implicit none + type(h5io) :: this + call h5_initialize() + this%is_initialized = .true. + end function h5io_constructor + + function h5io_constructor_from_file(filename) result(this) + implicit none + type(h5io) :: this + character(*), intent(in) :: filename + call h5_initialize() + this%is_initialized = .true. + call this%open(filename) + this%filename = trim(filename) + end function h5io_constructor_from_file + + ! destructor + + subroutine h5io_destructor(this) + implicit none + type(h5io) :: this + call h5_finalize() + this%is_initialized = .false. + end subroutine h5io_destructor + + ! object procedures + + ! note: with the pass statement "procedure :: open => h5io_open_file", the routine must have at least one argument and + ! the argument 'this' must be defined as polymorphic, i.e., by class() and not type() as above + + subroutine h5io_open_file(this,filename,status) + implicit none + class(h5io) :: this + character(*), intent(in) :: filename + character(*), intent(in), optional :: status + ! checks if initialized + if (.not. this%is_initialized) then + call h5_initialize() + this%is_initialized = .true. + endif + ! only main process creates file if needed + if (present(status)) then + if (status == 'new') call h5_create_file(filename) + endif + ! opens file + call h5_open_file(filename) + this%f_id = file_id + this%filename = trim(filename) + end subroutine h5io_open_file + + subroutine h5io_close_file(this) + implicit none + class(h5io) :: this + call h5_close_file() + this%f_id = file_id + end subroutine h5io_close_file + + subroutine h5io_write_dataset_i(this,dname,data_i,group) + implicit none + class(h5io) :: this + character(len=*), intent(in) :: dname + integer, dimension(:), intent(in) :: data_i + character(len=*), intent(in), optional :: group + ! writes either with or without group + if (present(group)) then + call h5_open_or_create_group(group) + call h5_write_dataset(dname,data_i) + else + call h5_write_dataset_no_group(dname,data_i) + endif + this%d_id = dataset_id + end subroutine h5io_write_dataset_i + + subroutine h5io_write_dataset_r(this,dname,data_r,group) + implicit none + class(h5io) :: this + character(len=*), intent(in) :: dname + real(kind=CUSTOM_REAL), dimension(:), intent(in) :: data_r + character(len=*), intent(in), optional :: group + ! writes either with or without group + if (present(group)) then + call h5_open_or_create_group(group) + call h5_write_dataset(dname,data_r) + else + call h5_write_dataset_no_group(dname,data_r) + endif + this%d_id = dataset_id + end subroutine h5io_write_dataset_r + +#endif + +end module manager_hdf5 + +!----------------------------------------------------- + +#if defined(USE_HDF5) + +! test function for object-oriented interface + + subroutine test_io_hdf5() + use constants, only: myrank + use manager_hdf5, only: h5io + implicit none + type(h5io) :: h5 + integer :: store_x(10) + integer :: i + + ! serial test by main process only + if (myrank /= 0) return + + ! initialize + store_x(:) = (/(i,i = 1,10)/) + + ! hdf5 + ! calls the object constructor, i.e. h5io_constructor() + h5 = h5io() + + ! open file + call h5%open("tmp_test.h5",status='new') ! object function call + + ! write out data + call h5%write("x",store_x) + + ! close file + call h5%close() + + ! h5 goes out of scope, will call the object destructor, i.e., h5io_destructor() + end subroutine test_io_hdf5 + +#endif diff --git a/src/shared/parallel.f90 b/src/shared/parallel.f90 index 45abc0741..ee51ef6f0 100644 --- a/src/shared/parallel.f90 +++ b/src/shared/parallel.f90 @@ -71,6 +71,7 @@ module my_mpi ! my MPI group for simultaneous runs integer :: my_local_mpi_comm_world integer :: my_local_mpi_comm_for_bcast + integer :: my_local_mpi_comm_inter ! MPI subgroup for hdf5 i/o server end module my_mpi @@ -1527,6 +1528,28 @@ subroutine gather_all_i(sendbuf, sendcnt, recvbuf, recvcount, NPROC) end subroutine gather_all_i +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gather_all_all_i(sendbuf, sendcnt, recvbuf, recvcount, NPROC) + + use my_mpi + + implicit none + + integer :: sendcnt, recvcount, NPROC + integer, dimension(sendcnt) :: sendbuf + integer, dimension(recvcount,0:NPROC-1) :: recvbuf + + integer :: ier + + call MPI_ALLGATHER(sendbuf,sendcnt,MPI_INTEGER, & + recvbuf,recvcount,MPI_INTEGER, & + my_local_mpi_comm_world,ier) + + end subroutine gather_all_all_i + ! !------------------------------------------------------------------------------------------------- ! @@ -1549,6 +1572,28 @@ subroutine gather_all_singlei(sendbuf, recvbuf, NPROC) end subroutine gather_all_singlei +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gather_all_all_singlei(sendbuf, recvbuf, NPROC) + + use my_mpi + + implicit none + + integer :: NPROC + integer :: sendbuf + integer, dimension(0:NPROC-1) :: recvbuf + + integer :: ier + + call MPI_ALLGATHER(sendbuf,1,MPI_INTEGER, & + recvbuf,1,MPI_INTEGER, & + my_local_mpi_comm_world,ier) + + end subroutine gather_all_all_singlei + ! !------------------------------------------------------------------------------------------------- ! diff --git a/src/shared/read_parameter_file.F90 b/src/shared/read_parameter_file.F90 index 0c69eba04..3d27ee97f 100644 --- a/src/shared/read_parameter_file.F90 +++ b/src/shared/read_parameter_file.F90 @@ -366,6 +366,17 @@ subroutine read_parameter_file() call read_value_double_precision(FILESYSTEM_IO_BANDWIDTH, 'FILESYSTEM_IO_BANDWIDTH', ier); ier = 0 endif + ! HDF5 file I/O + ! (optional) hdf5 database io flag + call read_value_logical(HDF5_ENABLED, 'HDF5_ENABLED', ier); ier = 0 + ! HDF file I/O server + if (HDF5_ENABLED) then + ! (optional) movie outputs + call read_value_logical(HDF5_FOR_MOVIES, 'HDF5_FOR_MOVIES', ier); ier = 0 + ! (optional) number of io dedicated nodes + call read_value_integer(HDF5_IO_NODES, 'HDF5_IO_NODES', ier); ier = 0 + endif + ! closes parameter file call close_parameter_file() @@ -399,5 +410,20 @@ subroutine read_parameter_file() endif #endif + ! checks HDF5 compilation support +#if !defined(USE_HDF5) + if (HDF5_ENABLED) then + print * + print *,'**************' + print *,'**************' + print *,'HDF5 is enabled in parameter file but the code was not compiled with HDF5' + print *,'See --with-hdf5 configure options.' + print *,'**************' + print *,'**************' + print * + stop 'an error occurred while reading the parameter file: HDF5 is enabled but code not built with HDF5' + endif +#endif + end subroutine read_parameter_file diff --git a/src/shared/rules.mk b/src/shared/rules.mk index a74a794d5..bfb558713 100644 --- a/src/shared/rules.mk +++ b/src/shared/rules.mk @@ -57,6 +57,7 @@ shared_OBJECTS = \ $O/get_model_parameters.shared.o \ $O/get_timestep_and_layers.shared.o \ $O/gll_library.shared.o \ + $O/hdf5_manager.shared_hdf5_module.o \ $O/heap_sort.shared.o \ $O/hex_nodes.shared.o \ $O/init_openmp.shared.o \ @@ -147,6 +148,9 @@ else shared_OBJECTS += $(adios_shared_STUBS) endif +## HDF5 file i/o + + ## ## ASDF ## @@ -277,6 +281,19 @@ $O/%.shared_asdf.o: $S/%.f90 $O/%.cc.o: $S/%.c ${SETUP}/config.h ${CC} -c $(CPPFLAGS) $(CFLAGS) -o $@ $< +## HDF5 +$O/%.shared_hdf5_module.o: $S/%.f90 $O/shared_par.shared_module.o + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< + +$O/%.shared_hdf5_module.o: $S/%.F90 $O/shared_par.shared_module.o + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< + +$O/%.shared_hdf5.o: $S/%.f90 $O/hdf5_manager.shared_hdf5_module.o + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< + +$O/%.shared_hdf5.o: $S/%.F90 $O/hdf5_manager.shared_hdf5_module.o + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< + ## c++ files $O/%.shared.o: $S/%.cpp ${SETUP}/config.h ${CXX} -c $(CXXFLAGS) $(PARALLEL_STL_DEF) -o $@ $< diff --git a/src/shared/shared_par.f90 b/src/shared/shared_par.f90 index eec27f7dd..4aa1b6294 100644 --- a/src/shared/shared_par.f90 +++ b/src/shared/shared_par.f90 @@ -208,6 +208,18 @@ module shared_input_parameters logical :: SHIFT_SIMULTANEOUS_RUNS = .false. double precision :: FILESYSTEM_IO_BANDWIDTH = 0.d0 + ! HDF5 file i/o + logical :: HDF5_ENABLED = .false. ! for all databases i/o in hdf5 + logical :: HDF5_FOR_MOVIES = .false. ! for movies (shakemap, surface movies, volume movies) + + ! HDF5 IO server + ! number of io dedicated nodes + integer :: HDF5_IO_NODES = 0 + + ! flag for io-dedicated/compute node. + logical :: IO_storage_task = .false. + logical :: IO_compute_task = .true. + end module shared_input_parameters ! diff --git a/src/specfem3D/SIEM_compute_kernels.F90 b/src/specfem3D/SIEM_compute_kernels.F90 index 94aebc21c..a92badadc 100644 --- a/src/specfem3D/SIEM_compute_kernels.F90 +++ b/src/specfem3D/SIEM_compute_kernels.F90 @@ -675,6 +675,12 @@ end subroutine write_ensight_perelementAS endif endif + if (HDF5_ENABLED) then + if (myrank == 0) then + print *,'Full gravity kernels: option HDF5_ENABLED not implemented yet, saving them as binary files' + endif + endif + ! scaling factors ! kernel unit [ s / km^3 ] scale_kl = real(scale_t * scale_displ_inv * 1.d9,kind=CUSTOM_REAL) diff --git a/src/specfem3D/get_attenuation.f90 b/src/specfem3D/get_attenuation.f90 index fa2c5a15e..d9052a848 100644 --- a/src/specfem3D/get_attenuation.f90 +++ b/src/specfem3D/get_attenuation.f90 @@ -36,7 +36,7 @@ subroutine get_attenuation_model_3D(iregion_code, & use constants_solver - use shared_parameters, only: ATT_F_C_SOURCE + use shared_parameters, only: ATT_F_C_SOURCE, HDF5_ENABLED use specfem_par, only: ATTENUATION_VAL,ADIOS_FOR_ARRAYS_SOLVER,LOCAL_PATH, & scale_t_inv @@ -97,7 +97,9 @@ subroutine get_attenuation_model_3D(iregion_code, & ! use the filename to determine the actual contents of the read if (ADIOS_FOR_ARRAYS_SOLVER) then ! ADIOS format - call read_attenuation_adios(iregion_code,factor_common, factor_scale, tau_s, vnspec, f_c_source) + call read_attenuation_adios(iregion_code, factor_common, factor_scale, tau_s, vnspec, f_c_source) + else if (HDF5_ENABLED) then + call read_attenuation_hdf5(iregion_code, factor_common, factor_scale, tau_s, vnspec, f_c_source) else ! binary format ! opens corresponding databases file diff --git a/src/specfem3D/noise_tomography.f90 b/src/specfem3D/noise_tomography.f90 index c55b18b52..faaacb301 100644 --- a/src/specfem3D/noise_tomography.f90 +++ b/src/specfem3D/noise_tomography.f90 @@ -1182,6 +1182,8 @@ subroutine save_kernels_strength_noise() ! kernel file output if (ADIOS_FOR_KERNELS) then call write_kernels_strength_noise_adios() + else if (HDF5_ENABLED) then + call write_kernels_strength_noise_hdf5() else ! binary file output call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH) diff --git a/src/specfem3D/prepare_attenuation.f90 b/src/specfem3D/prepare_attenuation.f90 index 12a1ae1bd..9c776f308 100644 --- a/src/specfem3D/prepare_attenuation.f90 +++ b/src/specfem3D/prepare_attenuation.f90 @@ -448,6 +448,8 @@ subroutine prepare_attenuation() if (ADIOS_FOR_SOLVER_MESHFILES) then ! adios file output call save_forward_model_at_shifted_frequency_adios(factor_scale_relaxed_crust_mantle,factor_scale_relaxed_inner_core) + else if (HDF5_ENABLED) then + call save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_crust_mantle,factor_scale_relaxed_inner_core) else ! outputs model files in binary format call save_forward_model_at_shifted_frequency(factor_scale_relaxed_crust_mantle,factor_scale_relaxed_inner_core) diff --git a/src/specfem3D/read_arrays_solver_hdf5.F90 b/src/specfem3D/read_arrays_solver_hdf5.F90 new file mode 100644 index 000000000..68e623fb3 --- /dev/null +++ b/src/specfem3D/read_arrays_solver_hdf5.F90 @@ -0,0 +1,1322 @@ + subroutine read_arrays_solver_hdf5(iregion_code, & + nspec,nglob,nglob_xy, & + nspec_iso,nspec_tiso,nspec_ani, & + rho_vp,rho_vs, & + xstore,ystore,zstore, & + xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz, & + rhostore, kappavstore,muvstore,kappahstore,muhstore,eta_anisostore, & + c11store,c12store,c13store,c14store,c15store,c16store,c22store, & + c23store,c24store,c25store,c26store,c33store,c34store,c35store, & + c36store,c44store,c45store,c46store,c55store,c56store,c66store, & + mu0store, & + ibool,idoubling,ispec_is_tiso, & + rmassx,rmassy,rmassz, & + nglob_oceans,rmass_ocean_load, & + b_rmassx,b_rmassy) + + + use constants_solver + use specfem_par, only: & + ABSORBING_CONDITIONS, & + LOCAL_PATH,ABSORBING_CONDITIONS + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + integer,intent(in) :: iregion_code + integer,intent(in) :: nspec,nglob,nglob_xy + integer,intent(in) :: nspec_iso,nspec_tiso,nspec_ani + + ! Stacey + real(kind=CUSTOM_REAL),dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(inout) :: rho_vp,rho_vs + + real(kind=CUSTOM_REAL), dimension(nglob),intent(inout) :: xstore,ystore,zstore + + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(inout) :: & + xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz + + ! material properties + real(kind=CUSTOM_REAL),dimension(NGLLX,NGLLY,NGLLZ,nspec_iso),intent(inout) :: & + rhostore,kappavstore,muvstore + + ! additional arrays for anisotropy stored only where needed to save memory + real(kind=CUSTOM_REAL),dimension(NGLLX,NGLLY,NGLLZ,nspec_tiso),intent(inout) :: & + kappahstore,muhstore,eta_anisostore + + ! additional arrays for full anisotropy + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nspec_ani),intent(inout) :: & + c11store,c12store,c13store,c14store,c15store,c16store, & + c22store,c23store,c24store,c25store,c26store,c33store,c34store, & + c35store,c36store,c44store,c45store,c46store,c55store,c56store,c66store + + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(inout) :: mu0store + + ! global addressing + integer,dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(inout) :: ibool + integer, dimension(nspec),intent(inout) :: idoubling + logical, dimension(nspec),intent(inout) :: ispec_is_tiso + + ! mass matrices and additional ocean load mass matrix + real(kind=CUSTOM_REAL), dimension(nglob_xy),intent(inout) :: rmassx,rmassy + real(kind=CUSTOM_REAL), dimension(nglob_xy),intent(inout) :: b_rmassx,b_rmassy + + real(kind=CUSTOM_REAL), dimension(nglob),intent(inout) :: rmassz + + integer,intent(in) :: nglob_oceans + real(kind=CUSTOM_REAL), dimension(nglob_oceans),intent(inout) :: rmass_ocean_load + +#ifdef USE_HDF5 + + ! local parameters + integer :: ier,lnspec,lnglob + ! group, dataset name + character(len=64) :: gname_region + + ! MPI variables + integer :: info, comm + + ! if collective read + logical :: if_col = .true. + + ! offset arrays + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nnodes + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nnodes_xy + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nnodes_oceans + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nelems + + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/solver_data.h5' + ! group name + write(gname_region, "('reg',i1)") iregion_code + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the hdf5 file + call h5_open_file_p_collect(name_database_hdf5) + ! open the group + call h5_open_group(gname_region) + + call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes", offset_nnodes, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes_xy", offset_nnodes_xy, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes_oceans", offset_nnodes_oceans, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("offset_nelems", offset_nelems, (/0/), if_col) + + lnspec = offset_nelems(myrank) + lnglob = offset_nnodes(myrank) + + ! checks dimensions + if (lnspec /= nspec) then + call h5_close_group() + call h5_close_file() + print *, 'Error at rank ', myrank + print *,'Error file dimension: nspec in file = ',lnspec,' but nspec desired:',nspec + print *,'please check file ', name_database_hdf5 + call exit_mpi(myrank,'Error dimensions in solver_data.h5') + endif + if (lnglob /= nglob) then + close(IIN) + print *, 'Error at rank ', myrank + print *,'Error file dimension: nglob in file = ',lnglob,' but nglob desired:',nglob + print *,'please check file ', name_database_hdf5 + call exit_mpi(myrank,'Error dimensions in solver_data.h5') + endif + + ! read xstore + call h5_read_dataset_collect_hyperslab_in_group("xstore", xstore, (/sum(offset_nnodes(0:myrank-1))/), if_col) + ! read ystore + call h5_read_dataset_collect_hyperslab_in_group("ystore", ystore, (/sum(offset_nnodes(0:myrank-1))/), if_col) + ! read zstore + call h5_read_dataset_collect_hyperslab_in_group("zstore", zstore, (/sum(offset_nnodes(0:myrank-1))/), if_col) + ! ibool + call h5_read_dataset_collect_hyperslab_in_group("ibool", ibool, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! idoubling + call h5_read_dataset_collect_hyperslab_in_group("idoubling", idoubling, (/sum(offset_nelems(0:myrank-1))/), if_col) + ! ispec_is_tiso + call h5_read_dataset_collect_hyperslab_in_group("ispec_is_tiso", ispec_is_tiso, (/sum(offset_nelems(0:myrank-1))/), if_col) + ! xix + call h5_read_dataset_collect_hyperslab_in_group("xixstore", xix, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! xiy + call h5_read_dataset_collect_hyperslab_in_group("xiystore", xiy, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! xiz + call h5_read_dataset_collect_hyperslab_in_group("xizstore", xiz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! etax + call h5_read_dataset_collect_hyperslab_in_group("etaxstore", etax, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! etay + call h5_read_dataset_collect_hyperslab_in_group("etaystore", etay, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! etaz + call h5_read_dataset_collect_hyperslab_in_group("etazstore", etaz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! gammax + call h5_read_dataset_collect_hyperslab_in_group("gammaxstore", gammax, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! gammay + call h5_read_dataset_collect_hyperslab_in_group("gammaystore", gammay, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! gammaz + call h5_read_dataset_collect_hyperslab_in_group("gammazstore", gammaz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + + if (iregion_code == IREGION_TRINFINITE .or. iregion_code == IREGION_INFINITE) then + call h5_close_group() + call h5_close_file() + return + endif + + ! rhostore + call h5_read_dataset_collect_hyperslab_in_group("rhostore", rhostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! kappavstore + call h5_read_dataset_collect_hyperslab_in_group("kappavstore", kappavstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + + if (iregion_code /= IREGION_OUTER_CORE) then + ! muvstore + call h5_read_dataset_collect_hyperslab_in_group("muvstore", muvstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + endif + + select case (iregion_code) + case (IREGION_CRUST_MANTLE) + ! crust/mantle + if (ANISOTROPIC_3D_MANTLE_VAL) then + ! c11store + call h5_read_dataset_collect_hyperslab_in_group("c11store", c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c12store + call h5_read_dataset_collect_hyperslab_in_group("c12store", c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c13store + call h5_read_dataset_collect_hyperslab_in_group("c13store", c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c14store + call h5_read_dataset_collect_hyperslab_in_group("c14store", c14store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c15store + call h5_read_dataset_collect_hyperslab_in_group("c15store", c15store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c16store + call h5_read_dataset_collect_hyperslab_in_group("c16store", c16store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c22store + call h5_read_dataset_collect_hyperslab_in_group("c22store", c22store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c23store + call h5_read_dataset_collect_hyperslab_in_group("c23store", c23store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c24store + call h5_read_dataset_collect_hyperslab_in_group("c24store", c24store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c25store + call h5_read_dataset_collect_hyperslab_in_group("c25store", c25store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c26store + call h5_read_dataset_collect_hyperslab_in_group("c26store", c26store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c33store + call h5_read_dataset_collect_hyperslab_in_group("c33store", c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c34store + call h5_read_dataset_collect_hyperslab_in_group("c34store", c34store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c35store + call h5_read_dataset_collect_hyperslab_in_group("c35store", c35store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c36store + call h5_read_dataset_collect_hyperslab_in_group("c36store", c36store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c44store + call h5_read_dataset_collect_hyperslab_in_group("c44store", c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c45store + call h5_read_dataset_collect_hyperslab_in_group("c45store", c45store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c46store + call h5_read_dataset_collect_hyperslab_in_group("c46store", c46store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c55store + call h5_read_dataset_collect_hyperslab_in_group("c55store", c55store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c56store + call h5_read_dataset_collect_hyperslab_in_group("c56store", c56store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c66store + call h5_read_dataset_collect_hyperslab_in_group("c66store", c66store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + else + if (TRANSVERSE_ISOTROPY_VAL) then + ! kappahstore + call h5_read_dataset_collect_hyperslab_in_group("kappahstore", kappahstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! muhstore + call h5_read_dataset_collect_hyperslab_in_group("muhstore", muhstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! eta_anisostore + call h5_read_dataset_collect_hyperslab_in_group("eta_anisostore", eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + endif + endif + + ! mu0store + call h5_read_dataset_collect_hyperslab_in_group("mu0store", mu0store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + + case (IREGION_INNER_CORE) + ! inner core + if (ANISOTROPIC_INNER_CORE_VAL) then + ! c11store + call h5_read_dataset_collect_hyperslab_in_group("c11store", c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c12store + call h5_read_dataset_collect_hyperslab_in_group("c12store", c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c13store + call h5_read_dataset_collect_hyperslab_in_group("c13store", c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c33store + call h5_read_dataset_collect_hyperslab_in_group("c33store", c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! c44store + call h5_read_dataset_collect_hyperslab_in_group("c44store", c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + endif + end select + + ! Stacey + if (ABSORBING_CONDITIONS) then + if (iregion_code == IREGION_CRUST_MANTLE) then + ! rho_vp + call h5_read_dataset_collect_hyperslab_in_group("rho_vp", rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! rho_vs + call h5_read_dataset_collect_hyperslab_in_group("rho_vs", rho_vs, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + else if (iregion_code == IREGION_OUTER_CORE) then + ! rho_vp + call h5_read_dataset_collect_hyperslab_in_group("rho_vp", rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + endif + endif + + ! mass matrices + if (((NCHUNKS_VAL /= 6 .and. ABSORBING_CONDITIONS) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_INNER_CORE)) then + ! rmassx + call h5_read_dataset_collect_hyperslab_in_group("rmassx", rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), if_col) + ! rmassy + call h5_read_dataset_collect_hyperslab_in_group("rmassy", rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), if_col) + endif + + ! rmassz + call h5_read_dataset_collect_hyperslab_in_group("rmassz", rmassz, (/sum(offset_nnodes(0:myrank-1))/), if_col) + + if (((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_INNER_CORE)) then + ! b_rmassx + call h5_read_dataset_collect_hyperslab_in_group("b_rmassx", b_rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), if_col) + ! b_rmassy + call h5_read_dataset_collect_hyperslab_in_group("b_rmassy", b_rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), if_col) + endif + + ! read additional ocean load mass matrix + if (OCEANS_VAL .and. iregion_code == IREGION_CRUST_MANTLE) then + ! rmass_ocean_load + call h5_read_dataset_collect_hyperslab_in_group("rmass_ocean_load", rmass_ocean_load, (/sum(offset_nnodes_oceans(0:myrank-1))/), if_col) + endif + + ! close group + call h5_close_group() + ! close file + call h5_close_file_p() + +#else + print* + print*,'ERROR: HDF5 support not enabled' + print*, 'Please recompile with HDF5 support with the --with-hdf5 option' + print* + stop +#endif + + +end subroutine read_arrays_solver_hdf5 + + +subroutine read_mesh_databases_MPI_hdf5(iregion_code) + + use specfem_par + use specfem_par_crustmantle + use specfem_par_outercore + use specfem_par_innercore + + use specfem_par_trinfinite + use specfem_par_infinite + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + integer, intent(in) :: iregion_code + + ! local parameters + integer :: ierr,rank,i + ! ADIOS variables + integer(kind=8) :: sel + integer(kind=8), dimension(1) :: start, count + + integer :: num_interfaces,max_nibool_interfaces, & + num_phase_ispec,num_colors_outer,num_colors_inner + integer :: nspec_inner,nspec_outer + + integer(kind=8) :: offset_my_neighbors, offset_nibool_interfaces, & + offset_ibool_interfaces, offset_phase_ispec_inner, & + offset_num_elem_colors + + ! temporary read arrays + !integer, dimension(:),allocatable :: tmp_nibool_interfaces,tmp_my_neighbors + !integer, dimension(:,:),allocatable :: tmp_ibool_interfaces,tmp_phase_ispec_inner + !integer, dimension(:),allocatable :: tmp_num_elem_colors + + character(len=128) :: region_name + integer :: nglob_tmp,nspec_tmp + + ! group, dataset name + character(len=64) :: gname_region + ! MPI variables + integer :: info, comm + ! if collective read + logical :: if_col = .true. + + ! offset arrays + integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_interfaces + integer, dimension(0:NPROCTOT_VAL-1) :: offset_max_nibool_interfaces + integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_phase_ispec + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_inner + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_outer + integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_colors_outer + integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_colors_inner + +#ifdef USE_HDF5 + + call h5_initialize() + + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/solver_data_mpi.h5' + write(gname_region, "('reg',i1)") iregion_code + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the hdf5 file + call h5_open_file_p_collect(name_database_hdf5) + ! open the group + call h5_open_group(gname_region) + + call h5_read_dataset_collect_hyperslab_in_group('offset_num_interfaces', offset_num_interfaces, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group('offset_max_nibool_interfaces', offset_max_nibool_interfaces, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group('offset_num_phase_ispec', offset_num_phase_ispec, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group('offset_nspec_inner', offset_nspec_inner, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group('offset_nspec_outer', offset_nspec_outer, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group('offset_num_colors_outer', offset_num_colors_outer, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group('offset_num_colors_inner', offset_num_colors_inner, (/0/), if_col) + + num_interfaces = offset_num_interfaces(myrank) + max_nibool_interfaces = offset_max_nibool_interfaces(myrank) + num_phase_ispec = offset_num_phase_ispec(myrank) + nspec_inner = offset_nspec_inner(myrank) + nspec_outer = offset_nspec_outer(myrank) + num_colors_outer = offset_num_colors_outer(myrank) + num_colors_inner = offset_num_colors_inner(myrank) + + select case(iregion_code) + case (IREGION_CRUST_MANTLE) + + num_interfaces_crust_mantle = num_interfaces + max_nibool_interfaces_cm = max_nibool_interfaces + num_phase_ispec_crust_mantle = num_phase_ispec + nspec_inner_crust_mantle = nspec_inner + nspec_outer_crust_mantle = nspec_outer + num_colors_outer_crust_mantle = num_colors_outer + num_colors_inner_crust_mantle = num_colors_inner + + allocate(my_neighbors_crust_mantle(num_interfaces), & + nibool_interfaces_crust_mantle(num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating my_neighbors_crust_mantle etc.') + if (num_interfaces > 0) then + call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & + my_neighbors_crust_mantle(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & + nibool_interfaces_crust_mantle(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + end if + + allocate(ibool_interfaces_crust_mantle(max_nibool_interfaces,num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating ibool_interfaces_crust_mantle etc.') + if (num_interfaces > 0) then + ibool_interfaces_crust_mantle(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & + ibool_interfaces_crust_mantle(1:max_nibool_interfaces,1:num_interfaces), & + (/0,sum(offset_num_interfaces(0:myrank-1))/), if_col) + end if + + allocate(phase_ispec_inner_crust_mantle(num_phase_ispec,2), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating phase_ispec_inner_crust_mantle etc.') + if (num_phase_ispec > 0) then + phase_ispec_inner_crust_mantle(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & + phase_ispec_inner_crust_mantle(1:num_phase_ispec,1:2), & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_col) + end if + + allocate(num_elem_colors_crust_mantle(num_colors_outer+num_colors_inner), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating num_elem_colors_crust_mantle etc.') + if (USE_MESH_COLORING_GPU) then + call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & + num_elem_colors_crust_mantle(1:(num_colors_outer+num_colors_inner)), & + (/sum(offset_num_colors_outer(0:myrank-1))/), if_col) + end if + + case (IREGION_OUTER_CORE) + + num_interfaces_outer_core = num_interfaces + max_nibool_interfaces_oc = max_nibool_interfaces + num_phase_ispec_outer_core = num_phase_ispec + nspec_inner_outer_core = nspec_inner + nspec_outer_outer_core = nspec_outer + num_colors_outer_outer_core = num_colors_outer + num_colors_inner_outer_core = num_colors_inner + + allocate(my_neighbors_outer_core(num_interfaces), & + nibool_interfaces_outer_core(num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating my_neighbors_outer_core etc.') + if (num_interfaces > 0) then + call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & + my_neighbors_outer_core(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & + nibool_interfaces_outer_core(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + end if + + allocate(ibool_interfaces_outer_core(max_nibool_interfaces,num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating ibool_interfaces_outer_core etc.') + if (num_interfaces > 0) then + ibool_interfaces_outer_core(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & + ibool_interfaces_outer_core(1:max_nibool_interfaces,1:num_interfaces), & + (/0,sum(offset_num_interfaces(0:myrank-1))/), if_col) + end if + + allocate(phase_ispec_inner_outer_core(num_phase_ispec,2), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating phase_ispec_inner_outer_core etc.') + if (num_phase_ispec > 0) then + phase_ispec_inner_outer_core(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & + phase_ispec_inner_outer_core(1:num_phase_ispec,1:2), & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_col) + end if + + allocate(num_elem_colors_outer_core(num_colors_outer+num_colors_inner), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating num_elem_colors_outer_core etc.') + if (USE_MESH_COLORING_GPU) then + call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & + num_elem_colors_outer_core(1:(num_colors_outer+num_colors_inner)), & + (/sum(offset_num_colors_outer(0:myrank-1))/), if_col) + end if + + case (IREGION_INNER_CORE) + + num_interfaces_inner_core = num_interfaces + max_nibool_interfaces_ic = max_nibool_interfaces + num_phase_ispec_inner_core = num_phase_ispec + nspec_inner_inner_core = nspec_inner + nspec_outer_inner_core = nspec_outer + num_colors_outer_inner_core = num_colors_outer + num_colors_inner_inner_core = num_colors_inner + + allocate(my_neighbors_inner_core(num_interfaces), & + nibool_interfaces_inner_core(num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating my_neighbors_inner_core etc.') + if (num_interfaces > 0) then + call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & + my_neighbors_inner_core(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & + nibool_interfaces_inner_core(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + end if + + allocate(ibool_interfaces_inner_core(max_nibool_interfaces,num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating ibool_interfaces_inner_core etc.') + if (num_interfaces > 0) then + ibool_interfaces_inner_core(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & + ibool_interfaces_inner_core(1:max_nibool_interfaces,1:num_interfaces), & + (/0,sum(offset_num_interfaces(0:myrank-1))/), if_col) + end if + + allocate(phase_ispec_inner_inner_core(num_phase_ispec,2), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating phase_ispec_inner_inner_core etc.') + if (num_phase_ispec > 0) then + phase_ispec_inner_inner_core(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & + phase_ispec_inner_inner_core(1:num_phase_ispec,1:2), & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_col) + end if + + allocate(num_elem_colors_inner_core(num_colors_outer+num_colors_inner), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating num_elem_colors_inner_core etc.') + if (USE_MESH_COLORING_GPU) then + call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & + num_elem_colors_inner_core(1:(num_colors_outer+num_colors_inner)), & + (/sum(offset_num_colors_outer(0:myrank-1))/), if_col) + end if + + case (IREGION_TRINFINITE) + + num_interfaces_trinfinite = num_interfaces + max_nibool_interfaces_trinfinite = max_nibool_interfaces + num_phase_ispec_trinfinite = num_phase_ispec + nspec_inner_trinfinite = nspec_inner + nspec_outer_trinfinite = nspec_outer + num_colors_outer_trinfinite = num_colors_outer + num_colors_inner_trinfinite = num_colors_inner + + allocate(my_neighbors_trinfinite(num_interfaces), & + nibool_interfaces_trinfinite(num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating my_neighbors_trinfinite etc.') + if (num_interfaces > 0) then + call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & + my_neighbors_trinfinite(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & + nibool_interfaces_trinfinite(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + end if + + allocate(ibool_interfaces_trinfinite(max_nibool_interfaces,num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating ibool_interfaces_trinfinite etc.') + if (num_interfaces > 0) then + ibool_interfaces_trinfinite(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & + ibool_interfaces_trinfinite(1:max_nibool_interfaces,1:num_interfaces), & + (/0,sum(offset_num_interfaces(0:myrank-1))/), if_col) + end if + + allocate(phase_ispec_inner_trinfinite(num_phase_ispec,2), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating phase_ispec_inner_trinfinite etc.') + if (num_phase_ispec > 0) then + phase_ispec_inner_trinfinite(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & + phase_ispec_inner_trinfinite(1:num_phase_ispec,1:2), & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_col) + end if + + allocate(num_elem_colors_trinfinite(num_colors_outer+num_colors_inner), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating num_elem_colors_trinfinite etc.') + if (USE_MESH_COLORING_GPU) then + call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & + num_elem_colors_trinfinite(1:(num_colors_outer+num_colors_inner)), & + (/sum(offset_num_colors_outer(0:myrank-1))/), if_col) + end if + + case (IREGION_INFINITE) + + num_interfaces_infinite = num_interfaces + max_nibool_interfaces_infinite = max_nibool_interfaces + num_phase_ispec_infinite = num_phase_ispec + nspec_inner_infinite = nspec_inner + nspec_outer_infinite = nspec_outer + num_colors_outer_infinite = num_colors_outer + num_colors_inner_infinite = num_colors_inner + + allocate(my_neighbors_infinite(num_interfaces), & + nibool_interfaces_infinite(num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating my_neighbors_infinite etc.') + if (num_interfaces > 0) then + call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & + my_neighbors_infinite(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & + nibool_interfaces_infinite(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + end if + + allocate(ibool_interfaces_infinite(max_nibool_interfaces,num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating ibool_interfaces_infinite etc.') + if (num_interfaces > 0) then + ibool_interfaces_infinite(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & + ibool_interfaces_infinite(1:max_nibool_interfaces,1:num_interfaces), & + (/0,sum(offset_num_interfaces(0:myrank-1))/), if_col) + end if + + allocate(phase_ispec_inner_infinite(num_phase_ispec,2), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating phase_ispec_inner_infinite etc.') + if (num_phase_ispec > 0) then + phase_ispec_inner_infinite(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & + phase_ispec_inner_infinite(1:num_phase_ispec,1:2), & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_col) + end if + + allocate(num_elem_colors_infinite(num_colors_outer+num_colors_inner), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating num_elem_colors_infinite etc.') + if (USE_MESH_COLORING_GPU) then + call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & + num_elem_colors_infinite(1:(num_colors_outer+num_colors_inner)), & + (/sum(offset_num_colors_outer(0:myrank-1))/), if_col) + end if + + case default + print*, 'ERROR: unknown region code' + stop + end select + + ! close group + call h5_close_group() + ! close file + call h5_close_file_p() + +#else + print* + print*,'ERROR: HDF5 support not enabled' + print*, 'Please recompile with HDF5 support with the --with-hdf5 option' + print* + stop +#endif + +end subroutine read_mesh_databases_MPI_hdf5 + + + +subroutine read_mesh_databases_coupling_hdf5() + + use constants + + use meshfem_par, only: & + myrank, LOCAL_PATH + + !use meshfem_models_par, only: & + ! HONOR_1D_SPHERICAL_MOHO + !SAVE_BOUNDARY_MESH,HONOR_1D_SPHERICAL_MOHO,SUPPRESS_CRUSTAL_MESH + + !use regions_mesh_par2, only: & + ! NSPEC2D_MOHO, NSPEC2D_400, NSPEC2D_670, & + ! ibelm_moho_top,ibelm_moho_bot,ibelm_400_top,ibelm_400_bot, & + ! ibelm_670_top,ibelm_670_bot,normal_moho,normal_400,normal_670, & + ! ispec2D_moho_top,ispec2D_moho_bot,ispec2D_400_top,ispec2D_400_bot, & + ! ispec2D_670_top,ispec2D_670_bot ! prname + + use shared_parameters, only: FULL_GRAVITY + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + + use specfem_par_trinfinite + use specfem_par_infinite + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + + character(len=64) :: gname_region + integer :: ierr + logical :: if_col = .true. + + ! MPI variables + integer :: info, comm + + ! offset arrays + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_xmin + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_xmax + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_ymin + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_ymax + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_top + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_bottom + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_moho_top + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_moho_bottom + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_400_top + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_400_bottom + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_670_top + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_670_bottom + !integer, dimension(0:NPROCTOT_VAL-1) :: tmp_array + + + ! dump integers + integer :: tmp_nspec2d_moho, tmp_nspec2d_400, tmp_nspec2d_670 + + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize hdf5 + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the hdf5 file + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/boundary.h5' + call h5_open_file_p_collect(name_database_hdf5) + + + if (NSPEC_CRUST_MANTLE > 0) then + ! open the group + write(gname_region, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_open_group(gname_region) + + ! read actual number of elements + ! nspec2D_xmin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_crust_mantle, (/myrank/), if_col) + ! nspec2D_xmax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_crust_mantle, (/myrank/), if_col) + ! nspec2D_ymin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_crust_mantle, (/myrank/), if_col) + ! nspec2D_ymax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_crust_mantle, (/myrank/), if_col) + + ! read offset arrays (stored length) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), if_col) + + ! ibelm_xmin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_crust_mantle, (/sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + ! ibelm_xmax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_crust_mantle, (/sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + ! ibelm_ymin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_crust_mantle, (/sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + ! ibelm_ymax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_crust_mantle, (/sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + ! ibelm_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_crust_mantle, (/sum(offset_nspec2D_top(0:myrank-1))/), if_col) + ! ibelm_bottom + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_crust_mantle, (/sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + + ! normal_xmin + call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_crust_mantle, (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + ! normal_xmax + call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_crust_mantle, (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + ! normal_ymin + call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_crust_mantle, (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + ! normal_ymax + call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_crust_mantle, (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + ! normal_top + call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_crust_mantle, (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), if_col) + ! normal_bottom + call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_crust_mantle, (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + + ! jacobian_xmin + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_crust_mantle, (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + ! jacobian_xmax + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_crust_mantle, (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + ! jacobian_ymin + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_crust_mantle, (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + ! jacobian_ymax + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_crust_mantle, (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + ! jacobian_top + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_crust_mantle, (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), if_col) + ! jacobian_bottom + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_crust_mantle, (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + + ! close group + call h5_close_group() + end if ! NSPEC_CRUST_MANTLE > 0 + + if (NSPEC_OUTER_CORE > 0) then + ! change group name + write(gname_region, "('reg',i1)") IREGION_OUTER_CORE + call h5_open_group(gname_region) + + ! read actual number of elements + ! nspec2D_xmin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_outer_core, (/myrank/), if_col) + ! nspec2D_xmax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_outer_core, (/myrank/), if_col) + ! nspec2D_ymin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_outer_core, (/myrank/), if_col) + ! nspec2D_ymax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_outer_core, (/myrank/), if_col) + + ! read offset arrays (stored length) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), if_col) + + ! ibelm_xmin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_outer_core, (/sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + ! ibelm_xmax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_outer_core, (/sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + ! ibelm_ymin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_outer_core, (/sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + ! ibelm_ymax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_outer_core, (/sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + ! ibelm_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_outer_core, (/sum(offset_nspec2D_top(0:myrank-1))/), if_col) + ! ibelm_bottom + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_outer_core, (/sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + + ! normal_xmin + call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_outer_core, (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + ! normal_xmax + call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_outer_core, (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + ! normal_ymin + call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_outer_core, (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + ! normal_ymax + call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_outer_core, (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + ! normal_top + call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_outer_core, (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), if_col) + ! normal_bot + call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_outer_core, (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + + ! jacobian_xmin + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_outer_core, (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + ! jacobian_xmax + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_outer_core, (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + ! jacobian_ymin + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_outer_core, (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + ! jacobian_ymax + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_outer_core, (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + ! jacobian_top + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_outer_core, (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), if_col) + ! jacobian_bottom + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_outer_core, (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + + ! close group + call h5_close_group() + end if ! NSPEC_OUTER_CORE > 0 + + if (NSPEC_INNER_CORE > 0) then + + ! change group name + write(gname_region, "('reg',i1)") IREGION_INNER_CORE + call h5_open_group(gname_region) + + ! read actual number of elements + ! nspec2D_xmin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_inner_core, (/myrank/), if_col) + ! nspec2D_xmax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_inner_core, (/myrank/), if_col) + ! nspec2D_ymin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_inner_core, (/myrank/), if_col) + ! nspec2D_ymax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_inner_core, (/myrank/), if_col) + + ! read offset arrays (stored length) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), if_col) + + ! ibelm_xmin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_inner_core, (/sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + ! ibelm_xmax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_inner_core, (/sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + ! ibelm_ymin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_inner_core, (/sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + ! ibelm_ymax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_inner_core, (/sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + + ! close group + call h5_close_group() + end if ! NSPEC_INNER_CORE > 0 + + if (FULL_GRAVITY) then + if (ADD_TRINF) then + if (NSPEC_TRINFINITE > 0) then + ! change group name + write(gname_region, "('reg',i1)") IREGION_TRINFINITE + call h5_open_group(gname_region) + + ! read actual number of elements + ! nspec2D_xmin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_trinfinite, (/myrank/), if_col) + ! nspec2D_xmax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_trinfinite, (/myrank/), if_col) + ! nspec2D_ymin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_trinfinite, (/myrank/), if_col) + ! nspec2D_ymax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_trinfinite, (/myrank/), if_col) + + ! read offset arrays (stored length) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), if_col) + + ! ibelm_xmin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_trinfinite, (/sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + ! ibelm_xmax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_trinfinite, (/sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + ! ibelm_ymin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_trinfinite, (/sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + ! ibelm_ymax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_trinfinite, (/sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + ! ibelm_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_trinfinite, (/sum(offset_nspec2D_top(0:myrank-1))/), if_col) + ! ibelm_bottom + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_trinfinite, (/sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + + ! close group + call h5_close_group() + end if ! NSPEC_TRINFINITE > 0 + end if ! ADD_TRINF + + if (NSPEC_INFINITE > 0) then + ! change group name + write(gname_region, "('reg',i1)") IREGION_INFINITE + call h5_open_group(gname_region) + + ! read actual number of elements + ! nspec2D_xmin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_infinite, (/myrank/), if_col) + ! nspec2D_xmax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_infinite, (/myrank/), if_col) + ! nspec2D_ymin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_infinite, (/myrank/), if_col) + ! nspec2D_ymax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_infinite, (/myrank/), if_col) + + ! read offset arrays (stored length) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), if_col) + + ! ibelm_xmin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_infinite, (/sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + ! ibelm_xmax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_infinite, (/sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + ! ibelm_ymin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_infinite, (/sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + ! ibelm_ymax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_infinite, (/sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + ! ibelm_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_infinite, (/sum(offset_nspec2D_top(0:myrank-1))/), if_col) + ! ibelm_bottom + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_infinite, (/sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + + ! close group + call h5_close_group() + end if ! NSPEC_INFINITE > 0 + end if ! FULL_GRAVITY + + ! close file + call h5_close_file_p() + + ! boundary mesh for crust and mantle + if (SAVE_BOUNDARY_MESH .and. SIMULATION_TYPE == 3) then + ! open boundary_disc.h5 file + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/boundary_disc.h5' + call h5_open_file_p_collect(name_database_hdf5) + + if (NSPEC_CRUST_MANTLE > 0) then + write(gname_region, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_open_group(gname_region) + + ! read NSPEC2D_MOHO + call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_MOHO", tmp_nspec2d_moho, (/myrank/), if_col) + ! read NSPEC2D_400 + call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_400", tmp_nspec2d_400, (/myrank/), if_col) + ! read NSPEC2D_670 + call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_670", tmp_nspec2d_670, (/myrank/), if_col) + + ! checks setup + if (tmp_nspec2d_moho /= NSPEC2D_MOHO .or. tmp_nspec2d_400 /= NSPEC2D_400 .or. tmp_nspec2d_670 /= NSPEC2D_670) then + print *,'Error: invalid NSPEC2D values read in for solver: ',tmp_nspec2d_moho,tmp_nspec2d_400,tmp_nspec2d_670,'(boundary_disc.h5)' + print *,' should be MOHO/400/670 : ',NSPEC2D_MOHO,NSPEC2D_400,NSPEC2D_670,'(mesh_parameters.h5)' + call exit_mpi(myrank, 'Error reading boundary_disc.h5 file') + endif + + ! read the actual number of elements + ! sub_NSPEC2D_MOHO_top + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_MOHO_top", offset_nspec2D_moho_top, (/0/), if_col) + ! sub_NSPEC2D_MOHO_bottom + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_MOHO_bottom", offset_nspec2D_moho_bottom, (/0/), if_col) + ! sub_NSPEC2D_400_top + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_400_top", offset_nspec2D_400_top, (/0/), if_col) + ! sub_NSPEC2D_400_bottom + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_400_bottom", offset_nspec2D_400_bottom, (/0/), if_col) + ! sub_NSPEC2D_670_top + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_670_top", offset_nspec2D_670_top, (/0/), if_col) + ! sub_NSPEC2D_670_bottom + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_670_bot", offset_nspec2D_670_bottom, (/0/), if_col) + + ! ibelm_moho_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_top", ibelm_moho_top, (/sum(offset_nspec2D_moho_top(0:myrank-1))/), if_col) + ! ibelm_moho_bot + call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_bot", ibelm_moho_bot, (/sum(offset_nspec2D_moho_bottom(0:myrank-1))/), if_col) + ! ibelm_400_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_top", ibelm_400_top, (/sum(offset_nspec2D_400_top(0:myrank-1))/), if_col) + ! ibelm_400_bot + call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_bot", ibelm_400_bot, (/sum(offset_nspec2D_400_bottom(0:myrank-1))/), if_col) + ! ibelm_670_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_top", ibelm_670_top, (/sum(offset_nspec2D_670_top(0:myrank-1))/), if_col) + ! ibelm_670_bot + call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_bot", ibelm_670_bot, (/sum(offset_nspec2D_670_bottom(0:myrank-1))/), if_col) + ! normal_moho + call h5_read_dataset_collect_hyperslab_in_group("normal_moho", normal_moho, (/0,0,0,sum(offset_nspec2D_moho_top(0:myrank-1))/), if_col) + ! normal_400 + call h5_read_dataset_collect_hyperslab_in_group("normal_400", normal_400, (/0,0,0,sum(offset_nspec2D_400_top(0:myrank-1))/), if_col) + ! normal_670 + call h5_read_dataset_collect_hyperslab_in_group("normal_670", normal_670, (/0,0,0,sum(offset_nspec2D_670_top(0:myrank-1))/), if_col) + + ! close group + call h5_close_group() + end if ! NSPEC_CRUST_MANTLE > 0 + end if ! SAVE_BOUNDARY_MESH .and. SIMULATION_TYPE == 3 + + + + +#else + print* + print*,'ERROR: HDF5 support not enabled' + print*, 'Please recompile with HDF5 support with the --with-hdf5 option' + print* + stop +#endif + +end subroutine read_mesh_databases_coupling_hdf5 + + +subroutine read_mesh_databases_stacey_hdf5 + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + + ! offset + integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_abs_boundary_faces + + ! hdf5 variables + integer :: comm, info, ier + character(len=64) :: gname_region + logical :: if_col = .true. ! read in collective mode + + + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize hdf5 + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! name of the file and group + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/stacey.h5' + + ! open the hdf5 file + call h5_open_file_p_collect(name_database_hdf5) + + if (NSPEC_CRUST_MANTLE > 0) then + + ! open the group + write(gname_region, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_open_group(gname_region) + + ! read the offset_num_abs_boundary_faces + call h5_read_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", offset_num_abs_boundary_faces, (/0/), if_col) + ! for this process + num_abs_boundary_faces_crust_mantle = offset_num_abs_boundary_faces(myrank) + + if (num_abs_boundary_faces_crust_mantle > 0) then + ! allocates absorbing boundary arrays + allocate(abs_boundary_ispec_crust_mantle(num_abs_boundary_faces_crust_mantle),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ispec') + allocate(abs_boundary_ijk_crust_mantle(3,NGLLSQUARE,num_abs_boundary_faces_crust_mantle),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ijk') + allocate(abs_boundary_jacobian2Dw_crust_mantle(NGLLSQUARE,num_abs_boundary_faces_crust_mantle),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_jacobian2Dw') + allocate(abs_boundary_normal_crust_mantle(NDIM,NGLLSQUARE,num_abs_boundary_faces_crust_mantle),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_normal') + allocate(abs_boundary_npoin_crust_mantle(num_abs_boundary_faces_crust_mantle),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_npoin') + if (ier /= 0) stop 'Error allocating array abs_boundary_ispec etc.' + + ! abs_boundary_ispec + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_crust_mantle, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + ! abs_boundary_npoin + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_crust_mantle, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + ! abs_boundary_ijk + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_crust_mantle, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + ! abs_boundary_jacobian2Dw + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_crust_mantle, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + ! abs_boundary_normal + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_crust_mantle, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + + else ! dummy + ! dummy arrays + allocate(abs_boundary_ispec_crust_mantle(1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ispec') + allocate(abs_boundary_ijk_crust_mantle(1,1,1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ijk') + allocate(abs_boundary_jacobian2Dw_crust_mantle(1,1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_jacobian2Dw') + allocate(abs_boundary_normal_crust_mantle(1,1,1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_normal') + allocate(abs_boundary_npoin_crust_mantle(1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_npoin') + abs_boundary_ispec_crust_mantle(:) = 0; abs_boundary_npoin_crust_mantle(:) = 0 + abs_boundary_ijk_crust_mantle(:,:,:) = 0 + abs_boundary_jacobian2Dw_crust_mantle(:,:) = 0.0; abs_boundary_normal_crust_mantle(:,:,:) = 0.0 + + endif ! num_abs_boundary_faces_crust_mantle > 0 + + ! close group + call h5_close_group() + end if ! NSPEC_CRUST_MANTLE > 0 + + if (NSPEC_OUTER_CORE > 0) then + ! open the group + write(gname_region, "('reg',i1)") IREGION_OUTER_CORE + call h5_open_group(gname_region) + + ! read the offset_num_abs_boundary_faces + call h5_read_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", offset_num_abs_boundary_faces, (/0/), if_col) + ! for this process + num_abs_boundary_faces_outer_core = offset_num_abs_boundary_faces(myrank) + + if (num_abs_boundary_faces_outer_core > 0) then + ! allocates absorbing boundary arrays + allocate(abs_boundary_ispec_outer_core(num_abs_boundary_faces_outer_core),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ispec') + allocate(abs_boundary_ijk_outer_core(3,NGLLSQUARE,num_abs_boundary_faces_outer_core),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ijk') + allocate(abs_boundary_jacobian2Dw_outer_core(NGLLSQUARE,num_abs_boundary_faces_outer_core),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_jacobian2Dw') + !allocate(abs_boundary_normal_outer_core(NDIM,NGLLSQUARE,num_abs_boundary_faces_outer_core),stat=ier) + !if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_normal') + allocate(abs_boundary_npoin_outer_core(num_abs_boundary_faces_outer_core),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_npoin') + if (ier /= 0) stop 'Error allocating array abs_boundary_ispec etc.' + + ! abs_boundary_ispec + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_outer_core, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + ! abs_boundary_npoin + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_outer_core, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + ! abs_boundary_ijk + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_outer_core, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + ! abs_boundary_jacobian2Dw + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_outer_core, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + ! abs_boundary_normal + !call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_outer_core, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + + else ! dummy + ! dummy arrays + allocate(abs_boundary_ispec_outer_core(1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ispec') + allocate(abs_boundary_ijk_outer_core(1,1,1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ijk') + allocate(abs_boundary_jacobian2Dw_outer_core(1,1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_jacobian2Dw') + !allocate(abs_boundary_normal_outer_core(1,1,1),stat=ier) + !if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_normal') + allocate(abs_boundary_npoin_outer_core(1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_npoin') + abs_boundary_ispec_outer_core(:) = 0 + abs_boundary_npoin_outer_core(:) = 0 + abs_boundary_ijk_outer_core(:,:,:) = 0 + abs_boundary_jacobian2Dw_outer_core(:,:) = 0.0 + !abs_boundary_normal_outer_core(:,:,:) = 0.0 + + endif ! num_abs_boundary_faces_outer_core > 0 + + ! close group + call h5_close_group() + end if ! NSPEC_OUTER_CORE > 0 + + ! close file + call h5_close_file_p() + + +#else + print* + print*,'ERROR: HDF5 support not enabled' + print*, 'Please recompile with HDF5 support with the --with-hdf5 option' + print* + stop +#endif + +end subroutine read_mesh_databases_stacey_hdf5 + + +subroutine read_attenuation_hdf5(iregion_code, factor_common, scale_factor, tau_s, vnspec, f_c_source) + + use constants_solver + use specfem_par, only: ATTENUATION_VAL,LOCAL_PATH + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + integer,intent(in) :: iregion_code + ! offset + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nelems + + integer,intent(in) :: vnspec + real(kind=CUSTOM_REAL), dimension(ATT1_VAL,ATT2_VAL,ATT3_VAL,vnspec),intent(inout) :: scale_factor + real(kind=CUSTOM_REAL), dimension(ATT1_VAL,ATT2_VAL,ATT3_VAL,N_SLS,vnspec),intent(inout) :: factor_common + double precision, dimension(N_SLS),intent(inout) :: tau_s + double precision,intent(inout) :: f_c_source + +#ifdef USE_HDF5 + ! hdf5 variables + logical :: if_col = .true. ! read in collective mode + character(len=64) :: gname_region + integer :: comm, info + double precision, dimension(0:NPROCTOT_VAL-1) :: tmp_dp_arr + +#endif + + if (.not. ATTENUATION_VAL) return + +#ifdef USE_HDF5 + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize hdf5 + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! get offset_nelems from solver_data.h5 + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/solver_data.h5' + write(gname_region, "('reg',i1)") iregion_code + ! open the hdf5 file + call h5_open_file_p_collect(name_database_hdf5) + ! open the group + call h5_open_group(gname_region) + ! read the offset_nelems + call h5_read_dataset_collect_hyperslab_in_group("offset_nelems", offset_nelems, (/0/), if_col) + ! close group and file + call h5_close_group() + call h5_close_file_p() + + ! open attenuation.h5 file + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/attenuation.h5' + ! open the hdf5 file + call h5_open_file_p_collect(name_database_hdf5) + ! open the group + call h5_open_group(gname_region) + + ! tau_s + call h5_read_dataset_collect_hyperslab_in_group("tau_s_store", tau_s, (/myrank*N_SLS/), if_col) + ! factor_common (tau_e_store) + call h5_read_dataset_collect_hyperslab_in_group("tau_e_store", factor_common, (/0,0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! scale_factor (Qmu_store) + call h5_read_dataset_collect_hyperslab_in_group("Qmu_store", scale_factor, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + ! f_c_source + call h5_read_dataset_collect_hyperslab_in_group("att_f_c_source", tmp_dp_arr, (/0/), if_col) + f_c_source = tmp_dp_arr(myrank) + + ! close group + call h5_close_group() + ! close file + call h5_close_file_p() + +#else + print* + print*,'ERROR: HDF5 support not enabled' + print*, 'Please recompile with HDF5 support with the --with-hdf5 option' + print* + stop +#endif + +end subroutine read_attenuation_hdf5 \ No newline at end of file diff --git a/src/specfem3D/read_forward_arrays.F90 b/src/specfem3D/read_forward_arrays.F90 index 763fae92e..2d4153d6c 100644 --- a/src/specfem3D/read_forward_arrays.F90 +++ b/src/specfem3D/read_forward_arrays.F90 @@ -57,6 +57,8 @@ subroutine read_forward_arrays_startrun() if (ADIOS_FOR_FORWARD_ARRAYS) then call read_intermediate_forward_arrays_adios() + else if (HDF5_ENABLED) then + call read_intermediate_forward_arrays_hdf5(neq) else write(outputname,"('dump_all_arrays',i6.6)") myrank outputname = trim(LOCAL_TMP_PATH) // '/' // outputname(1:len_trim(outputname)) @@ -166,6 +168,8 @@ subroutine read_forward_arrays() ! reads in file data if (ADIOS_FOR_FORWARD_ARRAYS) then call read_forward_arrays_adios() + else if (HDF5_ENABLED) then + call read_forward_arrays_hdf5() else write(outputname,'(a,i6.6,a)') 'proc',myrank,'_save_forward_arrays.bin' outputname = trim(LOCAL_TMP_PATH) // '/' // outputname(1:len_trim(outputname)) @@ -316,6 +320,8 @@ subroutine read_forward_arrays_undoatt() if (ADIOS_FOR_UNDO_ATTENUATION) then call read_forward_arrays_undoatt_adios(iteration_on_subset_tmp) + else if (HDF5_ENABLED) then + call read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) else ! reads in saved wavefield write(outputname,'(a,i6.6,a,i6.6,a)') 'proc',myrank,'_save_frame_at',iteration_on_subset_tmp,'.bin' diff --git a/src/specfem3D/read_forward_arrays_hdf5.F90 b/src/specfem3D/read_forward_arrays_hdf5.F90 new file mode 100644 index 000000000..73c87f62b --- /dev/null +++ b/src/specfem3D/read_forward_arrays_hdf5.F90 @@ -0,0 +1,440 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + subroutine read_intermediate_forward_arrays_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! full gravity + integer :: neq_read,neq1_read + + ! MPI variables + integer :: ierr, info, comm + + ! collective write + logical :: if_col = .true. + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name, dataset_name + + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/dump_all_arrays.h5' + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the file + call h5_open_file_p_collect(file_name) + + ! read the arrays + call h5_read_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + + if (ROTATION_VAL) then + call h5_read_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('A_array_rotation', B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + endif + + if (ATTENUATION_VAL) then + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + endif + + if (FULL_GRAVITY_VAL) then + call h5_read_dataset_scalar_collect_hyperslab('neq', neq_read, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab('neq1', neq1_read, (/myrank/), if_col) + + ! check if array sizes match + if (neq_read /= neq) then + print *,'Error reading forward array for startrun: rank ',myrank,'has read neq =',neq_read,' - shoud be ',neq + call exit_MPI(myrank,'Invalid forward array neq for startrun') + endif + if (neq1_read /= neq1) then + print *,'Error reading forward array for startrun: rank ',myrank,'has read neq1 =',neq1_read,' - shoud be ',neq1 + call exit_MPI(myrank,'Invalid forward array neq1 for startrun') + endif + + call h5_read_dataset_collect_hyperslab('pgrav1', pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), if_col) + endif + + ! close the file + call h5_close_file_p() + +#else + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') +#endif + + end subroutine read_intermediate_forward_arrays_hdf5 + + + subroutine read_forward_arrays_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! full gravity + integer :: b_neq_read, b_neq1_read + + ! MPI variables + integer :: ierr, info, comm + + ! collective write + logical :: if_col = .true. + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name, dataset_name + + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/dump_all_arrays.h5' + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the file + call h5_open_file_p_collect(file_name) + + ! read the arrays + call h5_read_dataset_collect_hyperslab('displ_crust_mantle', b_displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('displ_inner_core', b_displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', b_veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('veloc_inner_core', b_veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + + if (ROTATION_VAL) then + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + endif + + if (ATTENUATION_VAL) then + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + endif + + if (FULL_GRAVITY_VAL) then + call h5_read_dataset_scalar_collect_hyperslab('neq', b_neq_read, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab('neq1', b_neq1_read, (/myrank/), if_col) + + ! check if array sizes match + if (b_neq_read /= neq) then + print *,'Error reading forward array for startrun: rank ',myrank,'has read neq =',b_neq_read,' - shoud be ',neq + call exit_MPI(myrank,'Invalid forward array neq for startrun') + endif + if (b_neq1_read /= neq1) then + print *,'Error reading forward array for startrun: rank ',myrank,'has read neq1 =',b_neq1_read,' - shoud be ',neq1 + call exit_MPI(myrank,'Invalid forward array neq1 for startrun') + endif + + call h5_read_dataset_collect_hyperslab('pgrav1', b_pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), if_col) + endif + + ! close the file + call h5_close_file_p() + +#else + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') +#endif + + end subroutine read_forward_arrays_hdf5 + + subroutine read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + ! input parameter + integer, intent(in) :: iteration_on_subset_tmp + +#ifdef USE_HDF5 + + ! full gravity + integer :: b_neq_read, b_neq1_read + + ! MPI variables + integer :: ierr, info, comm + + ! collective write + logical :: if_col = .true. + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name, dataset_name + + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + + write(file_name, '(a,i6.6,a)') 'save_frame_at',iteration_on_subset_tmp,'.h5' + file_name = trim(LOCAL_PATH)//'/'//trim(file_name) + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the file + call h5_open_file_p_collect(file_name) + + ! read the arrays + call h5_read_dataset_collect_hyperslab('displ_crust_mantle', b_displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('displ_inner_core', b_displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', b_veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('veloc_inner_core', b_veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + + if (ROTATION_VAL) then + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + endif + + if (ATTENUATION_VAL) then + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + endif + + if (FULL_GRAVITY_VAL) then + call h5_read_dataset_scalar_collect_hyperslab('neq', b_neq_read, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab('neq1', b_neq1_read, (/myrank/), if_col) + + ! check if array sizes match + if (b_neq_read /= neq) then + print *,'Error reading forward array for startrun: rank ',myrank,'has read neq =',b_neq_read,' - shoud be ',neq + call exit_MPI(myrank,'Invalid forward array neq for startrun') + endif + if (b_neq1_read /= neq1) then + print *,'Error reading forward array for startrun: rank ',myrank,'has read neq1 =',b_neq1_read,' - shoud be ',neq1 + call exit_MPI(myrank,'Invalid forward array neq1 for startrun') + endif + + call h5_read_dataset_collect_hyperslab('pgrav1', b_pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), if_col) + endif + + ! close the file + call h5_close_file_p() + +#else + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') +#endif + + end subroutine read_forward_arrays_undoatt_hdf5 diff --git a/src/specfem3D/read_mesh_databases.F90 b/src/specfem3D/read_mesh_databases.F90 index 18010bf77..4e307d1ee 100644 --- a/src/specfem3D/read_mesh_databases.F90 +++ b/src/specfem3D/read_mesh_databases.F90 @@ -387,7 +387,30 @@ subroutine read_mesh_databases_CM() rmassx_crust_mantle,rmassy_crust_mantle,rmassz_crust_mantle, & NGLOB_CRUST_MANTLE_OCEANS,rmass_ocean_load, & b_rmassx_crust_mantle,b_rmassy_crust_mantle) - else + else if (HDF5_ENABLED) then + call read_arrays_solver_hdf5(IREGION_CRUST_MANTLE, & + NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE,NGLOB_XY_CM, & + nspec_iso,nspec_tiso,nspec_ani, & + rho_vp_crust_mantle,rho_vs_crust_mantle, & + xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, & + xix_crust_mantle,xiy_crust_mantle,xiz_crust_mantle, & + etax_crust_mantle,etay_crust_mantle,etaz_crust_mantle, & + gammax_crust_mantle,gammay_crust_mantle,gammaz_crust_mantle, & + rhostore_crust_mantle,kappavstore_crust_mantle,muvstore_crust_mantle, & + kappahstore_crust_mantle,muhstore_crust_mantle,eta_anisostore_crust_mantle, & + c11store_crust_mantle,c12store_crust_mantle,c13store_crust_mantle, & + c14store_crust_mantle,c15store_crust_mantle,c16store_crust_mantle, & + c22store_crust_mantle,c23store_crust_mantle,c24store_crust_mantle, & + c25store_crust_mantle,c26store_crust_mantle,c33store_crust_mantle, & + c34store_crust_mantle,c35store_crust_mantle,c36store_crust_mantle, & + c44store_crust_mantle,c45store_crust_mantle,c46store_crust_mantle, & + c55store_crust_mantle,c56store_crust_mantle,c66store_crust_mantle, & + mu0store_crust_mantle, & + ibool_crust_mantle,dummy_idoubling,ispec_is_tiso_crust_mantle, & + rmassx_crust_mantle,rmassy_crust_mantle,rmassz_crust_mantle, & + NGLOB_CRUST_MANTLE_OCEANS,rmass_ocean_load, & + b_rmassx_crust_mantle,b_rmassy_crust_mantle) + else call read_arrays_solver(IREGION_CRUST_MANTLE, & NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE,NGLOB_XY_CM, & nspec_iso,nspec_tiso,nspec_ani, & @@ -582,6 +605,29 @@ subroutine read_mesh_databases_OC() dummy_rmass,dummy_rmass,rmass_outer_core, & 1,dummy_array, & dummy_rmass,dummy_rmass) + else if (HDF5_ENABLED) then + call read_arrays_solver_hdf5(IREGION_OUTER_CORE, & + NSPEC_OUTER_CORE,NGLOB_OUTER_CORE,NGLOB_XY_dummy, & + nspec_iso,nspec_tiso,nspec_ani, & + vp_outer_core,dummy_array, & + xstore_outer_core,ystore_outer_core,zstore_outer_core, & + xix_outer_core,xiy_outer_core,xiz_outer_core, & + etax_outer_core,etay_outer_core,etaz_outer_core, & + gammax_outer_core,gammay_outer_core,gammaz_outer_core, & + rhostore_outer_core,kappavstore_outer_core,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array, & + ibool_outer_core,dummy_idoubling_outer_core,dummy_ispec_is_tiso, & + dummy_rmass,dummy_rmass,rmass_outer_core, & + 1, dummy_array, & + dummy_rmass,dummy_rmass) else call read_arrays_solver(IREGION_OUTER_CORE, & NSPEC_OUTER_CORE,NGLOB_OUTER_CORE,NGLOB_XY_dummy, & @@ -759,6 +805,29 @@ subroutine read_mesh_databases_IC() rmassx_inner_core,rmassy_inner_core,rmassz_inner_core, & 1,dummy_array, & b_rmassx_inner_core,b_rmassy_inner_core) + else if (HDF5_ENABLED) then + call read_arrays_solver_hdf5(IREGION_INNER_CORE, & + NSPEC_INNER_CORE,NGLOB_INNER_CORE,NGLOB_XY_IC, & + nspec_iso,nspec_tiso,nspec_ani, & + dummy_array,dummy_array, & + xstore_inner_core,ystore_inner_core,zstore_inner_core, & + xix_inner_core,xiy_inner_core,xiz_inner_core, & + etax_inner_core,etay_inner_core,etaz_inner_core, & + gammax_inner_core,gammay_inner_core,gammaz_inner_core, & + rhostore_inner_core,kappavstore_inner_core,muvstore_inner_core, & + dummy_array,dummy_array,dummy_array, & + c11store_inner_core,c12store_inner_core,c13store_inner_core, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,c33store_inner_core, & + dummy_array,dummy_array,dummy_array, & + c44store_inner_core,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array, & + ibool_inner_core,idoubling_inner_core,dummy_ispec_is_tiso, & + rmassx_inner_core,rmassy_inner_core,rmassz_inner_core, & + 1,dummy_array, & + b_rmassx_inner_core,b_rmassy_inner_core) else call read_arrays_solver(IREGION_INNER_CORE, & NSPEC_INNER_CORE,NGLOB_INNER_CORE,NGLOB_XY_IC, & @@ -931,6 +1000,29 @@ subroutine read_mesh_databases_TRINF() dummy_rmass,dummy_rmass,dummy_rmass, & 1,dummy_array, & dummy_rmass,dummy_rmass) + else if (HDF5_ENABLED) then + call read_arrays_solver_hdf5(IREGION_TRINFINITE, & + NSPEC_TRINFINITE,NGLOB_TRINFINITE,NGLOB_XY_dummy, & + nspec_iso,nspec_tiso,nspec_ani, & + dummy_array,dummy_array, & + xstore_trinfinite,ystore_trinfinite,zstore_trinfinite, & + xix_trinfinite,xiy_trinfinite,xiz_trinfinite, & + etax_trinfinite,etay_trinfinite,etaz_trinfinite, & + gammax_trinfinite,gammay_trinfinite,gammaz_trinfinite, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array, & + ibool_trinfinite,dummy_idoubling,dummy_ispec_is_tiso, & + dummy_rmass,dummy_rmass,dummy_rmass, & + 1,dummy_array, & + dummy_rmass,dummy_rmass) else call read_arrays_solver(IREGION_TRINFINITE, & NSPEC_TRINFINITE,NGLOB_TRINFINITE,NGLOB_XY_dummy, & @@ -1061,6 +1153,29 @@ subroutine read_mesh_databases_INF() dummy_rmass,dummy_rmass,dummy_rmass, & 1,dummy_array, & dummy_rmass,dummy_rmass) + else if (HDF5_ENABLED) then + call read_arrays_solver_hdf5(IREGION_INFINITE, & + NSPEC_INFINITE,NGLOB_INFINITE,NGLOB_XY_dummy, & + nspec_iso,nspec_tiso,nspec_ani, & + dummy_array,dummy_array, & + xstore_infinite,ystore_infinite,zstore_infinite, & + xix_infinite,xiy_infinite,xiz_infinite, & + etax_infinite,etay_infinite,etaz_infinite, & + gammax_infinite,gammay_infinite,gammaz_infinite, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array, & + ibool_infinite,dummy_idoubling,dummy_ispec_is_tiso, & + dummy_rmass,dummy_rmass,dummy_rmass, & + 1,dummy_array, & + dummy_rmass,dummy_rmass) else call read_arrays_solver(IREGION_INFINITE, & NSPEC_INFINITE,NGLOB_INFINITE,NGLOB_XY_dummy, & @@ -1238,6 +1353,8 @@ subroutine read_mesh_databases_coupling() if (I_should_read_the_database) then if (ADIOS_FOR_ARRAYS_SOLVER) then call read_mesh_databases_coupling_adios() + else if (HDF5_ENABLED) then + call read_mesh_databases_coupling_hdf5() else ! crust and mantle if (NSPEC_CRUST_MANTLE > 0) then @@ -1450,7 +1567,7 @@ subroutine read_mesh_databases_coupling() endif endif - endif ! ADIOS + endif ! ADIOS or HDF5 endif call bcast_mesh_databases_coupling() @@ -1670,6 +1787,8 @@ subroutine read_mesh_databases_MPI() if (I_should_read_the_database .and. NSPEC_CRUST_MANTLE > 0) then if (ADIOS_FOR_MPI_ARRAYS) then call read_mesh_databases_MPI_adios(IREGION_CRUST_MANTLE) + else if (HDF5_ENABLED) then + call read_mesh_databases_MPI_hdf5(IREGION_CRUST_MANTLE) else call read_mesh_databases_MPI_CM() endif @@ -1767,6 +1886,8 @@ subroutine read_mesh_databases_MPI() if (I_should_read_the_database .and. NSPEC_OUTER_CORE > 0) then if (ADIOS_FOR_MPI_ARRAYS) then call read_mesh_databases_MPI_adios(IREGION_OUTER_CORE) + else if (HDF5_ENABLED) then + call read_mesh_databases_MPI_hdf5(IREGION_OUTER_CORE) else call read_mesh_databases_MPI_OC() endif @@ -1861,6 +1982,8 @@ subroutine read_mesh_databases_MPI() if (I_should_read_the_database .and. NSPEC_INNER_CORE > 0) then if (ADIOS_FOR_MPI_ARRAYS) then call read_mesh_databases_MPI_adios(IREGION_INNER_CORE) + else if (HDF5_ENABLED) then + call read_mesh_databases_MPI_hdf5(IREGION_INNER_CORE) else call read_mesh_databases_MPI_IC() endif @@ -1961,6 +2084,8 @@ subroutine read_mesh_databases_MPI() if (I_should_read_the_database .and. NSPEC_TRINFINITE > 0) then if (ADIOS_FOR_MPI_ARRAYS) then call read_mesh_databases_MPI_adios(IREGION_TRINFINITE) + else if (HDF5_ENABLED) then + call read_mesh_databases_MPI_hdf5(IREGION_TRINFINITE) else call read_mesh_databases_MPI_TRINF() endif @@ -2056,6 +2181,8 @@ subroutine read_mesh_databases_MPI() if (I_should_read_the_database .and. NSPEC_INFINITE > 0) then if (ADIOS_FOR_MPI_ARRAYS) then call read_mesh_databases_MPI_adios(IREGION_INFINITE) + else if (HDF5_ENABLED) then + call read_mesh_databases_MPI_hdf5(IREGION_INFINITE) else call read_mesh_databases_MPI_INF() endif @@ -2671,6 +2798,8 @@ subroutine read_mesh_databases_stacey() if (I_should_read_the_database) then if (ADIOS_FOR_ARRAYS_SOLVER) then call read_mesh_databases_stacey_adios() + else if (HDF5_ENABLED) then + call read_mesh_databases_stacey_hdf5() else ! crust and mantle if (NSPEC_CRUST_MANTLE > 0) then @@ -2798,6 +2927,8 @@ subroutine read_mesh_databases_regular_kl() ! checks setup if (ADIOS_FOR_KERNELS) & call exit_mpi(myrank,'saving regular kernels in ADIOS file format is not supported yet') + if (HDF5_ENABLED) & + call exit_mpi(myrank,'saving regular kernels in HDF5 file format is not supported yet') ! assuming 6 chunks full global simulations right now if (NCHUNKS_VAL /= 6 .or. NPROC_XI_VAL /= NPROC_ETA_VAL) & call exit_MPI(myrank, 'Only deal with 6 chunks at this moment') diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk index c3f9836f2..df4a71310 100644 --- a/src/specfem3D/rules.mk +++ b/src/specfem3D/rules.mk @@ -110,7 +110,9 @@ specfem3D_SOLVER_OBJECTS += \ $O/print_stf_file.solverstatic.o \ $O/read_adjoint_sources.solverstatic.o \ $O/read_arrays_solver.solverstatic.o \ + $O/read_arrays_solver_hdf5.solverstatic.o \ $O/read_forward_arrays.solverstatic.o \ + $O/read_forward_arrays_hdf5.solverstatic.o \ $O/read_mesh_parameters.solverstatic.o \ $O/read_mesh_databases.solverstatic.o \ $O/read_topography_bathymetry.solverstatic.o \ @@ -125,7 +127,9 @@ specfem3D_SOLVER_OBJECTS += \ $O/SIEM_solver_mpi.solverstatic.o \ $O/SIEM_solver_petsc.solverstatic.o \ $O/save_forward_arrays.solverstatic.o \ + $O/save_forward_arrays_hdf5.solverstatic.o \ $O/save_kernels.solverstatic.o \ + $O/save_kernels_hdf5.solverstatic.o \ $O/save_regular_kernels.solverstatic.o \ $O/setup_GLL_points.solverstatic.o \ $O/setup_sources_receivers.solverstatic.o \ @@ -181,6 +185,7 @@ specfem3D_SHARED_OBJECTS = \ $O/gll_library.shared.o \ $O/heap_sort.shared.o \ $O/hex_nodes.shared.o \ + $O/hdf5_manager.shared_hdf5_module.o \ $O/init_openmp.shared.o \ $O/intgrl.shared.o \ $O/lagrange_poly.shared.o \ @@ -474,3 +479,12 @@ $O/%.visualc.o: $S/%.c ${SETUP}/config.h ### $O/%.checknetcdf.o: $S/%.f90 $O/shared_par.shared_module.o $O/specfem3D_par.solverstatic_module.o ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(NETCDF_INCLUDE) -c -o $@ $< + +### +### HDF5 +### +$O/%.spec_hdf5_module.o: $S/%.F90 $O/shared_par.shared_module.o $O/specfem3D_par.solverstatic_module.o $O/hdf5_manager.shared_hdf5_module.o + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(HDF5_INCLUDE) -c -o $@ $< + +$O/%.spec_hdf5_module.o: $S/%.f90 $O/shared_par.shared_module.o $O/specfem3D_par.solverstatic_module.o $O/hdf5_manager.shared_hdf5_module.o + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(HDF5_INCLUDE) -c -o $@ $< \ No newline at end of file diff --git a/src/specfem3D/save_forward_arrays.F90 b/src/specfem3D/save_forward_arrays.F90 index 2f44a4f52..637cca9ef 100644 --- a/src/specfem3D/save_forward_arrays.F90 +++ b/src/specfem3D/save_forward_arrays.F90 @@ -66,6 +66,8 @@ subroutine save_forward_arrays() ! saves checkpoint if (ADIOS_FOR_FORWARD_ARRAYS) then call save_intermediate_forward_arrays_adios() + else if (HDF5_ENABLED) then + call save_intermediate_forward_arrays_hdf5() else write(outputname,"('dump_all_arrays',i6.6)") myrank open(unit=IOUT,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname), & @@ -141,6 +143,8 @@ subroutine save_forward_arrays() if (ADIOS_FOR_FORWARD_ARRAYS) then call save_forward_arrays_adios() + else if (HDF5_ENABLED) then + call save_forward_arrays_hdf5() else write(outputname,'(a,i6.6,a)') 'proc',myrank,'_save_forward_arrays.bin' outputname = trim(LOCAL_TMP_PATH)//'/'//trim(outputname) @@ -246,6 +250,8 @@ subroutine save_forward_arrays_undoatt() if (ADIOS_FOR_UNDO_ATTENUATION) then call save_forward_arrays_undoatt_adios() + else if (HDF5_ENABLED) then + call save_forward_arrays_undoatt_hdf5() else ! current subset iteration iteration_on_subset_tmp = iteration_on_subset diff --git a/src/specfem3D/save_forward_arrays_hdf5.F90 b/src/specfem3D/save_forward_arrays_hdf5.F90 new file mode 100644 index 000000000..29b36d84b --- /dev/null +++ b/src/specfem3D/save_forward_arrays_hdf5.F90 @@ -0,0 +1,877 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + +subroutine save_intermediate_forward_arrays_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + + ! MPI variables + integer :: ierr, info, comm + + ! collective write + logical :: if_col = .true. + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name + + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/dump_all_arrays.h5' + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create file and datasets by myrank==0 + if (myrank == 0) then + call h5_create_file(file_name) + + ! create datasets + call h5_create_dataset_gen('displ_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + + if (ROTATION_VAL) then + call h5_create_dataset_gen('A_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('B_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + endif + + if (ATTENUATION_VAL) then + call h5_create_dataset_gen('R_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + + call h5_create_dataset_gen('R_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_create_dataset_gen('neq', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('neq1', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('pgrav1', (/sum(offset_pgrav1)/), 1, CUSTOM_REAL) + endif ! FULL_GRAVITY_VAL + + ! close file + call h5_close_file() + endif ! myrank == 0 + + call synchronize_all() + + ! write data from all ranks + call h5_open_file_p_collect(file_name) + + ! write datasets + call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + + if (ROTATION_VAL) then + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + endif + + if (ATTENUATION_VAL) then + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), if_col) + call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), if_col) + call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), if_col) + endif ! FULL_GRAVITY_VAL + + ! close file + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + + +end subroutine save_intermediate_forward_arrays_hdf5 + + +subroutine save_forward_arrays_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! MPI variables + integer :: ierr, info, comm + + ! collective write + logical :: if_col = .true. + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name, dataset_name + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/save_forward_arrays.h5' + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create file and datasets by myrank==0 + if (myrank == 0) then + call h5_create_dataset_gen('displ_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + + if (ROTATION_VAL) then + call h5_create_dataset_gen('A_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('B_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + endif + + if (ATTENUATION_VAL) then + call h5_create_dataset_gen('R_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + + call h5_create_dataset_gen('R_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_create_dataset_gen('neq', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('neq1', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('pgrav1', (/sum(offset_pgrav1)/), 1, CUSTOM_REAL) + endif + + ! close file + call h5_close_file() + + endif ! myrank == 0 + + call synchronize_all() + + ! write data from all ranks + call h5_open_file_p_collect(file_name) + + ! write datasets + call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + if (ROTATION_VAL) then + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + endif + if (ATTENUATION_VAL) then + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), if_col) + call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), if_col) + call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), if_col) + endif ! FULL_GRAVITY_VAL + + ! close file + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + + +end subroutine save_forward_arrays_hdf5 + + +subroutine save_forward_arrays_undoatt_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + + ! MPI variables + integer :: ierr, info, comm + + ! collective write + logical :: if_col = .true. + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name + + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + + write(file_name, '(a,i6.6,a)') 'save_frame_at',iteration_on_subset,'.h5' + file_name = trim(LOCAL_PATH)//'/'//trim(file_name) + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create file and datasets by myrank==0 + if (myrank == 0) then + call h5_create_file(file_name) + + ! create datasets + call h5_create_dataset_gen('displ_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + + if (ROTATION_VAL) then + call h5_create_dataset_gen('A_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('B_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + endif + + if (ATTENUATION_VAL) then + call h5_create_dataset_gen('R_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + + call h5_create_dataset_gen('R_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_create_dataset_gen('neq', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('neq1', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('pgrav1', (/sum(offset_pgrav1)/), 1, CUSTOM_REAL) + endif ! FULL_GRAVITY_VAL + + ! close file + call h5_close_file() + endif ! myrank == 0 + + call synchronize_all() + + ! write data from all ranks + call h5_open_file_p_collect(file_name) + + ! write datasets + call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + + if (ROTATION_VAL) then + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + endif + + if (ATTENUATION_VAL) then + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), if_col) + call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), if_col) + call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), if_col) + endif ! FULL_GRAVITY_VAL + + ! close file + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine save_forward_arrays_undoatt_hdf5 + + +subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_crust_mantle,factor_scale_relaxed_inner_core) + + use constants + use shared_parameters, only: R_PLANET,RHOAV,LOCAL_PATH,TRANSVERSE_ISOTROPY + + use specfem_par_crustmantle + use specfem_par_innercore + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + real(kind=CUSTOM_REAL),dimension(ATT1_VAL,ATT2_VAL,ATT3_VAL,ATT4_VAL) :: factor_scale_relaxed_crust_mantle + real(kind=CUSTOM_REAL),dimension(ATT1_VAL,ATT2_VAL,ATT3_VAL,ATT5_VAL) :: factor_scale_relaxed_inner_core + +#ifdef USE_HDF5 + + ! local parameters + integer :: ier + real(kind=CUSTOM_REAL) :: scaleval1,scaleval2,scale_factor_r + real(kind=CUSTOM_REAL),dimension(:,:,:,:),allocatable :: temp_store + real(kind=CUSTOM_REAL),dimension(:,:,:,:),allocatable :: muv_shifted,muh_shifted + integer :: iregion_code,nspec,nglob,i,j,k,ispec + + ! collective write + logical :: if_col = .true. + + ! debug + logical, parameter :: OUTPUT_RELAXED_MODEL = .false. + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic + character(len=MAX_STRING_LEN) :: file_name, group_name + + ! gather the offset arrays + call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_cm, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_oc, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_ic, NPROCTOT_VAL) + + ! file name + file_name = trim(LOCAL_PATH)//'/'//'model_shifted.h5' + + ! create file and datasets by myrank==0 + if (myrank == 0) then + call h5_create_file(file_name) + + if (NSPEC_CRUST_MANTLE > 0) then + ! safety check + if (ANISOTROPIC_3D_MANTLE_VAL) & + call exit_mpi(myrank,'ANISOTROPIC_3D_MANTLE not supported yet for shifted model file output') + + ! group name + write(group_name, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_create_group(group_name) + call h5_open_group(group_name) + + ! create datasets + call h5_create_dataset_gen_in_group('muv_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('muh_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + + if (TRANSVERSE_ISOTROPY) then + call h5_create_dataset_gen_in_group('vpv_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('vph_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('vsv_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('vsh_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + else ! isotropic + call h5_create_dataset_gen_in_group('vp_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('vs_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + endif + + ! close group + call h5_close_group() + endif ! NSPEC_CRUST_MANTLE > 0 + + if (NSPEC_INNER_CORE > 0) then + if (ANISOTROPIC_INNER_CORE_VAL) then + call exit_mpi(myrank,'ANISOTROPIC_INNER_CORE not supported yet for shifted model file output') + else + ! only isotropic inner core supported + + ! group name + write(group_name, "('reg',i1)") IREGION_INNER_CORE + call h5_create_group(group_name) + call h5_open_group(group_name) + + ! create datasets + call h5_create_dataset_gen_in_group('vp_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_ic)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('vs_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_ic)/), 4, CUSTOM_REAL) + endif + + ! close group + call h5_close_group() + endif + + ! output relaxed model values + if (OUTPUT_RELAXED_MODEL) then + ! checks + if (.not. TRANSVERSE_ISOTROPY) stop 'Outputting relaxed model requires TRANSVERSE_ISOTROPY' + + if (NSPEC_CRUST_MANTLE > 0) then + ! group name + write(group_name, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_create_group(group_name) + ! open group + call h5_open_group(group_name) + + ! create datasets + call h5_create_dataset_gen_in_group('muv_relaxed', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('muh_relaxed', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('kappav_relaxed', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('kappah_relaxed', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + endif + + endif ! OUTPUT_RELAXED_MODEL + + ! close file + call h5_close_file() + endif ! myrank == 0 + + call synchronize_all() + + ! + ! write data from all ranks + ! + call h5_open_file_p_collect(file_name) + + ! scaling factors to re-dimensionalize units + scaleval1 = real( sqrt(PI*GRAV*RHOAV)*(R_PLANET/1000.0d0), kind=CUSTOM_REAL) ! velocities + + if (NSPEC_CRUST_MANTLE > 0) then + ! open group + write(group_name, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_open_group(group_name) + + ! uses temporary array + allocate(temp_store(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE), & + muv_shifted(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE), & + muh_shifted(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE), stat=ier) + if (ier /= 0) stop 'Error allocating temp_store array' + temp_store(:,:,:,:) = 0._CUSTOM_REAL + + ! safety check + if (ANISOTROPIC_3D_MANTLE_VAL) & + call exit_mpi(myrank,'ANISOTROPIC_3D_MANTLE not supported yet for shifted model file output') + + ! user output + if (myrank == 0) then + write(IMAIN,*) ' shifted model files in directory: ',trim(LOCAL_PATH) + endif + + ! user output + if (myrank == 0) write(IMAIN,*) ' crust/mantle:' + + ! moduli (muv,muh) are at relaxed values (only Qmu implemented), + ! scales back to have values at center frequency + muv_shifted(:,:,:,:) = muvstore_crust_mantle(:,:,:,:) + muh_shifted(:,:,:,:) = muhstore_crust_mantle(:,:,:,:) + do ispec = 1,NSPEC_CRUST_MANTLE + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + if (ATTENUATION_3D_VAL .or. ATTENUATION_1D_WITH_3D_STORAGE_VAL) then + scale_factor_r = factor_scale_relaxed_crust_mantle(i,j,k,ispec) + else + scale_factor_r = factor_scale_relaxed_crust_mantle(1,1,1,ispec) + endif + ! scaling back from relaxed to values at shifted frequency + ! (see in prepare_attenuation.f90 for how muv,muh are scaled to become relaxed moduli) + ! muv + muv_shifted(i,j,k,ispec) = muv_shifted(i,j,k,ispec) / scale_factor_r + ! muh + if (ispec_is_tiso_crust_mantle(ispec)) then + muh_shifted(i,j,k,ispec) = muh_shifted(i,j,k,ispec) / scale_factor_r + endif + enddo + enddo + enddo + enddo + + if (TRANSVERSE_ISOTROPY) then + ! vpv (at relaxed values) + temp_store(:,:,:,:) = sqrt((kappavstore_crust_mantle(:,:,:,:) & + + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & + * scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vpv_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + ! vph + temp_store(:,:,:,:) = sqrt((kappahstore_crust_mantle(:,:,:,:) & + + FOUR_THIRDS * muh_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & + * scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vph_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + ! vsv + temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vsv_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + ! vsh + temp_store(:,:,:,:) = sqrt( muh_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vsh_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + else ! isotropic + ! vp + temp_store(:,:,:,:) = sqrt((kappavstore_crust_mantle(:,:,:,:) & + + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & + * scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + ! vs + temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + + endif + + deallocate(temp_store,muv_shifted,muh_shifted) + + ! close group + call h5_close_group() + endif ! NSPEC_CRUST_MANTLE > 0 + + if (NSPEC_INNER_CORE > 0) then + ! open group + write(group_name, "('reg',i1)") IREGION_INNER_CORE + call h5_open_group(group_name) + + ! uses temporary array + allocate(temp_store(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE), & + muv_shifted(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE), stat=ier) + if (ier /= 0) stop 'Error allocating temp_store array' + temp_store(:,:,:,:) = 0._CUSTOM_REAL + + ! user output + if (myrank == 0) write(IMAIN,*) ' inner core:' + + ! moduli (muv,muh) are at relaxed values, scale back to have shifted values at center frequency + muv_shifted(:,:,:,:) = muvstore_inner_core(:,:,:,:) + do ispec = 1,NSPEC_INNER_CORE + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + if (ATTENUATION_3D_VAL .or. ATTENUATION_1D_WITH_3D_STORAGE_VAL) then + scale_factor_r = factor_scale_relaxed_inner_core(i,j,k,ispec) + else + scale_factor_r = factor_scale_relaxed_inner_core(1,1,1,ispec) + endif + + ! inverts to scale relaxed back to shifted factor + ! scaling back from relaxed to values at shifted frequency + ! (see in prepare_attenuation.f90 for how muv,muh are scaled to become relaxed moduli) + ! muv + muv_shifted(i,j,k,ispec) = muv_shifted(i,j,k,ispec) / scale_factor_r + enddo + enddo + enddo + enddo + + if (ANISOTROPIC_INNER_CORE_VAL) then + call exit_mpi(myrank,'ANISOTROPIC_INNER_CORE not supported yet for shifted model file output') + else + ! isotropic model + ! vp + temp_store(:,:,:,:) = sqrt((kappavstore_inner_core(:,:,:,:) & + + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_inner_core(:,:,:,:)) & + * scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), if_col) + ! vs + temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_inner_core(:,:,:,:) )*scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), if_col) + endif + + deallocate(temp_store,muv_shifted) + + ! close group + call h5_close_group() + + endif ! NSPEC_INNER_CORE > 0 + + if (OUTPUT_RELAXED_MODEL) then + ! user output + if (myrank == 0) then + write(IMAIN,*) ' outputting relaxed model:' + call flush_IMAIN() + endif + ! checks + if (.not. TRANSVERSE_ISOTROPY) stop 'Outputting relaxed model requires TRANSVERSE_ISOTROPY' + + ! scaling factor to re-dimensionalize units + ! the scale of GPa--[g/cm^3][(km/s)^2] + scaleval1 = real( ((sqrt(PI*GRAV*RHOAV)*R_PLANET/1000.d0)**2)*(RHOAV/1000.d0), kind=CUSTOM_REAL) ! moduli GPa + + if (NSPEC_CRUST_MANTLE > 0) then + ! open group + write(group_name, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_open_group(group_name) + + ! muv_relaxed + call h5_write_dataset_collect_hyperslab_in_group('muv_relaxed', muvstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + ! muh_relaxed + call h5_write_dataset_collect_hyperslab_in_group('muh_relaxed', muhstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + ! kappav_relaxed + call h5_write_dataset_collect_hyperslab_in_group('kappav_relaxed', kappavstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + ! kappah_relaxed + call h5_write_dataset_collect_hyperslab_in_group('kappah_relaxed', kappahstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + + ! close group + call h5_close_group() + + endif ! NSPEC_CRUST_MANTLE > 0 + + endif ! OUTPUT_RELAXED_MODEL + + ! close file + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine save_forward_model_at_shifted_frequency_hdf5 diff --git a/src/specfem3D/save_kernels.F90 b/src/specfem3D/save_kernels.F90 index 7295e8d47..5f7e0cf37 100644 --- a/src/specfem3D/save_kernels.F90 +++ b/src/specfem3D/save_kernels.F90 @@ -1298,6 +1298,13 @@ subroutine save_kernels_crust_mantle_ani() bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle, & bulk_betav_kl_crust_mantle,bulk_betah_kl_crust_mantle, & Gc_prime_kl_crust_mantle, Gs_prime_kl_crust_mantle) + else if (HDF5_ENABLED) then + call write_kernels_cm_ani_hdf5(alphav_kl_crust_mantle,alphah_kl_crust_mantle, & + betav_kl_crust_mantle,betah_kl_crust_mantle, & + eta_kl_crust_mantle, & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle, & + bulk_betav_kl_crust_mantle,bulk_betah_kl_crust_mantle, & + Gc_prime_kl_crust_mantle, Gs_prime_kl_crust_mantle) else ! binary file output call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH) @@ -1647,6 +1654,9 @@ subroutine save_kernels_crust_mantle_iso() if (ADIOS_FOR_KERNELS) then call write_kernels_cm_iso_adios(mu_kl_crust_mantle, kappa_kl_crust_mantle, rhonotprime_kl_crust_mantle, & bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle) + else if (HDF5_ENABLED) then + call write_kernels_cm_iso_hdf5(mu_kl_crust_mantle, kappa_kl_crust_mantle, rhonotprime_kl_crust_mantle, & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle) else call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH) @@ -1742,6 +1752,8 @@ subroutine save_kernels_outer_core(rhostore_outer_core,kappavstore_outer_core,rh ! writes out kernels to file if (ADIOS_FOR_KERNELS) then call write_kernels_oc_adios() + else if (HDF5_ENABLED) then + call write_kernel_oc_hdf5() else call create_name_database(prname,myrank,IREGION_OUTER_CORE,LOCAL_TMP_PATH) @@ -1751,7 +1763,6 @@ subroutine save_kernels_outer_core(rhostore_outer_core,kappavstore_outer_core,rh open(unit=IOUT,file=trim(prname)//'alpha_kernel.bin',status='unknown',form='unformatted',action='write') write(IOUT) alpha_kl_outer_core close(IOUT) - endif end subroutine save_kernels_outer_core @@ -1814,6 +1825,8 @@ subroutine save_kernels_inner_core(rhostore_inner_core,muvstore_inner_core,kappa ! writes out kernels to file if (ADIOS_FOR_KERNELS) then call write_kernels_ic_adios() + else if (HDF5_ENABLED) then + call write_kernels_ic_hdf5() else call create_name_database(prname,myrank,IREGION_INNER_CORE,LOCAL_TMP_PATH) @@ -1862,6 +1875,8 @@ subroutine save_kernels_boundary_kl() ! writes out kernels to file if (ADIOS_FOR_KERNELS) then call write_kernels_boundary_kl_adios() + else if (HDF5_ENABLED) then + call write_kernels_boundary_kl_hdf5() else call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH) @@ -1944,6 +1959,9 @@ subroutine save_kernels_source_derivatives() ! writes out kernels to file if (ADIOS_FOR_KERNELS) then call write_kernels_source_derivatives_adios() + !else if (HDF5_ENABLED) then + ! ! TODO ADD HDF5 + ! call write_kernels_source_derivatives_hdf5() else ! kernel file output do irec_local = 1, nrec_local @@ -2001,6 +2019,8 @@ subroutine save_kernels_Hessian() ! writes out kernels to file if (ADIOS_FOR_KERNELS) then call write_kernels_Hessian_adios() + else if (HDF5_ENABLED) then + call write_kernels_Hessian_hdf5() else ! stores into file call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH) diff --git a/src/specfem3D/save_kernels_hdf5.F90 b/src/specfem3D/save_kernels_hdf5.F90 new file mode 100644 index 000000000..a9a3e0eea --- /dev/null +++ b/src/specfem3D/save_kernels_hdf5.F90 @@ -0,0 +1,675 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +subroutine write_kernels_strength_noise_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_noise + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_adj + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: ierr, info, comm + logical :: if_col = .true. + + ! gather the number of elements in each region + call gather_all_all_singlei(size(sigma_kl_crust_mantle,4), offset_nspec_cm_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + call h5_create_dataset_gen('sigma_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + call h5_write_dataset_collect_hyperslab('sigma_kernel', sigma_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_strength_noise_hdf5 + + +subroutine write_kernels_cm_ani_hdf5(alphav_kl_crust_mantle,alphah_kl_crust_mantle, & + betav_kl_crust_mantle,betah_kl_crust_mantle, & + eta_kl_crust_mantle, & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle, & + bulk_betav_kl_crust_mantle,bulk_betah_kl_crust_mantle, & + Gc_prime_kl_crust_mantle, Gs_prime_kl_crust_mantle) + + + use specfem_par + use specfem_par_crustmantle + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + ! input Parameters + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: & + alphav_kl_crust_mantle,alphah_kl_crust_mantle, & + betav_kl_crust_mantle,betah_kl_crust_mantle, & + eta_kl_crust_mantle + + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle, & + bulk_betav_kl_crust_mantle,bulk_betah_kl_crust_mantle + + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: & + Gc_prime_kl_crust_mantle, Gs_prime_kl_crust_mantle + +#ifdef USE_HDF5 + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: ierr, info, comm + logical :: if_col = .true. + + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_adj + + ! check if anything to do + if (.not. ANISOTROPIC_KL) return + + ! gather the number of elements in each region + !call gather_all_all_singlei(size(alphav_kl_crust_mantle,4), offset_nspec_cm_adj, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_CRUST_MANTLE_ADJOINT, offset_nspec_cm_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! create or open file + call h5_create_or_open_file(file_name) + ! create dataset + if (SAVE_TRANSVERSE_KL_ONLY) then + call h5_create_dataset_gen('alphav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('alphah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('betav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('betah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('eta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('bulk_c_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_betav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_betah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('alpha_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + else if (SAVE_AZIMUTHAL_ANISO_KL_ONLY) then + call h5_create_dataset_gen('alphav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('alphah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('betav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('betah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('bulk_c_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_betav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_betah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('eta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('Gc_prime_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Gs_prime_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + ! check isotropic kernel + if (.false.) then + call h5_create_dataset_gen('alpha_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + endif + ! check anisotropic kernels + if (.false.) then + call h5_create_dataset_gen('A_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('C_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('L_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('N_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('F_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Gc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Gs_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Jc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Kc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Mc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Bc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Hc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Ec_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Dc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + endif + + else + ! fully anisotropic kernels + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('cijkl_kernel', (/21,NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 5, CUSTOM_REAL) + + endif + + ! close file + call h5_close_file() + + endif ! myrank == 0 + + ! synchronize all + call synchronize_all() + + ! write data from all ranks + call h5_open_file_p_collect(file_name) + + ! write data + if (SAVE_TRANSVERSE_KL_ONLY) then + call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + else if (SAVE_AZIMUTHAL_ANISO_KL_ONLY) then + call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + call h5_write_dataset_collect_hyperslab('Gc_prime_kernel', Gc_prime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('Gs_prime_kernel', Gs_prime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + ! check isotropic kernel + if (.false.) then + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + endif + + ! check anisotropic kernels + !if (.false.) then + ! call h5_write_dataset_collect_hyperslab('A_kernel', A_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('C_kernel', C_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('L_kernel', L_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('N_kernel', N_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('F_kernel', F_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('Gc_kernel', Gc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('Gs_kernel', Gs_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('Jc_kernel', Jc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('Kc_kernel', Kc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('Mc_kernel', Mc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('Bc_kernel', Bc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('Hc_kernel', Hc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('Ec_kernel', Ec_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('Dc_kernel', Dc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + !endif + + else + + ! fully anisotropic kernels + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('cijkl_kernel', cijkl_kl_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + endif + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_cm_ani_hdf5 + + +subroutine write_kernels_cm_iso_hdf5(mu_kl_crust_mantle, kappa_kl_crust_mantle, rhonotprime_kl_crust_mantle, & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle) + + use specfem_par + use specfem_par_crustmantle + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + ! Parameters + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: & + mu_kl_crust_mantle, kappa_kl_crust_mantle, rhonotprime_kl_crust_mantle, & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_adj + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: ierr, info, comm + logical :: if_col = .true. + + ! checks if anything to do + if (ANISOTROPIC_KL) return + + ! gather the number of elements in each region + call gather_all_all_singlei(NSPEC_CRUST_MANTLE_ADJOINT, offset_nspec_cm_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + call h5_create_dataset_gen('rhonotprime_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('kappa_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('mu_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('alpha_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('bulk_c_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + call h5_write_dataset_collect_hyperslab('rhonotprime_kernel', rhonotprime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('kappa_kernel', kappa_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('mu_kernel', mu_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_cm_iso_hdf5 + + +subroutine write_kernels_oc_hdf5() + + use specfem_par + use specfem_par_outercore + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_adj + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: ierr, info, comm + logical :: if_col = .true. + + ! gather the number of elements in each region + call gather_all_all_singlei(NSPEC_OUTER_CORE_ADJOINT, offset_nspec_oc_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_oc_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('alpha_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_oc_adj)/), 4, CUSTOM_REAL) + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_outer_core, (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_outer_core, (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), if_col) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_oc_hdf5 + + +subroutine write_kernels_ic_hdf5() + + use specfem_par + use specfem_par_innercore + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_adj + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: ierr, info, comm + logical :: if_col = .true. + + ! gather the number of elements in each region + call gather_all_all_singlei(NSPEC_INNER_CORE_ADJOINT, offset_nspec_ic_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_ic_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('alpha_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_ic_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_ic_adj)/), 4, CUSTOM_REAL) + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), if_col) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_ic_hdf5 + + +subroutine write_kernels_boundary_kl_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2d_moho + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2d_400 + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2d_670 + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2d_cmb + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2d_icb + + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: ierr, info, comm + logical :: if_col = .true. + + if (.not. SAVE_KERNELS_BOUNDARY) return + + ! gather the number of elements in each region + call gather_all_all_singlei(NSPEC2D_MOHO, offset_nspec2d_moho, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC2D_400, offset_nspec2d_400, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC2D_670, offset_nspec2d_670, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC2D_CMB, offset_nspec2d_cmb, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC2D_ICB, offset_nspec2d_icb, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + if (.not. SUPPRESS_CRUSTAL_MESH .and. HONOR_1D_SPHERICAL_MOHO) then + call h5_create_dataset_gen('moho_kernel', (/0,0,sum(offset_nspec2d_moho(0:myrank-1))/), 3, CUSTOM_REAL) + endif + call h5_create_dataset_gen('d400_kernel', (/0,0,sum(offset_nspec2d_400(0:myrank-1))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen('d670_kernel', (/0,0,sum(offset_nspec2d_670(0:myrank-1))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen('CMB_kernel', (/0,0,sum(offset_nspec2d_cmb(0:myrank-1))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen('ICB_kernel', (/0,0,sum(offset_nspec2d_icb(0:myrank-1))/), 3, CUSTOM_REAL) + + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + if (.not. SUPPRESS_CRUSTAL_MESH .and. HONOR_1D_SPHERICAL_MOHO) then + call h5_write_dataset_collect_hyperslab('moho_kernel', moho_kl, (/0,0,sum(offset_nspec2d_moho(0:myrank-1))/), if_col) + endif + call h5_write_dataset_collect_hyperslab('d400_kernel', d400_kl, (/0,0,sum(offset_nspec2d_400(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('d670_kernel', d670_kl, (/0,0,sum(offset_nspec2d_670(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('CMB_kernel', CMB_kl, (/0,0,sum(offset_nspec2d_CMB(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('ICB_kernel', ICB_kl, (/0,0,sum(offset_nspec2d_ICB(0:myrank-1))/), if_col) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_boundary_kl_hdf5 + + +subroutine write_kernels_Hessian_hdf5() + + use specfem_par + use specfem_par_crustmantle + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_adj + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: ierr, info, comm + logical :: if_col = .true. + + ! gather the number of elements in each region + call gather_all_all_singlei(NSPEC_CRUST_MANTLE_ADJOINT, offset_nspec_cm_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + call h5_create_dataset_gen('hess_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('hess_rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('hess_kappa_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('hess_mu_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + call h5_write_dataset_collect_hyperslab('hess_kernel', hess_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('hess_rho_kernel', hess_rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('hess_kappa_kernel', hess_kappa_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('hess_mu_kernel', hess_mu_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_Hessian_hdf5 diff --git a/src/specfem3D/save_regular_kernels.f90 b/src/specfem3D/save_regular_kernels.f90 index d88b4039e..e37981260 100644 --- a/src/specfem3D/save_regular_kernels.f90 +++ b/src/specfem3D/save_regular_kernels.f90 @@ -390,6 +390,9 @@ subroutine save_regular_kernels_cm() if (ADIOS_FOR_KERNELS) then ! check implementation call exit_mpi(myrank,'saving regular kernels in ADIOS file format is not supported yet') + else if (HDF5_ENABLED) then + ! TODO ADD HDF5 + call exit_mpi(myrank,'saving regular kernels in HDF5 file format is not supported yet') else ! sets up database name call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_PATH) diff --git a/src/specfem3D/write_movie_output.f90 b/src/specfem3D/write_movie_output.f90 index 6d2eb005b..a244bc8bd 100644 --- a/src/specfem3D/write_movie_output.f90 +++ b/src/specfem3D/write_movie_output.f90 @@ -75,6 +75,7 @@ subroutine write_movie_output() endif endif + ! TODO ADD HDF5 and IO_SERVER ! save velocity here to avoid static offset on displacement for movies call write_movie_surface() @@ -134,6 +135,7 @@ subroutine write_movie_output() epsilondev_xy_crust_mantle,epsilondev_xz_crust_mantle, & epsilondev_yz_crust_mantle) endif + ! TODO ADD HDF5 and IO_SERVER call write_movie_volume_strains(NSPEC_CRUST_MANTLE_STRAIN_ONLY, & eps_trace_over_3_crust_mantle, & NSPEC_CRUST_MANTLE_STR_OR_ATT, & @@ -141,6 +143,7 @@ subroutine write_movie_output() epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle) case (2, 3) + ! TODO ADD HDF5 and IO_SERVER ! output the Time Integral of Strain, or \mu*TIS call write_movie_volume_strains(NSPEC_CRUST_MANTLE_3DMOVIE, & Ieps_trace_over_3_crust_mantle, & @@ -167,6 +170,7 @@ subroutine write_movie_output() call transfer_fields_oc_from_device(NGLOB_OUTER_CORE, & displ_outer_core,veloc_outer_core,accel_outer_core,Mesh_pointer) endif + ! TODO ADD HDF5 and IO_SERVER call write_movie_volume_divcurl(NSPEC_CRUST_MANTLE_STRAIN_ONLY,eps_trace_over_3_crust_mantle, & div_displ_outer_core, & accel_outer_core,kappavstore_outer_core,rhostore_outer_core,ibool_outer_core, & @@ -184,6 +188,7 @@ subroutine write_movie_output() call transfer_displ_cm_from_device(NDIM*NGLOB_CRUST_MANTLE,displ_crust_mantle,Mesh_pointer) endif scalingval = scale_displ + ! TODO ADD HDF5 and IO_SERVER call write_movie_volume_vector(npoints_3dmovie, & ibool_crust_mantle, & displ_crust_mantle, & @@ -195,6 +200,7 @@ subroutine write_movie_output() call transfer_veloc_cm_from_device(NDIM*NGLOB_CRUST_MANTLE,veloc_crust_mantle,Mesh_pointer) endif scalingval = scale_veloc + ! TODO ADD HDF5 and IO_SERVER call write_movie_volume_vector(npoints_3dmovie, & ibool_crust_mantle, & veloc_crust_mantle, & @@ -209,6 +215,7 @@ subroutine write_movie_output() call transfer_displ_ic_from_device(NDIM*NGLOB_INNER_CORE,displ_inner_core,Mesh_pointer) call transfer_displ_oc_from_device(NGLOB_OUTER_CORE,displ_outer_core,Mesh_pointer) endif + ! TODO ADD HDF5 and IO_SERVER call write_movie_volume_displnorm(displ_crust_mantle,displ_inner_core,displ_outer_core, & ibool_crust_mantle,ibool_inner_core,ibool_outer_core) @@ -221,6 +228,7 @@ subroutine write_movie_output() call transfer_veloc_ic_from_device(NDIM*NGLOB_INNER_CORE,veloc_inner_core,Mesh_pointer) call transfer_veloc_oc_from_device(NGLOB_OUTER_CORE,veloc_outer_core,Mesh_pointer) endif + ! TODO ADD HDF5 and IO_SERVER call write_movie_volume_velnorm(veloc_crust_mantle,veloc_inner_core,veloc_outer_core, & ibool_crust_mantle,ibool_inner_core,ibool_outer_core) @@ -233,6 +241,7 @@ subroutine write_movie_output() call transfer_accel_ic_from_device(NDIM*NGLOB_INNER_CORE,accel_inner_core,Mesh_pointer) call transfer_accel_oc_from_device(NGLOB_OUTER_CORE,accel_outer_core,Mesh_pointer) endif + ! TODO ADD HDF5 and IO_SERVER call write_movie_volume_accelnorm(accel_crust_mantle,accel_inner_core,accel_outer_core, & ibool_crust_mantle,ibool_inner_core,ibool_outer_core) diff --git a/src/specfem3D/write_seismograms.f90 b/src/specfem3D/write_seismograms.f90 index 1472388b0..80855333d 100644 --- a/src/specfem3D/write_seismograms.f90 +++ b/src/specfem3D/write_seismograms.f90 @@ -51,6 +51,9 @@ subroutine write_seismograms() implicit none + ! HDF5 is not implemented because ASDF is already available + ! TODO ADD IO_SERVER for seismograms (low priority because it's not I/O intensive) + ! local parameters ! timing double precision, external :: wtime From 760850e5a1d41e9790447f43b0b6c16680cc6f64 Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Mon, 5 Aug 2024 22:55:45 -0600 Subject: [PATCH 02/19] add HDF5: update Par_file --- DATA/Par_file | 4 +--- .../DATA/Par_file | 4 ++++ .../prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file | 4 ++++ .../prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file | 4 ++++ .../prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file | 4 ++++ .../undo_attenuation_benchmark_GJI_2016/DATA/Par_file | 4 ++++ EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file | 4 ++++ EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file | 4 ++++ EXAMPLES/global_s362ani_shakemovie/DATA/Par_file | 4 ++++ EXAMPLES/global_small/DATA/Par_file | 4 ++++ EXAMPLES/mars_global/DATA/Par_file | 4 ++++ EXAMPLES/mars_regional/DATA/Par_file | 4 ++++ EXAMPLES/moon_global/DATA/Par_file | 4 ++++ .../global_long/DATA/Par_file_NOISE_1_attenuation | 4 ++++ .../global_long/DATA/Par_file_NOISE_1_noattenuation | 4 ++++ .../global_long/DATA/Par_file_NOISE_2_attenuation | 4 ++++ .../global_long/DATA/Par_file_NOISE_2_noattenuation | 4 ++++ .../global_long/DATA/Par_file_NOISE_3_noattenuation | 4 ++++ .../global_short/DATA/Par_file_NOISE_1_attenuation | 4 ++++ .../global_short/DATA/Par_file_NOISE_1_noattenuation | 4 ++++ .../global_short/DATA/Par_file_NOISE_2_attenuation | 4 ++++ .../global_short/DATA/Par_file_NOISE_2_noattenuation | 4 ++++ .../global_short/DATA/Par_file_NOISE_3_noattenuation | 4 ++++ .../noise_examples/regional/DATA/Par_file_NOISE_1_attenuation | 4 ++++ .../regional/DATA/Par_file_NOISE_1_noattenuation | 4 ++++ .../noise_examples/regional/DATA/Par_file_NOISE_2_attenuation | 4 ++++ .../regional/DATA/Par_file_NOISE_2_noattenuation | 4 ++++ .../regional/DATA/Par_file_NOISE_3_noattenuation | 4 ++++ .../test_global/DATA/Par_file_NOISE_1_attenuation | 4 ++++ .../test_global/DATA/Par_file_NOISE_1_noattenuation | 4 ++++ .../test_global/DATA/Par_file_NOISE_2_attenuation | 4 ++++ .../test_global/DATA/Par_file_NOISE_2_noattenuation | 4 ++++ .../test_global/DATA/Par_file_NOISE_3_noattenuation | 4 ++++ .../test_regional/DATA/Par_file_NOISE_1_attenuation | 4 ++++ .../test_regional/DATA/Par_file_NOISE_1_noattenuation | 4 ++++ .../test_regional/DATA/Par_file_NOISE_2_attenuation | 4 ++++ .../test_regional/DATA/Par_file_NOISE_2_noattenuation | 4 ++++ .../test_regional/DATA/Par_file_NOISE_3_noattenuation | 4 ++++ EXAMPLES/point_force/DATA/Par_file | 4 ++++ EXAMPLES/regional_EMC_model/DATA/Par_file | 4 ++++ EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM | 4 ++++ EXAMPLES/regional_Greece_noise_small/DATA/Par_file | 4 ++++ EXAMPLES/regional_Greece_small/DATA/Par_file | 4 ++++ EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file | 4 ++++ EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file | 4 ++++ .../regional_Mexico_noise_non_uniform/DATA/Par_file_step1 | 4 ++++ .../regional_Mexico_noise_non_uniform/DATA/Par_file_step2 | 4 ++++ EXAMPLES/regional_MiddleEast/DATA/Par_file | 4 ++++ EXAMPLES/regional_s40rts/DATA/Par_file | 4 ++++ EXAMPLES/regional_sgloberani/DATA/Par_file | 4 ++++ EXAMPLES/regional_simultaneous_runs/DATA/Par_file | 4 ++++ EXAMPLES/regional_small_adjoint/DATA/Par_file | 4 ++++ EXAMPLES/regular_kernel/DATA/Par_file | 4 ++++ .../DATA/Par_file | 4 ++++ .../DATA/Par_file | 4 ++++ 55 files changed, 217 insertions(+), 3 deletions(-) diff --git a/DATA/Par_file b/DATA/Par_file index fb53c6442..3208f64d2 100644 --- a/DATA/Par_file +++ b/DATA/Par_file @@ -432,7 +432,5 @@ ADIOS_FOR_UNDO_ATTENUATION = .true. # HDF5 Database I/O # (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) -HDF5_ENABLED = .true. -HDF5_FOR_MOVIES = .false. -HDF5_IO_NODES = 0 # HDF5 IO server with number of IO dedicated procs +HDF5_ENABLED = .false. diff --git a/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file b/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file index 6c33f13a1..19bca021b 100644 --- a/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file +++ b/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file index 6159fbb94..634cf6419 100644 --- a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file +++ b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file index 8a54d8510..3b6dfc476 100644 --- a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file +++ b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file b/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file index 79e12f62b..d7a9f40ad 100644 --- a/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file +++ b/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file b/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file index 5c096c033..914e4ab83 100644 --- a/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file +++ b/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file b/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file index 9a525ceb9..4738537b8 100644 --- a/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file +++ b/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file b/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file index 9a525ceb9..4738537b8 100644 --- a/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file +++ b/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file b/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file index 17e375429..2f231de7e 100644 --- a/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file +++ b/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/global_small/DATA/Par_file b/EXAMPLES/global_small/DATA/Par_file index eb37a3af8..ee62ad37d 100644 --- a/EXAMPLES/global_small/DATA/Par_file +++ b/EXAMPLES/global_small/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/mars_global/DATA/Par_file b/EXAMPLES/mars_global/DATA/Par_file index 79e0e182e..9c552bf75 100644 --- a/EXAMPLES/mars_global/DATA/Par_file +++ b/EXAMPLES/mars_global/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/mars_regional/DATA/Par_file b/EXAMPLES/mars_regional/DATA/Par_file index 267ca2b5a..a5a82cde4 100644 --- a/EXAMPLES/mars_regional/DATA/Par_file +++ b/EXAMPLES/mars_regional/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/moon_global/DATA/Par_file b/EXAMPLES/moon_global/DATA/Par_file index 88d8b3b03..33f8df16a 100644 --- a/EXAMPLES/moon_global/DATA/Par_file +++ b/EXAMPLES/moon_global/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation index 46f994bee..74e61d567 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation index beda34cab..3a2978a72 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation index 10a55fa7f..732914b35 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation index a6dee9868..e04fe69a9 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation index 5619dd35d..7496e4e49 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation index c07198bb2..103b08a64 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation index b7e303498..f19fd40cc 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation index 6278a1544..6e6513ddb 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation index 9165ae7f0..376c5bb99 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation index 6b2418ba0..ef4c3eb9a 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation index ad5e47c29..a30d5c936 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation index 58664b314..dda5fddaf 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation index 0b883f0d9..7fb9db9fa 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation index 1df56738b..f3ae8f8d2 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation index bd75b9a9b..b637a89f4 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation index 77f765ca7..f83bb8ee9 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation index 30d7359da..828770f97 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation index 72a63808e..ef13ee0d8 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation index ad3b136de..1ee922174 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation index 8491244e6..01d2aa7d2 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation index f03efb743..614911048 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation index 8404b4041..43360d3ed 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation index a21255839..5e0f6e5a8 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation index 3e1d8d887..c1c66e496 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation index 8cabf7fcb..fb398be66 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/point_force/DATA/Par_file b/EXAMPLES/point_force/DATA/Par_file index ad8da35bd..22b65e3ae 100644 --- a/EXAMPLES/point_force/DATA/Par_file +++ b/EXAMPLES/point_force/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_EMC_model/DATA/Par_file b/EXAMPLES/regional_EMC_model/DATA/Par_file index 3655ac2de..da297f043 100644 --- a/EXAMPLES/regional_EMC_model/DATA/Par_file +++ b/EXAMPLES/regional_EMC_model/DATA/Par_file @@ -450,3 +450,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM b/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM index bbf05aeeb..885b00977 100644 --- a/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM +++ b/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM @@ -450,3 +450,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Greece_noise_small/DATA/Par_file b/EXAMPLES/regional_Greece_noise_small/DATA/Par_file index 40f5b1803..816a58a08 100644 --- a/EXAMPLES/regional_Greece_noise_small/DATA/Par_file +++ b/EXAMPLES/regional_Greece_noise_small/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Greece_small/DATA/Par_file b/EXAMPLES/regional_Greece_small/DATA/Par_file index 01b2852b3..73d25329f 100644 --- a/EXAMPLES/regional_Greece_small/DATA/Par_file +++ b/EXAMPLES/regional_Greece_small/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file b/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file index 04a9b4e6f..154b719fc 100644 --- a/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file +++ b/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file index 0b49f1757..b91c982ae 100644 --- a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file +++ b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 index 8cc029108..cec185420 100644 --- a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 +++ b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 index 0b49f1757..b91c982ae 100644 --- a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 +++ b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_MiddleEast/DATA/Par_file b/EXAMPLES/regional_MiddleEast/DATA/Par_file index defd61a91..06ecd9e45 100644 --- a/EXAMPLES/regional_MiddleEast/DATA/Par_file +++ b/EXAMPLES/regional_MiddleEast/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_s40rts/DATA/Par_file b/EXAMPLES/regional_s40rts/DATA/Par_file index b1d1f6851..7f981472c 100644 --- a/EXAMPLES/regional_s40rts/DATA/Par_file +++ b/EXAMPLES/regional_s40rts/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_sgloberani/DATA/Par_file b/EXAMPLES/regional_sgloberani/DATA/Par_file index 3e392f0c1..144f8441f 100644 --- a/EXAMPLES/regional_sgloberani/DATA/Par_file +++ b/EXAMPLES/regional_sgloberani/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_simultaneous_runs/DATA/Par_file b/EXAMPLES/regional_simultaneous_runs/DATA/Par_file index ea90c3dfb..02186559f 100644 --- a/EXAMPLES/regional_simultaneous_runs/DATA/Par_file +++ b/EXAMPLES/regional_simultaneous_runs/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_small_adjoint/DATA/Par_file b/EXAMPLES/regional_small_adjoint/DATA/Par_file index bd40baea2..33cab9d6a 100644 --- a/EXAMPLES/regional_small_adjoint/DATA/Par_file +++ b/EXAMPLES/regional_small_adjoint/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regular_kernel/DATA/Par_file b/EXAMPLES/regular_kernel/DATA/Par_file index db42b20bf..8d3c9a42f 100644 --- a/EXAMPLES/regular_kernel/DATA/Par_file +++ b/EXAMPLES/regular_kernel/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file b/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file index c4eee50e8..f924108c7 100644 --- a/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file +++ b/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file b/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file index c74f29023..12b90cfa0 100644 --- a/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file +++ b/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file @@ -430,3 +430,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + From 7b12e36b0e00f54b01fab58249bdd89188d3eaed Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Tue, 6 Aug 2024 11:21:50 -0500 Subject: [PATCH 03/19] add HDF5: compiler warnings --- src/meshfem3D/save_arrays_solver_hdf5.F90 | 11 +++-------- src/meshfem3D/save_model_meshfiles_hdf5.F90 | 3 +-- src/specfem3D/read_arrays_solver_hdf5.F90 | 21 ++------------------- src/specfem3D/read_forward_arrays_hdf5.F90 | 12 ++++++------ src/specfem3D/save_forward_arrays_hdf5.F90 | 12 ++++++------ src/specfem3D/save_kernels_hdf5.F90 | 14 +++++++------- 6 files changed, 25 insertions(+), 48 deletions(-) diff --git a/src/meshfem3D/save_arrays_solver_hdf5.F90 b/src/meshfem3D/save_arrays_solver_hdf5.F90 index 7e2fbb0f4..a375b3666 100644 --- a/src/meshfem3D/save_arrays_solver_hdf5.F90 +++ b/src/meshfem3D/save_arrays_solver_hdf5.F90 @@ -15,7 +15,6 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & nglob, iregion_code, & NCHUNKS,ABSORBING_CONDITIONS, & ROTATION,EXACT_MASS_MATRIX_FOR_ROTATION, & - OUTPUT_FILES,xstore_glob,ystore_glob,zstore_glob, & NPROCTOT, & ATT1,ATT2,ATT3 use regions_mesh_par2, only: & @@ -36,7 +35,7 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & nspec2D_xmin,nspec2D_xmax,nspec2D_ymin,nspec2D_ymax, & ispec_is_tiso, & tau_s_store,tau_e_store,Qmu_store, & - prname, nglob_oceans, nglob_xy + nglob_oceans, nglob_xy @@ -66,7 +65,7 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & logical :: if_collective = .true. ! processor dependent group names - character(len=64) :: gname_region, fname + character(len=64) :: gname_region ! offset arrays integer, dimension(0:NPROCTOT-1) :: offset_nnodes @@ -625,8 +624,7 @@ subroutine save_arrays_boundary_hdf5() ibelm_moho_top,ibelm_moho_bot,ibelm_400_top,ibelm_400_bot, & ibelm_670_top,ibelm_670_bot,normal_moho,normal_400,normal_670, & ispec2D_moho_top,ispec2D_moho_bot,ispec2D_400_top,ispec2D_400_bot, & - ispec2D_670_top,ispec2D_670_bot, & - prname + ispec2D_670_top,ispec2D_670_bot use manager_hdf5 #endif @@ -636,7 +634,6 @@ subroutine save_arrays_boundary_hdf5() #ifdef USE_HDF5 ! local parameters - integer :: ier logical :: if_collective = .true. @@ -825,7 +822,6 @@ subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & #ifdef USE_HDF5 ! local parameters character(len=64) :: gname_region - integer :: ier logical :: if_collective = .true. ! offset arrays @@ -1007,7 +1003,6 @@ subroutine get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & ! variables for HDF5 character(len=64) :: gname_region - integer :: ier logical :: if_col = .true. ! collective write ! file name diff --git a/src/meshfem3D/save_model_meshfiles_hdf5.F90 b/src/meshfem3D/save_model_meshfiles_hdf5.F90 index 2cef2f83b..0da0b8037 100644 --- a/src/meshfem3D/save_model_meshfiles_hdf5.F90 +++ b/src/meshfem3D/save_model_meshfiles_hdf5.F90 @@ -41,8 +41,7 @@ subroutine save_model_meshfiles_hdf5() use regions_mesh_par2, only: & rhostore,kappavstore,kappahstore,muvstore,muhstore,eta_anisostore, & - Qmu_store,Gc_prime_store,Gs_prime_store,mu0store, & - prname + Qmu_store,Gc_prime_store,Gs_prime_store,mu0store use model_heterogen_mantle_par diff --git a/src/specfem3D/read_arrays_solver_hdf5.F90 b/src/specfem3D/read_arrays_solver_hdf5.F90 index 68e623fb3..540270120 100644 --- a/src/specfem3D/read_arrays_solver_hdf5.F90 +++ b/src/specfem3D/read_arrays_solver_hdf5.F90 @@ -71,7 +71,7 @@ subroutine read_arrays_solver_hdf5(iregion_code, & #ifdef USE_HDF5 ! local parameters - integer :: ier,lnspec,lnglob + integer :: lnspec,lnglob ! group, dataset name character(len=64) :: gname_region @@ -327,27 +327,11 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) integer, intent(in) :: iregion_code ! local parameters - integer :: ierr,rank,i - ! ADIOS variables - integer(kind=8) :: sel - integer(kind=8), dimension(1) :: start, count - + integer :: ierr integer :: num_interfaces,max_nibool_interfaces, & num_phase_ispec,num_colors_outer,num_colors_inner integer :: nspec_inner,nspec_outer - integer(kind=8) :: offset_my_neighbors, offset_nibool_interfaces, & - offset_ibool_interfaces, offset_phase_ispec_inner, & - offset_num_elem_colors - - ! temporary read arrays - !integer, dimension(:),allocatable :: tmp_nibool_interfaces,tmp_my_neighbors - !integer, dimension(:,:),allocatable :: tmp_ibool_interfaces,tmp_phase_ispec_inner - !integer, dimension(:),allocatable :: tmp_num_elem_colors - - character(len=128) :: region_name - integer :: nglob_tmp,nspec_tmp - ! group, dataset name character(len=64) :: gname_region ! MPI variables @@ -698,7 +682,6 @@ subroutine read_mesh_databases_coupling_hdf5() #ifdef USE_HDF5 character(len=64) :: gname_region - integer :: ierr logical :: if_col = .true. ! MPI variables diff --git a/src/specfem3D/read_forward_arrays_hdf5.F90 b/src/specfem3D/read_forward_arrays_hdf5.F90 index 73c87f62b..903bb73c7 100644 --- a/src/specfem3D/read_forward_arrays_hdf5.F90 +++ b/src/specfem3D/read_forward_arrays_hdf5.F90 @@ -44,7 +44,7 @@ subroutine read_intermediate_forward_arrays_hdf5() integer :: neq_read,neq1_read ! MPI variables - integer :: ierr, info, comm + integer :: info, comm ! collective write logical :: if_col = .true. @@ -61,7 +61,7 @@ subroutine read_intermediate_forward_arrays_hdf5() integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 - character(len=MAX_STRING_LEN) :: file_name, dataset_name + character(len=MAX_STRING_LEN) :: file_name ! gather the offset arrays call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) @@ -181,7 +181,7 @@ subroutine read_forward_arrays_hdf5() integer :: b_neq_read, b_neq1_read ! MPI variables - integer :: ierr, info, comm + integer :: info, comm ! collective write logical :: if_col = .true. @@ -198,7 +198,7 @@ subroutine read_forward_arrays_hdf5() integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 - character(len=MAX_STRING_LEN) :: file_name, dataset_name + character(len=MAX_STRING_LEN) :: file_name ! gather the offset arrays call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) @@ -321,7 +321,7 @@ subroutine read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) integer :: b_neq_read, b_neq1_read ! MPI variables - integer :: ierr, info, comm + integer :: info, comm ! collective write logical :: if_col = .true. @@ -338,7 +338,7 @@ subroutine read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 - character(len=MAX_STRING_LEN) :: file_name, dataset_name + character(len=MAX_STRING_LEN) :: file_name ! gather the offset arrays call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) diff --git a/src/specfem3D/save_forward_arrays_hdf5.F90 b/src/specfem3D/save_forward_arrays_hdf5.F90 index 29b36d84b..7f6b42b55 100644 --- a/src/specfem3D/save_forward_arrays_hdf5.F90 +++ b/src/specfem3D/save_forward_arrays_hdf5.F90 @@ -43,7 +43,7 @@ subroutine save_intermediate_forward_arrays_hdf5() #ifdef USE_HDF5 ! MPI variables - integer :: ierr, info, comm + integer :: info, comm ! collective write logical :: if_col = .true. @@ -225,7 +225,7 @@ subroutine save_forward_arrays_hdf5() #ifdef USE_HDF5 ! MPI variables - integer :: ierr, info, comm + integer :: info, comm ! collective write logical :: if_col = .true. @@ -242,7 +242,7 @@ subroutine save_forward_arrays_hdf5() integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 - character(len=MAX_STRING_LEN) :: file_name, dataset_name + character(len=MAX_STRING_LEN) :: file_name file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/save_forward_arrays.h5' @@ -387,7 +387,7 @@ subroutine save_forward_arrays_undoatt_hdf5() #ifdef USE_HDF5 ! MPI variables - integer :: ierr, info, comm + integer :: info, comm ! collective write logical :: if_col = .true. @@ -574,10 +574,10 @@ subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_cru ! local parameters integer :: ier - real(kind=CUSTOM_REAL) :: scaleval1,scaleval2,scale_factor_r + real(kind=CUSTOM_REAL) :: scaleval1,scale_factor_r real(kind=CUSTOM_REAL),dimension(:,:,:,:),allocatable :: temp_store real(kind=CUSTOM_REAL),dimension(:,:,:,:),allocatable :: muv_shifted,muh_shifted - integer :: iregion_code,nspec,nglob,i,j,k,ispec + integer :: i,j,k,ispec ! collective write logical :: if_col = .true. diff --git a/src/specfem3D/save_kernels_hdf5.F90 b/src/specfem3D/save_kernels_hdf5.F90 index a9a3e0eea..0dfdfc49e 100644 --- a/src/specfem3D/save_kernels_hdf5.F90 +++ b/src/specfem3D/save_kernels_hdf5.F90 @@ -43,7 +43,7 @@ subroutine write_kernels_strength_noise_hdf5() ! local parameters character(len=MAX_STRING_LEN) :: file_name - integer :: ierr, info, comm + integer :: info, comm logical :: if_col = .true. ! gather the number of elements in each region @@ -121,7 +121,7 @@ subroutine write_kernels_cm_ani_hdf5(alphav_kl_crust_mantle,alphah_kl_crust_mant #ifdef USE_HDF5 ! local parameters character(len=MAX_STRING_LEN) :: file_name - integer :: ierr, info, comm + integer :: info, comm logical :: if_col = .true. ! offset array @@ -323,7 +323,7 @@ subroutine write_kernels_cm_iso_hdf5(mu_kl_crust_mantle, kappa_kl_crust_mantle, ! local parameters character(len=MAX_STRING_LEN) :: file_name - integer :: ierr, info, comm + integer :: info, comm logical :: if_col = .true. ! checks if anything to do @@ -407,7 +407,7 @@ subroutine write_kernels_oc_hdf5() ! local parameters character(len=MAX_STRING_LEN) :: file_name - integer :: ierr, info, comm + integer :: info, comm logical :: if_col = .true. ! gather the number of elements in each region @@ -471,7 +471,7 @@ subroutine write_kernels_ic_hdf5() ! local parameters character(len=MAX_STRING_LEN) :: file_name - integer :: ierr, info, comm + integer :: info, comm logical :: if_col = .true. ! gather the number of elements in each region @@ -543,7 +543,7 @@ subroutine write_kernels_boundary_kl_hdf5() ! local parameters character(len=MAX_STRING_LEN) :: file_name - integer :: ierr, info, comm + integer :: info, comm logical :: if_col = .true. if (.not. SAVE_KERNELS_BOUNDARY) return @@ -624,7 +624,7 @@ subroutine write_kernels_Hessian_hdf5() ! local parameters character(len=MAX_STRING_LEN) :: file_name - integer :: ierr, info, comm + integer :: info, comm logical :: if_col = .true. ! gather the number of elements in each region From fc3cd815c15c3b260f4267979ff4221d15476771 Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Tue, 6 Aug 2024 14:43:30 -0500 Subject: [PATCH 04/19] add HDF5: debug for boundary array --- src/meshfem3D/save_arrays_solver_hdf5.F90 | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/meshfem3D/save_arrays_solver_hdf5.F90 b/src/meshfem3D/save_arrays_solver_hdf5.F90 index a375b3666..058ca220f 100644 --- a/src/meshfem3D/save_arrays_solver_hdf5.F90 +++ b/src/meshfem3D/save_arrays_solver_hdf5.F90 @@ -994,6 +994,8 @@ subroutine get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & #ifdef USE_HDF5 ! local parameters integer, dimension(0:NPROCTOT-1) :: offset_num_abs_boundary_faces + ! MPI parameters + integer :: comm, info ! dummy arrays integer, dimension(1,1), parameter :: i2d_dummy = reshape((/0/),(/1,1/)) @@ -1005,6 +1007,15 @@ subroutine get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & character(len=64) :: gname_region logical :: if_col = .true. ! collective write + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT) + ! file name name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'stacey.h5' ! group name From 99af7b3eb8a4f556a4d82fe6bbf806ea001dc57b Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Tue, 6 Aug 2024 13:44:11 -0600 Subject: [PATCH 05/19] compiler warning --- src/meshfem3D/save_model_meshfiles_hdf5.F90 | 9 ++++----- src/specfem3D/read_arrays_solver_hdf5.F90 | 16 ++++++++-------- src/specfem3D/save_forward_arrays_hdf5.F90 | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/meshfem3D/save_model_meshfiles_hdf5.F90 b/src/meshfem3D/save_model_meshfiles_hdf5.F90 index 0da0b8037..df1077d93 100644 --- a/src/meshfem3D/save_model_meshfiles_hdf5.F90 +++ b/src/meshfem3D/save_model_meshfiles_hdf5.F90 @@ -28,9 +28,10 @@ subroutine save_model_meshfiles_hdf5() + use constants +#ifdef USE_HDF5 ! outputs model files in binary format - use constants use shared_parameters, only: R_PLANET,RHOAV,LOCAL_PATH use meshfem_par, only: nspec,iregion_code,NPROCTOT @@ -45,19 +46,17 @@ subroutine save_model_meshfiles_hdf5() use model_heterogen_mantle_par -#ifdef USE_HDF5 use manager_hdf5 #endif implicit none +#ifdef USE_HDF5 ! local parameters integer :: i,j,k,ispec,ier real(kind=CUSTOM_REAL) :: scaleval1,scaleval2,scaleval,scale_GPa real(kind=CUSTOM_REAL),dimension(:,:,:,:),allocatable :: temp_store -#ifdef USE_HDF5 - ! dset_name and group_name character(len=64) :: dset_name, gname_region @@ -278,4 +277,4 @@ subroutine save_model_meshfiles_hdf5() #endif -end subroutine save_model_meshfiles_hdf5 \ No newline at end of file +end subroutine save_model_meshfiles_hdf5 diff --git a/src/specfem3D/read_arrays_solver_hdf5.F90 b/src/specfem3D/read_arrays_solver_hdf5.F90 index 540270120..997043c2f 100644 --- a/src/specfem3D/read_arrays_solver_hdf5.F90 +++ b/src/specfem3D/read_arrays_solver_hdf5.F90 @@ -16,11 +16,11 @@ subroutine read_arrays_solver_hdf5(iregion_code, & use constants_solver + +#ifdef USE_HDF5 use specfem_par, only: & ABSORBING_CONDITIONS, & LOCAL_PATH,ABSORBING_CONDITIONS - -#ifdef USE_HDF5 use manager_hdf5 #endif @@ -325,6 +325,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) implicit none integer, intent(in) :: iregion_code +#ifdef USE_HDF5 ! local parameters integer :: ierr @@ -348,7 +349,6 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_colors_outer integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_colors_inner -#ifdef USE_HDF5 call h5_initialize() @@ -647,6 +647,7 @@ end subroutine read_mesh_databases_MPI_hdf5 subroutine read_mesh_databases_coupling_hdf5() +#ifdef USE_HDF5 use constants use meshfem_par, only: & @@ -673,7 +674,6 @@ subroutine read_mesh_databases_coupling_hdf5() use specfem_par_trinfinite use specfem_par_infinite -#ifdef USE_HDF5 use manager_hdf5 #endif @@ -1222,17 +1222,15 @@ end subroutine read_mesh_databases_stacey_hdf5 subroutine read_attenuation_hdf5(iregion_code, factor_common, scale_factor, tau_s, vnspec, f_c_source) use constants_solver - use specfem_par, only: ATTENUATION_VAL,LOCAL_PATH #ifdef USE_HDF5 + use specfem_par, only: ATTENUATION_VAL,LOCAL_PATH use manager_hdf5 #endif implicit none integer,intent(in) :: iregion_code - ! offset - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nelems integer,intent(in) :: vnspec real(kind=CUSTOM_REAL), dimension(ATT1_VAL,ATT2_VAL,ATT3_VAL,vnspec),intent(inout) :: scale_factor @@ -1241,6 +1239,8 @@ subroutine read_attenuation_hdf5(iregion_code, factor_common, scale_factor, tau_ double precision,intent(inout) :: f_c_source #ifdef USE_HDF5 + ! offset + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nelems ! hdf5 variables logical :: if_col = .true. ! read in collective mode character(len=64) :: gname_region @@ -1302,4 +1302,4 @@ subroutine read_attenuation_hdf5(iregion_code, factor_common, scale_factor, tau_ stop #endif -end subroutine read_attenuation_hdf5 \ No newline at end of file +end subroutine read_attenuation_hdf5 diff --git a/src/specfem3D/save_forward_arrays_hdf5.F90 b/src/specfem3D/save_forward_arrays_hdf5.F90 index 7f6b42b55..8d6cd8e38 100644 --- a/src/specfem3D/save_forward_arrays_hdf5.F90 +++ b/src/specfem3D/save_forward_arrays_hdf5.F90 @@ -556,12 +556,12 @@ end subroutine save_forward_arrays_undoatt_hdf5 subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_crust_mantle,factor_scale_relaxed_inner_core) use constants - use shared_parameters, only: R_PLANET,RHOAV,LOCAL_PATH,TRANSVERSE_ISOTROPY use specfem_par_crustmantle use specfem_par_innercore #ifdef USE_HDF5 + use shared_parameters, only: R_PLANET,RHOAV,LOCAL_PATH,TRANSVERSE_ISOTROPY use manager_hdf5 #endif From 282cc138ef9ec446d5c53317d053095b01cee175 Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Tue, 6 Aug 2024 15:02:43 -0500 Subject: [PATCH 06/19] add HDF5: unnecessary variables --- src/meshfem3D/create_regions_mesh.F90 | 1 - 1 file changed, 1 deletion(-) diff --git a/src/meshfem3D/create_regions_mesh.F90 b/src/meshfem3D/create_regions_mesh.F90 index 676fa75ec..3b661bc36 100644 --- a/src/meshfem3D/create_regions_mesh.F90 +++ b/src/meshfem3D/create_regions_mesh.F90 @@ -463,7 +463,6 @@ subroutine create_regions_mesh(npointot, & NSPEC2D_TOP,NSPEC2D_BOTTOM) else if (HDF5_ENABLED) then call save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & - NSPEC2DMAX_XMIN_XMAX, NSPEC2DMAX_YMIN_YMAX, & NSPEC2D_TOP,NSPEC2D_BOTTOM) else call save_arrays_solver(idoubling,ibool,xstore,ystore,zstore, & From 0ab985c5a4ed37a7b20751c04822fa173327481b Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Tue, 6 Aug 2024 15:12:45 -0500 Subject: [PATCH 07/19] reduce input variable --- src/specfem3D/read_forward_arrays.F90 | 2 +- src/specfem3D/read_forward_arrays_hdf5.F90 | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/specfem3D/read_forward_arrays.F90 b/src/specfem3D/read_forward_arrays.F90 index 2d4153d6c..4e5d35980 100644 --- a/src/specfem3D/read_forward_arrays.F90 +++ b/src/specfem3D/read_forward_arrays.F90 @@ -58,7 +58,7 @@ subroutine read_forward_arrays_startrun() if (ADIOS_FOR_FORWARD_ARRAYS) then call read_intermediate_forward_arrays_adios() else if (HDF5_ENABLED) then - call read_intermediate_forward_arrays_hdf5(neq) + call read_intermediate_forward_arrays_hdf5() else write(outputname,"('dump_all_arrays',i6.6)") myrank outputname = trim(LOCAL_TMP_PATH) // '/' // outputname(1:len_trim(outputname)) diff --git a/src/specfem3D/read_forward_arrays_hdf5.F90 b/src/specfem3D/read_forward_arrays_hdf5.F90 index 903bb73c7..b5288e21d 100644 --- a/src/specfem3D/read_forward_arrays_hdf5.F90 +++ b/src/specfem3D/read_forward_arrays_hdf5.F90 @@ -136,17 +136,6 @@ subroutine read_intermediate_forward_arrays_hdf5() if (FULL_GRAVITY_VAL) then call h5_read_dataset_scalar_collect_hyperslab('neq', neq_read, (/myrank/), if_col) call h5_read_dataset_scalar_collect_hyperslab('neq1', neq1_read, (/myrank/), if_col) - - ! check if array sizes match - if (neq_read /= neq) then - print *,'Error reading forward array for startrun: rank ',myrank,'has read neq =',neq_read,' - shoud be ',neq - call exit_MPI(myrank,'Invalid forward array neq for startrun') - endif - if (neq1_read /= neq1) then - print *,'Error reading forward array for startrun: rank ',myrank,'has read neq1 =',neq1_read,' - shoud be ',neq1 - call exit_MPI(myrank,'Invalid forward array neq1 for startrun') - endif - call h5_read_dataset_collect_hyperslab('pgrav1', pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), if_col) endif From aa3b112e576639ac06b124dd35f24f57f926190e Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Tue, 6 Aug 2024 22:34:52 -0500 Subject: [PATCH 08/19] missing arrays --- src/specfem3D/read_arrays_solver_hdf5.F90 | 4 ++++ src/specfem3D/save_kernels.F90 | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/specfem3D/read_arrays_solver_hdf5.F90 b/src/specfem3D/read_arrays_solver_hdf5.F90 index 997043c2f..2bf02ba02 100644 --- a/src/specfem3D/read_arrays_solver_hdf5.F90 +++ b/src/specfem3D/read_arrays_solver_hdf5.F90 @@ -882,6 +882,10 @@ subroutine read_mesh_databases_coupling_hdf5() call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_inner_core, (/sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) ! ibelm_ymax call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_inner_core, (/sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + ! ibelm_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_inner_core, (/sum(offset_nspec2D_top(0:myrank-1))/), if_col) + ! ibelm_bottom + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_inner_core, (/sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) ! close group call h5_close_group() diff --git a/src/specfem3D/save_kernels.F90 b/src/specfem3D/save_kernels.F90 index 5f7e0cf37..9017ec8c7 100644 --- a/src/specfem3D/save_kernels.F90 +++ b/src/specfem3D/save_kernels.F90 @@ -1753,7 +1753,7 @@ subroutine save_kernels_outer_core(rhostore_outer_core,kappavstore_outer_core,rh if (ADIOS_FOR_KERNELS) then call write_kernels_oc_adios() else if (HDF5_ENABLED) then - call write_kernel_oc_hdf5() + call write_kernels_oc_hdf5() else call create_name_database(prname,myrank,IREGION_OUTER_CORE,LOCAL_TMP_PATH) From f76163d5bbe8646613f1551f6446c150522c2072 Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Wed, 7 Aug 2024 09:28:28 -0500 Subject: [PATCH 09/19] add HDF5: debug kernel save --- src/specfem3D/save_forward_arrays_hdf5.F90 | 67 ++++++++++++++-------- 1 file changed, 43 insertions(+), 24 deletions(-) diff --git a/src/specfem3D/save_forward_arrays_hdf5.F90 b/src/specfem3D/save_forward_arrays_hdf5.F90 index 8d6cd8e38..65ed6aa21 100644 --- a/src/specfem3D/save_forward_arrays_hdf5.F90 +++ b/src/specfem3D/save_forward_arrays_hdf5.F90 @@ -63,9 +63,9 @@ subroutine save_intermediate_forward_arrays_hdf5() character(len=MAX_STRING_LEN) :: file_name ! gather the offset arrays - call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) - call gather_all_all_singlei(size(displ_inner_core), offset_nglob_oc, NPROCTOT_VAL) - call gather_all_all_singlei(size(displ_outer_core), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_crust_mantle,2), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core,2), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core,1), offset_nglob_oc, NPROCTOT_VAL) call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) if (ROTATION_VAL) then @@ -98,9 +98,9 @@ subroutine save_intermediate_forward_arrays_hdf5() call h5_create_dataset_gen('displ_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('veloc_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('accel_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('displ_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('veloc_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('accel_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('accel_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) @@ -153,9 +153,9 @@ subroutine save_intermediate_forward_arrays_hdf5() call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) @@ -244,6 +244,23 @@ subroutine save_forward_arrays_hdf5() character(len=MAX_STRING_LEN) :: file_name + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle,2), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core,2), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core,1), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/save_forward_arrays.h5' ! get MPI parameters @@ -257,12 +274,14 @@ subroutine save_forward_arrays_hdf5() ! create file and datasets by myrank==0 if (myrank == 0) then + call h5_create_file(file_name) + call h5_create_dataset_gen('displ_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('veloc_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('accel_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('displ_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('veloc_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('accel_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('accel_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) @@ -316,9 +335,9 @@ subroutine save_forward_arrays_hdf5() call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) @@ -407,9 +426,9 @@ subroutine save_forward_arrays_undoatt_hdf5() character(len=MAX_STRING_LEN) :: file_name ! gather the offset arrays - call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) - call gather_all_all_singlei(size(displ_inner_core), offset_nglob_oc, NPROCTOT_VAL) - call gather_all_all_singlei(size(displ_outer_core), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_crust_mantle,2), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core,2), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core,1), offset_nglob_oc, NPROCTOT_VAL) call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) if (ROTATION_VAL) then @@ -443,9 +462,9 @@ subroutine save_forward_arrays_undoatt_hdf5() call h5_create_dataset_gen('displ_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('veloc_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('accel_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('displ_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('veloc_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('accel_outer_core', (/NDIM, sum(offset_nglob_oc)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('accel_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) @@ -498,9 +517,9 @@ subroutine save_forward_arrays_undoatt_hdf5() call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/0, sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) From 99f47864dbe22063f7a43f0c6d023308afd33050 Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Wed, 7 Aug 2024 10:03:33 -0500 Subject: [PATCH 10/19] add check for save_regular_kl with hdf5 and adios --- src/specfem3D/initialize_simulation.F90 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/specfem3D/initialize_simulation.F90 b/src/specfem3D/initialize_simulation.F90 index f2c81d7f2..ca8e6f59e 100644 --- a/src/specfem3D/initialize_simulation.F90 +++ b/src/specfem3D/initialize_simulation.F90 @@ -569,6 +569,8 @@ subroutine initialize_simulation_check() call exit_mpi(myrank,'SAVE_AZIMUTHAL_ANISO_KL_ONLY needs anisotropic kernel flag ANISOTROPIC_KL set to .true.') if (SAVE_REGULAR_KL .and. SAVE_AZIMUTHAL_ANISO_KL_ONLY) & call exit_mpi(myrank,'SAVE_AZIMUTHAL_ANISO_KL_ONLY not implemented yet for SAVE_REGULAR_KL kernels') + if (SAVE_REGULAR_KL .and. (HDF5_ENABLED .or. ADIOS_FOR_KERNELS)) & + call exit_mpi(myrank,'SAVE_REGULAR_KL not implemented yet for HDF5 or ADIOS output') endif ! check for GPU runs From b1f0e55f93e281fdbdc59c8858ea0dbb1cd6d8ea Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Wed, 7 Aug 2024 10:04:19 -0500 Subject: [PATCH 11/19] add HDF5: debug reading forward array --- src/specfem3D/read_forward_arrays_hdf5.F90 | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/specfem3D/read_forward_arrays_hdf5.F90 b/src/specfem3D/read_forward_arrays_hdf5.F90 index b5288e21d..04954ff9e 100644 --- a/src/specfem3D/read_forward_arrays_hdf5.F90 +++ b/src/specfem3D/read_forward_arrays_hdf5.F90 @@ -190,9 +190,9 @@ subroutine read_forward_arrays_hdf5() character(len=MAX_STRING_LEN) :: file_name ! gather the offset arrays - call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) - call gather_all_all_singlei(size(displ_inner_core), offset_nglob_oc, NPROCTOT_VAL) - call gather_all_all_singlei(size(displ_outer_core), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_crust_mantle,2), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core,2), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core,1), offset_nglob_oc, NPROCTOT_VAL) call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) if (ROTATION_VAL) then @@ -222,13 +222,13 @@ subroutine read_forward_arrays_hdf5() ! read the arrays call h5_read_dataset_collect_hyperslab('displ_crust_mantle', b_displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('displ_inner_core', b_displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', b_veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('veloc_inner_core', b_veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) @@ -330,9 +330,9 @@ subroutine read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) character(len=MAX_STRING_LEN) :: file_name ! gather the offset arrays - call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) - call gather_all_all_singlei(size(displ_inner_core), offset_nglob_oc, NPROCTOT_VAL) - call gather_all_all_singlei(size(displ_outer_core), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_crust_mantle,2), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core,2), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core,1), offset_nglob_oc, NPROCTOT_VAL) call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) if (ROTATION_VAL) then @@ -363,13 +363,13 @@ subroutine read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) ! read the arrays call h5_read_dataset_collect_hyperslab('displ_crust_mantle', b_displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('displ_inner_core', b_displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', b_veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('veloc_inner_core', b_veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) From bdfc990cdf8e8b38ba268e0ec6251666ac5630d4 Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Sun, 11 Aug 2024 17:52:07 -0500 Subject: [PATCH 12/19] add HDF5: surface movie done --- src/specfem3D/finalize_simulation.F90 | 3 + src/specfem3D/prepare_movie.f90 | 24 +- src/specfem3D/rules.mk | 24 +- src/specfem3D/specfem3D_par.F90 | 55 + src/specfem3D/write_movie_output.f90 | 172 ++- src/specfem3D/write_movie_surface.f90 | 2 + src/specfem3D/write_movie_surface_hdf5.F90 | 565 +++++++ src/specfem3D/write_movie_volume_hdf5.F90 | 1549 ++++++++++++++++++++ 8 files changed, 2329 insertions(+), 65 deletions(-) create mode 100644 src/specfem3D/write_movie_surface_hdf5.F90 create mode 100644 src/specfem3D/write_movie_volume_hdf5.F90 diff --git a/src/specfem3D/finalize_simulation.F90 b/src/specfem3D/finalize_simulation.F90 index 1ee16f937..42e277a33 100644 --- a/src/specfem3D/finalize_simulation.F90 +++ b/src/specfem3D/finalize_simulation.F90 @@ -292,6 +292,9 @@ subroutine finalize_simulation_cleanup() if (MOVIE_SURFACE) then deallocate(store_val_ux,store_val_uy,store_val_uz) deallocate(store_val_ux_all,store_val_uy_all,store_val_uz_all) + if (HDF5_ENABLED) then + call movie_surface_finalize_hdf5() + endif endif if (MOVIE_VOLUME) then deallocate(nu_3dmovie) diff --git a/src/specfem3D/prepare_movie.f90 b/src/specfem3D/prepare_movie.f90 index 168d85a3b..765d5ffa1 100644 --- a/src/specfem3D/prepare_movie.f90 +++ b/src/specfem3D/prepare_movie.f90 @@ -65,14 +65,18 @@ subroutine prepare_movie_surface() ! note: for noise tomography, must NOT be coarse (have to be saved on all GLL points) if (MOVIE_COARSE) then ! checks setup - if (NGLLX /= NGLLY) & + if (NGLLX /= NGLLY .and. .not. HDF5_ENABLED) & call exit_MPI(myrank,'MOVIE_COARSE together with MOVIE_SURFACE requires NGLLX=NGLLY') ! number of points nmovie_points = 2 * 2 * NSPEC2D_TOP(IREGION_CRUST_MANTLE) NIT = NGLLX - 1 else ! number of points - nmovie_points = NGLLX * NGLLY * NSPEC2D_TOP(IREGION_CRUST_MANTLE) + if (.not. HDF5_ENABLED) then + nmovie_points = NGLLX * NGLLY * NSPEC2D_TOP(IREGION_CRUST_MANTLE) + else ! HDF5 + nmovie_points = 4 * (NGLLX-1) * (NGLLY-1) * NSPEC2D_TOP(IREGION_CRUST_MANTLE) + end if NIT = 1 endif @@ -81,7 +85,11 @@ subroutine prepare_movie_surface() ! those arrays are not necessary for noise tomography, so only allocate them in MOVIE_SURFACE case ! writes out movie point locations to file - call write_movie_surface_mesh() + if (HDF5_ENABLED) then + call write_movie_surface_mesh_hdf5() + else + call write_movie_surface_mesh() + endif ! allocates movie surface arrays for wavefield values allocate(store_val_ux(nmovie_points), & @@ -90,6 +98,7 @@ subroutine prepare_movie_surface() if (ier /= 0 ) call exit_MPI(myrank,'Error allocating movie surface arrays') ! allocates arrays for gathering wavefield values + ! TODO ADD HDF5 (store_val_*_all is not used) if (myrank == 0) then ! only main needs full arrays allocate(store_val_ux_all(nmovie_points,0:NPROCTOT_VAL-1), & @@ -184,8 +193,13 @@ subroutine prepare_movie_volume() allocate(nu_3dmovie(3,3,npoints_3dmovie),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating nu for 3D movie') - call write_movie_volume_mesh(nu_3dmovie,num_ibool_3dmovie,mask_3dmovie,mask_ibool_3dmovie, & - muvstore_crust_mantle_3dmovie,npoints_3dmovie) + if (HDF5_ENABLED) then + call write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovie,mask_ibool_3dmovie, & + muvstore_crust_mantle_3dmovie,npoints_3dmovie) + else + call write_movie_volume_mesh(nu_3dmovie,num_ibool_3dmovie,mask_3dmovie,mask_ibool_3dmovie, & + muvstore_crust_mantle_3dmovie,npoints_3dmovie) + endif if (myrank == 0) then write(IMAIN,*) ' Writing to movie3D*** files on local disk databases directory' diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk index df4a71310..486ac866f 100644 --- a/src/specfem3D/rules.mk +++ b/src/specfem3D/rules.mk @@ -138,7 +138,9 @@ specfem3D_SOLVER_OBJECTS += \ $O/update_displacement_Newmark.solverstatic.o \ $O/write_movie_output.solverstatic.o \ $O/write_movie_volume.solverstatic.o \ + $O/write_movie_volume_hdf5.solverstatic.o \ $O/write_movie_surface.solverstatic.o \ + $O/write_movie_surface_hdf5.solverstatic.o \ $O/write_output_ASCII.solverstatic.o \ $O/write_output_SAC.solverstatic.o \ $O/write_seismograms.solverstatic.o \ @@ -353,6 +355,17 @@ ifeq ($(PETSC),yes) specfem3D_MODULES += $(FC_MODDIR)/siem_solver_petsc.$(FC_MODEXT) endif + +## +## HDF5 +## + +ifeq ($(HDF5),yes) +specfem3D_MODULES += \ + $(FC_MODDIR)/specfem_par_movie_hdf5.$(FC_MODEXT) \ + $(EMPTY_MACRO) +endif + ####################################### #### @@ -423,6 +436,7 @@ $O/SIEM_solver_mpi.solverstatic.o: $O/SIEM_math_library.shared.o $O/SIEM_solver_petsc.solverstatic.o: $O/SIEM_math_library.shared.o $O/SIEM_compute_seismograms.solverstatic.o: $O/SIEM_math_library.shared.o + ### ### specfem3D - optimized flags and dependence on values from mesher here ### @@ -483,8 +497,8 @@ $O/%.checknetcdf.o: $S/%.f90 $O/shared_par.shared_module.o $O/specfem3D_par.solv ### ### HDF5 ### -$O/%.spec_hdf5_module.o: $S/%.F90 $O/shared_par.shared_module.o $O/specfem3D_par.solverstatic_module.o $O/hdf5_manager.shared_hdf5_module.o - ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(HDF5_INCLUDE) -c -o $@ $< - -$O/%.spec_hdf5_module.o: $S/%.f90 $O/shared_par.shared_module.o $O/specfem3D_par.solverstatic_module.o $O/hdf5_manager.shared_hdf5_module.o - ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(HDF5_INCLUDE) -c -o $@ $< \ No newline at end of file +#$O/%.spec_hdf5_module.o: $S/%.F90 $O/shared_par.shared_module.o $O/specfem3D_par.solverstatic_module.o $O/hdf5_manager.shared_hdf5_module.o +# ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(HDF5_INCLUDE) -c -o $@ $< +# +#$O/%.spec_hdf5_module.o: $S/%.f90 $O/shared_par.shared_module.o $O/specfem3D_par.solverstatic_module.o $O/hdf5_manager.shared_hdf5_module.o +# ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(HDF5_INCLUDE) -c -o $@ $< \ No newline at end of file diff --git a/src/specfem3D/specfem3D_par.F90 b/src/specfem3D/specfem3D_par.F90 index 9bbadf92e..ccee18320 100644 --- a/src/specfem3D/specfem3D_par.F90 +++ b/src/specfem3D/specfem3D_par.F90 @@ -1521,4 +1521,59 @@ end module my_libxsmm #endif +!===================================================================== + +#ifdef USE_HDF5 + + module specfem_par_movie_hdf5 + + use constants_solver, only: NPROCTOT_VAL,MAX_STRING_LEN,CUSTOM_REAL + use manager_hdf5 + + + logical :: if_col = .true. + integer :: info, comm + character(len=MAX_STRING_LEN) :: file_name, group_name + + ! surface movie + integer :: npoints_surf_mov_all_proc + integer, dimension(:), allocatable :: offset_poin + + ! volume movie + + + ! xdmf + integer :: xdmf_surf = 30000 + integer :: xdmf_vol = 30001 + integer :: surf_xdmf_pos = 0 + integer :: vol_xdmf_pos = 0 +contains + + !------------------------------------------- + + function r2c(k) result(str) + + ! "Convert an real to string." + + implicit none + real(kind=CUSTOM_REAL), intent(in) :: k + character(len=20) str + write (str, *) k + str = adjustl(str) + end function r2c + + !function i2c(k) result(str) + !! "Convert an integer to string." + ! implicit none + ! integer, intent(in) :: k + ! character(len=20) str + ! write (str, "(i20)") k + ! str = adjustl(str) + !end function i2c + +! + + end module specfem_par_movie_hdf5 + +#endif diff --git a/src/specfem3D/write_movie_output.f90 b/src/specfem3D/write_movie_output.f90 index a244bc8bd..bd931141d 100644 --- a/src/specfem3D/write_movie_output.f90 +++ b/src/specfem3D/write_movie_output.f90 @@ -30,7 +30,8 @@ subroutine write_movie_output() use specfem_par, only: deltat,it,myrank,Mesh_pointer, & GPU_MODE,NTSTEP_BETWEEN_FRAMES, & MOVIE_START,MOVIE_STOP,MOVIE_SURFACE,MOVIE_VOLUME,MOVIE_VOLUME_TYPE, & - scale_displ,scale_veloc + scale_displ,scale_veloc, & + HDF5_ENABLED use specfem_par_crustmantle, only: displ_crust_mantle,veloc_crust_mantle,accel_crust_mantle, & eps_trace_over_3_crust_mantle,epsilondev_xx_crust_mantle,epsilondev_xy_crust_mantle,epsilondev_xz_crust_mantle, & @@ -75,9 +76,13 @@ subroutine write_movie_output() endif endif - ! TODO ADD HDF5 and IO_SERVER + ! TODO ADD IO_SERVER ! save velocity here to avoid static offset on displacement for movies - call write_movie_surface() + if (HDF5_ENABLED) then + call write_movie_surface_hdf5() + else + call write_movie_surface() + endif ! executes an external script on the node if (RUN_EXTERNAL_MOVIE_SCRIPT) then @@ -110,14 +115,14 @@ subroutine write_movie_output() ! integrates strain call movie_volume_integrate_strain(deltat,NSPEC_CRUST_MANTLE_3DMOVIE, & - eps_trace_over_3_crust_mantle, & - epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle, & - epsilondev_xy_crust_mantle,epsilondev_xz_crust_mantle, & - epsilondev_yz_crust_mantle, & - Ieps_trace_over_3_crust_mantle, & - Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle, & - Iepsilondev_xy_crust_mantle,Iepsilondev_xz_crust_mantle, & - Iepsilondev_yz_crust_mantle) + eps_trace_over_3_crust_mantle, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle, & + epsilondev_xy_crust_mantle,epsilondev_xz_crust_mantle, & + epsilondev_yz_crust_mantle, & + Ieps_trace_over_3_crust_mantle, & + Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle, & + Iepsilondev_xy_crust_mantle,Iepsilondev_xz_crust_mantle, & + Iepsilondev_yz_crust_mantle) endif ! file output @@ -135,21 +140,38 @@ subroutine write_movie_output() epsilondev_xy_crust_mantle,epsilondev_xz_crust_mantle, & epsilondev_yz_crust_mantle) endif - ! TODO ADD HDF5 and IO_SERVER - call write_movie_volume_strains(NSPEC_CRUST_MANTLE_STRAIN_ONLY, & - eps_trace_over_3_crust_mantle, & - NSPEC_CRUST_MANTLE_STR_OR_ATT, & - epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & - epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle) + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_strains_hdf5(NSPEC_CRUST_MANTLE_STRAIN_ONLY, & + eps_trace_over_3_crust_mantle, & + NSPEC_CRUST_MANTLE_STR_OR_ATT, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle) + + else + call write_movie_volume_strains(NSPEC_CRUST_MANTLE_STRAIN_ONLY, & + eps_trace_over_3_crust_mantle, & + NSPEC_CRUST_MANTLE_STR_OR_ATT, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle) + endif case (2, 3) - ! TODO ADD HDF5 and IO_SERVER + ! TODO ADD IO_SERVER ! output the Time Integral of Strain, or \mu*TIS - call write_movie_volume_strains(NSPEC_CRUST_MANTLE_3DMOVIE, & - Ieps_trace_over_3_crust_mantle, & - NSPEC_CRUST_MANTLE_3DMOVIE, & - Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle,Iepsilondev_xy_crust_mantle, & - Iepsilondev_xz_crust_mantle,Iepsilondev_yz_crust_mantle) + if (HDF5_ENABLED) then + call write_movie_volume_strains_hdf5(NSPEC_CRUST_MANTLE_3DMOVIE, & + Ieps_trace_over_3_crust_mantle, & + NSPEC_CRUST_MANTLE_3DMOVIE, & + Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle,Iepsilondev_xy_crust_mantle, & + Iepsilondev_xz_crust_mantle,Iepsilondev_yz_crust_mantle) + else + call write_movie_volume_strains(NSPEC_CRUST_MANTLE_3DMOVIE, & + Ieps_trace_over_3_crust_mantle, & + NSPEC_CRUST_MANTLE_3DMOVIE, & + Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle,Iepsilondev_xy_crust_mantle, & + Iepsilondev_xz_crust_mantle,Iepsilondev_yz_crust_mantle) + endif case (4) ! output divergence and curl in whole volume @@ -170,41 +192,66 @@ subroutine write_movie_output() call transfer_fields_oc_from_device(NGLOB_OUTER_CORE, & displ_outer_core,veloc_outer_core,accel_outer_core,Mesh_pointer) endif - ! TODO ADD HDF5 and IO_SERVER - call write_movie_volume_divcurl(NSPEC_CRUST_MANTLE_STRAIN_ONLY,eps_trace_over_3_crust_mantle, & - div_displ_outer_core, & - accel_outer_core,kappavstore_outer_core,rhostore_outer_core,ibool_outer_core, & - NSPEC_INNER_CORE_STRAIN_ONLY,eps_trace_over_3_inner_core, & - NSPEC_CRUST_MANTLE_STR_OR_ATT, & - epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & - epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, & - NSPEC_INNER_CORE_STR_OR_ATT, & - epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, & - epsilondev_xz_inner_core,epsilondev_yz_inner_core) - + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_divcurl_hdf5(NSPEC_CRUST_MANTLE_STRAIN_ONLY,eps_trace_over_3_crust_mantle, & + div_displ_outer_core, & + accel_outer_core,kappavstore_outer_core,rhostore_outer_core,ibool_outer_core, & + NSPEC_INNER_CORE_STRAIN_ONLY, eps_trace_over_3_inner_core, & + NSPEC_CRUST_MANTLE_STR_OR_ATT, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, & + NSPEC_INNER_CORE_STR_OR_ATT, & + epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, & + epsilondev_xz_inner_core,epsilondev_yz_inner_core) + else + call write_movie_volume_divcurl(NSPEC_CRUST_MANTLE_STRAIN_ONLY,eps_trace_over_3_crust_mantle, & + div_displ_outer_core, & + accel_outer_core,kappavstore_outer_core,rhostore_outer_core,ibool_outer_core, & + NSPEC_INNER_CORE_STRAIN_ONLY,eps_trace_over_3_inner_core, & + NSPEC_CRUST_MANTLE_STR_OR_ATT, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, & + NSPEC_INNER_CORE_STR_OR_ATT, & + epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, & + epsilondev_xz_inner_core,epsilondev_yz_inner_core) + endif case (5) ! output displacement if (GPU_MODE) then call transfer_displ_cm_from_device(NDIM*NGLOB_CRUST_MANTLE,displ_crust_mantle,Mesh_pointer) endif scalingval = scale_displ - ! TODO ADD HDF5 and IO_SERVER - call write_movie_volume_vector(npoints_3dmovie, & - ibool_crust_mantle, & - displ_crust_mantle, & - scalingval,mask_3dmovie,nu_3dmovie) - + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_vector_hdf5(npoints_3dmovie, & + ibool_crust_mantle, & + displ_crust_mantle, & + scalingval,mask_3dmovie,nu_3dmovie) + else + call write_movie_volume_vector(npoints_3dmovie, & + ibool_crust_mantle, & + displ_crust_mantle, & + scalingval,mask_3dmovie,nu_3dmovie) + endif case (6) ! output velocity if (GPU_MODE) then call transfer_veloc_cm_from_device(NDIM*NGLOB_CRUST_MANTLE,veloc_crust_mantle,Mesh_pointer) endif scalingval = scale_veloc - ! TODO ADD HDF5 and IO_SERVER - call write_movie_volume_vector(npoints_3dmovie, & - ibool_crust_mantle, & - veloc_crust_mantle, & - scalingval,mask_3dmovie,nu_3dmovie) + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_vector_hdf5(npoints_3dmovie, & + ibool_crust_mantle, & + veloc_crust_mantle, & + scalingval,mask_3dmovie,nu_3dmovie) + else + call write_movie_volume_vector(npoints_3dmovie, & + ibool_crust_mantle, & + veloc_crust_mantle, & + scalingval,mask_3dmovie,nu_3dmovie) + endif case (7) ! output norm of displacement @@ -215,9 +262,14 @@ subroutine write_movie_output() call transfer_displ_ic_from_device(NDIM*NGLOB_INNER_CORE,displ_inner_core,Mesh_pointer) call transfer_displ_oc_from_device(NGLOB_OUTER_CORE,displ_outer_core,Mesh_pointer) endif - ! TODO ADD HDF5 and IO_SERVER - call write_movie_volume_displnorm(displ_crust_mantle,displ_inner_core,displ_outer_core, & - ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core,displ_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + else + call write_movie_volume_displnorm(displ_crust_mantle,displ_inner_core,displ_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + endif case (8) ! output norm of velocity @@ -228,9 +280,14 @@ subroutine write_movie_output() call transfer_veloc_ic_from_device(NDIM*NGLOB_INNER_CORE,veloc_inner_core,Mesh_pointer) call transfer_veloc_oc_from_device(NGLOB_OUTER_CORE,veloc_outer_core,Mesh_pointer) endif - ! TODO ADD HDF5 and IO_SERVER - call write_movie_volume_velnorm(veloc_crust_mantle,veloc_inner_core,veloc_outer_core, & - ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,veloc_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + else + call write_movie_volume_velnorm(veloc_crust_mantle,veloc_inner_core,veloc_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + endif case (9) ! output norm of acceleration @@ -241,9 +298,14 @@ subroutine write_movie_output() call transfer_accel_ic_from_device(NDIM*NGLOB_INNER_CORE,accel_inner_core,Mesh_pointer) call transfer_accel_oc_from_device(NGLOB_OUTER_CORE,accel_outer_core,Mesh_pointer) endif - ! TODO ADD HDF5 and IO_SERVER - call write_movie_volume_accelnorm(accel_crust_mantle,accel_inner_core,accel_outer_core, & - ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core,accel_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + else + call write_movie_volume_accelnorm(accel_crust_mantle,accel_inner_core,accel_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + endif case default call exit_MPI(myrank, 'MOVIE_VOLUME_TYPE has to be in range from 1 to 9') diff --git a/src/specfem3D/write_movie_surface.f90 b/src/specfem3D/write_movie_surface.f90 index 8e70ca87c..29b3c24ea 100644 --- a/src/specfem3D/write_movie_surface.f90 +++ b/src/specfem3D/write_movie_surface.f90 @@ -37,6 +37,8 @@ subroutine movie_surface_count_points() ! local parameters integer :: ispec2D,i,j,npoin + if (HDF5_ENABLED) return + ! gets number of points on surface mesh npoin = 0 do ispec2D = 1, NSPEC_TOP ! NSPEC2D_TOP(IREGION_CRUST_MANTLE) diff --git a/src/specfem3D/write_movie_surface_hdf5.F90 b/src/specfem3D/write_movie_surface_hdf5.F90 new file mode 100644 index 000000000..c856a10a8 --- /dev/null +++ b/src/specfem3D/write_movie_surface_hdf5.F90 @@ -0,0 +1,565 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +subroutine movie_surface_init() +#ifdef USE_HDF5 + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + integer :: ier + + allocate(offset_poin(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_poin array') + + npoints_surf_mov_all_proc = 0 + +#else + + write(*,*) 'Error: HDF5 is not enabled in this version of Specfem3D_Globe.' + write(*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop +#endif +end subroutine movie_surface_init + + +subroutine movie_surface_finalize_hdf5() +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 + + implicit none + + deallocate(offset_poin) + +#else + + write(*,*) 'Error: HDF5 is not enabled in this version of Specfem3D_Globe.' + write(*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop +#endif +end subroutine movie_surface_finalize_hdf5 + +subroutine write_movie_surface_mesh_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_movie + +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 + + implicit none + + ! local parameters + real(kind=CUSTOM_REAL), dimension(:), allocatable :: store_val_x,store_val_y,store_val_z + integer :: ipoin,ispec2D,ispec,i,j,k,ier,iglob1,iglob2,iglob3,iglob4,npoin + + call movie_surface_init() + + ! gather npoints on each process + call gather_all_all_singlei(nmovie_points,offset_poin,NPROCTOT_VAL) + ! total number of points on all processes + npoints_surf_mov_all_proc = sum(offset_poin) + + ! allocates movie surface arrays + allocate(store_val_x(nmovie_points), & + store_val_y(nmovie_points), & + store_val_z(nmovie_points),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating movie surface location arrays') + + ! initialize h5 file for surface movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create file and dataset + file_name = trim(OUTPUT_FILES)//"/movie_surface.h5" + + if (myrank == 0) then + call h5_create_file(file_name) + ! create group surf_coord + call h5_create_group("surf_coord") + call h5_open_group("surf_coord") + ! create datasets x, y, z + call h5_create_dataset_gen_in_group("x", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("y", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("z", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + + ! close group surf_coord + call h5_close_group() + + ! close file + call h5_close_file() + endif + + ! gets coordinates of surface mesh + ipoin = 0 + do ispec2D = 1, NSPEC_TOP ! NSPEC2D_TOP(IREGION_CRUST_MANTLE) + ispec = ibelm_top_crust_mantle(ispec2D) + ! in case of global, NCHUNKS_VAL == 6 simulations, be aware that for + ! the cubed sphere, the mapping changes for different chunks, + ! i.e. e.g. x(1,1) and x(5,5) flip left and right sides of the elements in geographical coordinates. + ! for future consideration, like in create_movie_GMT_global.f90 ... + k = NGLLZ + ! loop on all the points inside the element + if (.not. MOVIE_COARSE) then + do j = 1, NGLLY-1, 1 + do i = 1, NGLLX-1, 1 + ! stores values + iglob1 = ibool_crust_mantle(i,j,k,ispec) + iglob2 = ibool_crust_mantle(i+1,j,k,ispec) + iglob3 = ibool_crust_mantle(i+1,j+1,k,ispec) + iglob4 = ibool_crust_mantle(i,j+1,k,ispec) + ! iglob1 + ipoin = ipoin + 1 + store_val_x(ipoin) = rstore_crust_mantle(1,iglob1) ! radius r (normalized) + store_val_y(ipoin) = rstore_crust_mantle(2,iglob1) ! colatitude theta (in radian) + store_val_z(ipoin) = rstore_crust_mantle(3,iglob1) ! longitude phi (in radian) + ! iglob2 + ipoin = ipoin + 1 + store_val_x(ipoin) = rstore_crust_mantle(1,iglob2) ! radius r (normalized) + store_val_y(ipoin) = rstore_crust_mantle(2,iglob2) ! colatitude theta (in radian) + store_val_z(ipoin) = rstore_crust_mantle(3,iglob2) ! longitude phi (in radian) + ! iglob3 + ipoin = ipoin + 1 + store_val_x(ipoin) = rstore_crust_mantle(1,iglob3) ! radius r (normalized) + store_val_y(ipoin) = rstore_crust_mantle(2,iglob3) ! colatitude theta (in radian) + store_val_z(ipoin) = rstore_crust_mantle(3,iglob3) ! longitude phi (in radian) + ! iglob4 + ipoin = ipoin + 1 + store_val_x(ipoin) = rstore_crust_mantle(1,iglob4) ! radius r (normalized) + store_val_y(ipoin) = rstore_crust_mantle(2,iglob4) ! colatitude theta (in radian) + store_val_z(ipoin) = rstore_crust_mantle(3,iglob4) ! longitude phi (in radian) + enddo + enddo + else ! MOVIE_COARSE + iglob1 = ibool_crust_mantle(1,1,k,ispec) + iglob2 = ibool_crust_mantle(NGLLX-1,1,k,ispec) + iglob3 = ibool_crust_mantle(NGLLX-1,NGLLY-1,k,ispec) + iglob4 = ibool_crust_mantle(1,NGLLY-1,k,ispec) + + ipoin = ipoin + 1 + store_val_x(ipoin) = rstore_crust_mantle(1,iglob1) ! radius r (normalized) + store_val_y(ipoin) = rstore_crust_mantle(2,iglob1) ! colatitude theta (in radian) + store_val_z(ipoin) = rstore_crust_mantle(3,iglob1) ! longitude phi (in radian) + + ipoin = ipoin + 1 + store_val_x(ipoin) = rstore_crust_mantle(1,iglob2) ! radius r (normalized) + store_val_y(ipoin) = rstore_crust_mantle(2,iglob2) ! colatitude theta (in radian) + store_val_z(ipoin) = rstore_crust_mantle(3,iglob2) ! longitude phi (in radian) + + ipoin = ipoin + 1 + store_val_x(ipoin) = rstore_crust_mantle(1,iglob3) ! radius r (normalized) + store_val_y(ipoin) = rstore_crust_mantle(2,iglob3) ! colatitude theta (in radian) + store_val_z(ipoin) = rstore_crust_mantle(3,iglob3) ! longitude phi (in radian) + + ipoin = ipoin + 1 + store_val_x(ipoin) = rstore_crust_mantle(1,iglob4) ! radius r (normalized) + store_val_y(ipoin) = rstore_crust_mantle(2,iglob4) ! colatitude theta (in radian) + store_val_z(ipoin) = rstore_crust_mantle(3,iglob4) ! longitude phi (in radian) + + endif + enddo + npoin = ipoin + if (npoin /= nmovie_points ) call exit_mpi(myrank,'Error number of movie points not equal to nmovie_points') + + call synchronize_all() + + ! write data to h5 file + call h5_open_file_p_collect(file_name) + call h5_open_group("surf_coord") + + ! write x, y, z + call h5_write_dataset_collect_hyperslab_in_group("x", store_val_x, (/sum(offset_poin(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("y", store_val_y, (/sum(offset_poin(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("z", store_val_z, (/sum(offset_poin(0:myrank-1))/), if_col) + + ! close group and file + call h5_close_group() + call h5_close_file_p() + + deallocate(store_val_x,store_val_y,store_val_z) + + if (myrank == 0) then + ! write xdmf header + call write_xdmf_surface_header(npoints_surf_mov_all_proc) + endif + +#else + + write(*,*) 'Error: HDF5 is not enabled in this version of Specfem3D_Globe.' + write(*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + +end subroutine write_movie_surface_mesh_hdf5 + + +subroutine write_movie_surface_hdf5() + +#ifdef USE_HDF5 + use specfem_par + use specfem_par_crustmantle + use specfem_par_movie + use specfem_par_movie_hdf5 + + implicit none + + ! local parameters + integer :: ipoin,ispec2D,ispec,i,j,k,iglob1,iglob2,iglob3,iglob4 + + ! by default: save velocity here to avoid static offset on displacement for movies + + ! gets coordinates of surface mesh and surface displacement + ipoin = 0 + do ispec2D = 1, NSPEC_TOP ! NSPEC2D_TOP(IREGION_CRUST_MANTLE) + ispec = ibelm_top_crust_mantle(ispec2D) + + ! in case of global, NCHUNKS_VAL == 6 simulations, be aware that for + ! the cubed sphere, the mapping changes for different chunks, + ! i.e. e.g. x(1,1) and x(5,5) flip left and right sides of the elements in geographical coordinates. + ! for future consideration, like in create_movie_GMT_global.f90 ... + k = NGLLZ + + ! loop on all the points inside the element + if (.not. MOVIE_COARSE) then + do j = 1, NGLLY-1, 1 + do i = 1, NGLLX-1, 1 + ! stores values + iglob1 = ibool_crust_mantle(i,j,k,ispec) + iglob2 = ibool_crust_mantle(i+1,j,k,ispec) + iglob3 = ibool_crust_mantle(i+1,j+1,k,ispec) + iglob4 = ibool_crust_mantle(i,j+1,k,ispec) + + if (MOVIE_VOLUME_TYPE == 5) then + ! stores displacement + ! iglob1 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob2 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob3 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob4 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + else + ! stores velocity + ! iglob1 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob2 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob3 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob4 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + endif + enddo + enddo + else ! MOVIE_COARSE + iglob1 = ibool_crust_mantle(1,1,k,ispec) + iglob2 = ibool_crust_mantle(NGLLX-1,1,k,ispec) + iglob3 = ibool_crust_mantle(NGLLX-1,NGLLY-1,k,ispec) + iglob4 = ibool_crust_mantle(1,NGLLY-1,k,ispec) + + if (MOVIE_VOLUME_TYPE == 5) then + ! stores displacement + ! iglob1 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob2 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob3 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob4 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + else + ! stores velocity + ! iglob1 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob2 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob3 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob4 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + endif + endif + enddo + ! TODO ADD IOSERVER + + ! initialize h5 file for surface movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create file and dataset + file_name = trim(OUTPUT_FILES)//"/movie_surface.h5" + group_name = "it_"//trim(i2c(it)) + + ! create dataset + if (myrank == 0) then + call h5_open_file(file_name) + call h5_create_group(group_name) + call h5_open_group(group_name) + + ! create datasets ux, uy, uz + call h5_create_dataset_gen_in_group("ux", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("uy", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("uz", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + + ! close group + call h5_close_group() + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! write data to h5 file + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + ! write ux, uy, uz + call h5_write_dataset_collect_hyperslab_in_group("ux", store_val_ux, (/sum(offset_poin(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("uy", store_val_uy, (/sum(offset_poin(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("uz", store_val_uz, (/sum(offset_poin(0:myrank-1))/), if_col) + + ! close group and file + call h5_close_group() + call h5_close_file_p() + + ! write xdmf body + call write_xdmf_surface_body(it, npoints_surf_mov_all_proc) + +#else + + write(*,*) 'Error: HDF5 is not enabled in this version of Specfem3D_Globe.' + write(*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + +end subroutine write_movie_surface_hdf5 + + + +! +! xdmf output routines +! +#ifdef USE_HDF5 + + subroutine write_xdmf_surface_header(num_nodes) + + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + integer, intent(in) :: num_nodes + ! local parameters + integer :: num_elm + character(len=MAX_STRING_LEN) :: fname_xdmf_surf + character(len=MAX_STRING_LEN) :: fname_h5_data_surf_xdmf + + ! checks if anything do, only main process writes out xdmf file + if (myrank /= 0) return + + ! writeout xdmf file for surface movie + fname_xdmf_surf = trim(OUTPUT_FILES) // "/movie_surface.xmf" + fname_h5_data_surf_xdmf = "./movie_surface.h5" ! relative to movie_surface.xmf file + ! note: this seems not to work and point to a wrong directory: + ! fname_h5_data_surf_xdmf = trim(OUTPUT_FILES) // "/movie_surface.h5" + + num_elm = num_nodes / 4 + + open(unit=xdmf_surf, file=trim(fname_xdmf_surf), recl=256) + + write(xdmf_surf,'(a)') '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) ' '//trim(fname_h5_data_surf_xdmf)//':/surf_coord/x' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) ' '//trim(fname_h5_data_surf_xdmf)//':/surf_coord/y' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) ' '//trim(fname_h5_data_surf_xdmf)//':/surf_coord/z' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + + write(xdmf_surf,*) '' + ! 17 lines + + ! file finish + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + ! 20 lines + + ! position where the additional data will be inserted + surf_xdmf_pos = 17 + + close(xdmf_surf) + + end subroutine write_xdmf_surface_header + +#endif + + +#ifdef USE_HDF5 + + subroutine write_xdmf_surface_body(it_io, num_nodes) + + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + integer, intent(in) :: it_io + integer, intent(in) :: num_nodes + ! local parameters + integer :: i + character(len=20) :: it_str + character(len=MAX_STRING_LEN) :: fname_xdmf_surf + character(len=MAX_STRING_LEN) :: fname_h5_data_surf_xdmf + + ! checks if anything do, only main process writes out xdmf file + if (myrank /= 0) return + + ! append data section to xdmf file for surface movie + fname_xdmf_surf = trim(OUTPUT_FILES)//"/movie_surface.xmf" + fname_h5_data_surf_xdmf = "./movie_surface.h5" ! relative to movie_surface.xmf file + ! this seems to point to a wrong directory: + ! fname_h5_data_surf_xdmf = trim(OUTPUT_FILES) // "/movie_surface.h5" + + ! open xdmf file + open(unit=xdmf_surf, file=trim(fname_xdmf_surf), status='old', recl=256) + + ! skip lines till the position where we want to write new information + do i = 1, surf_xdmf_pos + read(xdmf_surf, *) + enddo + + !write(it_str, "(i6.6)") it_io + it_str = i2c(it_io) + + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + ! 20 lines + + ! file finish + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + + close(xdmf_surf) + + ! updates file record position + surf_xdmf_pos = surf_xdmf_pos + 20 + + end subroutine write_xdmf_surface_body + +#endif + diff --git a/src/specfem3D/write_movie_volume_hdf5.F90 b/src/specfem3D/write_movie_volume_hdf5.F90 new file mode 100644 index 000000000..06a6e7cdd --- /dev/null +++ b/src/specfem3D/write_movie_volume_hdf5.F90 @@ -0,0 +1,1549 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + subroutine write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovie,mask_ibool_3dmovie, & + muvstore_crust_mantle_3dmovie,npoints_3dmovie) + + use specfem_par + use specfem_par_crustmantle, only: ibool_crust_mantle,rstore_crust_mantle + +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 +#endif + + implicit none + + integer,intent(in) :: npoints_3dmovie + integer, dimension(NGLOB_CRUST_MANTLE_3DMOVIE),intent(in) :: num_ibool_3dmovie + + real(kind=CUSTOM_REAL), dimension(3,3,npoints_3dmovie),intent(inout) :: nu_3dmovie + + logical, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_3DMOVIE),intent(in) :: mask_3dmovie + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_3DMOVIE),intent(in) :: muvstore_crust_mantle_3dmovie + logical, dimension(NGLOB_CRUST_MANTLE_3DMOVIE),intent(in) :: mask_ibool_3dmovie + +#ifdef USE_HDF5 + ! local parameters + integer :: ipoints_3dmovie,ispec,i,j,k,iNIT + integer :: iglob,iglob_center + real(kind=CUSTOM_REAL) :: rval,thetaval,phival,xval,yval,zval,st,ct,sp,cp + real(kind=CUSTOM_REAL), dimension(npoints_3dmovie) :: store_val3D_x,store_val3D_y, store_val3D_z + real(kind=CUSTOM_REAL), dimension(npoints_3dmovie) :: store_val3D_mu + integer, dimension(0:NPROCTOT_VAL-1) :: offset_poin_vol + integer :: npoints_vol_mov_all_proc + integer, dimension(9,npoints_3dmovie) :: elm_conn + + ! safety check + if (NDIM /= 3) stop 'movie volume output requires NDIM = 3' + + ! prepare offset array + call gather_all_all_singlei(npoints_3dmovie, offset_poin_vol, NPROCTOT_VAL) + npoints_vol_mov_all_proc = sum(offset_poin_vol) + + ! stepping + if (MOVIE_COARSE) then + iNIT = NGLLX-1 + else + iNIT = 1 + endif + + ! loops over all elements + ipoints_3dmovie = 0 + do ispec = 1,NSPEC_CRUST_MANTLE + + ! checks center of element for movie flag + iglob_center = ibool_crust_mantle((NGLLX+1)/2,(NGLLY+1)/2,(NGLLZ+1)/2,ispec) + + ! checks if movie element + if (mask_ibool_3dmovie(iglob_center)) then + + ! stores element coordinates + do k = 1,NGLLZ,iNIT + do j = 1,NGLLY,iNIT + do i = 1,NGLLX,iNIT + ! only store points once + if (mask_3dmovie(i,j,k,ispec)) then + ! point increment + ipoints_3dmovie = ipoints_3dmovie + 1 + + ! gets point position + iglob = ibool_crust_mantle(i,j,k,ispec) + + rval = rstore_crust_mantle(1,iglob) + thetaval = rstore_crust_mantle(2,iglob) + phival = rstore_crust_mantle(3,iglob) + + !x,y,z store have been converted to r theta phi already, need to revert back for xyz output + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + + store_val3D_x(ipoints_3dmovie) = xval + store_val3D_y(ipoints_3dmovie) = yval + store_val3D_z(ipoints_3dmovie) = zval + store_val3D_mu(ipoints_3dmovie) = muvstore_crust_mantle_3dmovie(i,j,k,ispec) + + st = sin(thetaval) + ct = cos(thetaval) + sp = sin(phival) + cp = cos(phival) + + nu_3dmovie(1,1,ipoints_3dmovie) = -ct*cp + nu_3dmovie(1,2,ipoints_3dmovie) = -ct*sp + nu_3dmovie(1,3,ipoints_3dmovie) = st + nu_3dmovie(2,1,ipoints_3dmovie) = -sp + nu_3dmovie(2,2,ipoints_3dmovie) = cp + nu_3dmovie(2,3,ipoints_3dmovie) = 0.d0 + nu_3dmovie(3,1,ipoints_3dmovie) = st*cp + nu_3dmovie(3,2,ipoints_3dmovie) = st*sp + nu_3dmovie(3,3,ipoints_3dmovie) = ct + endif !mask_3dmovie + enddo !i + enddo !j + enddo !k + endif + + enddo !ispec + + ! create elm_conn for movie + call get_conn_for_movie(elm_conn, sum(offset_poin_vol(0:myrank-1)), iNIT, npoints_3dmovie, num_ibool_3dmovie, mask_ibool_3dmovie) + + ! TODO ADD IOSERVER + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! file name and group name + file_name = 'movie_volume.h5' + group_name = 'mesh' + + ! create the file, group and dataset + if (myrank == 0) then + call h5_create_file(file_name) + call h5_open_or_create_group(group_name) + + ! create the dataset + call h5_create_dataset_gen_in_group('elm_conn', (/9, npoints_vol_mov_all_proc/), 2, 1) + call h5_create_dataset_gen_in_group('x', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('y', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('z', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + + ! close the group + call h5_close_group() + ! close the file + call h5_close_file() + + endif + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + ! write the data + call h5_write_dataset_collect_hyperslab_in_group('elm_conn', elm_conn, (/0, sum(offset_poin_vol(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('x', store_val3D_x, (/sum(offset_poin_vol(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('y', store_val3D_y, (/sum(offset_poin_vol(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('z', store_val3D_z, (/sum(offset_poin_vol(0:myrank-1))/), if_col) + + ! close the group + call h5_close_group() + ! close the file + call h5_close_file_p() + + ! write xdmf for all timesteps + call write_xdmf_vol_hdf5(.true., .false., .false., .false., .false., .false.) + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_mesh_hdf5 + + + subroutine write_movie_volume_strains_hdf5(vnspec_eps_cm, & + eps_trace_over_3_crust_mantle, & + vnspec_cm, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle) + + use constants_solver + use shared_parameters, only: OUTPUT_FILES,MOVIE_VOLUME_TYPE,MOVIE_COARSE + use specfem_par, only: it + use specfem_par_movie, only: npoints_3dmovie,muvstore_crust_mantle_3dmovie,mask_3dmovie,nu_3dmovie + +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 +#endif + + implicit none + + ! input + integer,intent(in) :: vnspec_eps_cm + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_eps_cm),intent(in) :: eps_trace_over_3_crust_mantle + + integer,intent(in) :: vnspec_cm + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_cm),intent(in) :: & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle + +#ifdef USE_HDF5 + ! variables + real(kind=CUSTOM_REAL) :: muv_3dmovie + real(kind=CUSTOM_REAL),dimension(3,3) :: eps_loc,eps_loc_new + real(kind=CUSTOM_REAL),dimension(:),allocatable :: store_val3d_NN,store_val3d_EE,store_val3d_ZZ, & + store_val3d_NE,store_val3d_NZ,store_val3d_EZ + integer :: ipoints_3dmovie,i,j,k,ispec,iNIT,ier + character(len=1) :: movie_prefix + integer, dimension(0:NPROCTOT_VAL-1) :: offset_poin_vol + integer :: npoints_vol_mov_all_proc + + ! prepare offset array + call gather_all_all_singlei(npoints_3dmovie, offset_poin_vol, NPROCTOT_VAL) + npoints_vol_mov_all_proc = sum(offset_poin_vol) + + ! check + if (NDIM /= 3) call exit_MPI(myrank, 'write_movie_volume_strains() requires NDIM = 3') + if (vnspec_cm /= NSPEC_CRUST_MANTLE) call exit_MPI(myrank,'Invalid vnspec_cm value for write_movie_volume_strains() routine') + + ! allocates arrays + allocate(store_val3d_NN(npoints_3dmovie), & + store_val3d_EE(npoints_3dmovie), & + store_val3d_ZZ(npoints_3dmovie), & + store_val3d_NE(npoints_3dmovie), & + store_val3d_NZ(npoints_3dmovie), & + store_val3d_EZ(npoints_3dmovie), & + stat=ier) + if (ier /= 0 ) call exit_mpi(myrank,'Error allocating store_val3d_ .. arrays') + + if (MOVIE_VOLUME_TYPE == 1) then + movie_prefix='E' ! strain + else if (MOVIE_VOLUME_TYPE == 2) then + movie_prefix='S' ! time integral of strain + else if (MOVIE_VOLUME_TYPE == 3) then + movie_prefix='P' ! potency, or integral of strain x \mu + endif + + ! stepping + if (MOVIE_COARSE) then + iNIT = NGLLX-1 + else + iNIT = 1 + endif + + ipoints_3dmovie = 0 + do ispec = 1,NSPEC_CRUST_MANTLE + do k = 1,NGLLZ,iNIT + do j = 1,NGLLY,iNIT + do i = 1,NGLLX,iNIT + if (mask_3dmovie(i,j,k,ispec)) then + ipoints_3dmovie = ipoints_3dmovie + 1 + muv_3dmovie = muvstore_crust_mantle_3dmovie(i,j,k,ispec) + + eps_loc(1,1) = eps_trace_over_3_crust_mantle(i,j,k,ispec) + epsilondev_xx_crust_mantle(i,j,k,ispec) + eps_loc(2,2) = eps_trace_over_3_crust_mantle(i,j,k,ispec) + epsilondev_yy_crust_mantle(i,j,k,ispec) + eps_loc(3,3) = eps_trace_over_3_crust_mantle(i,j,k,ispec) & + - epsilondev_xx_crust_mantle(i,j,k,ispec) & + - epsilondev_yy_crust_mantle(i,j,k,ispec) + + eps_loc(1,2) = epsilondev_xy_crust_mantle(i,j,k,ispec) + eps_loc(1,3) = epsilondev_xz_crust_mantle(i,j,k,ispec) + eps_loc(2,3) = epsilondev_yz_crust_mantle(i,j,k,ispec) + + eps_loc(2,1) = eps_loc(1,2) + eps_loc(3,1) = eps_loc(1,3) + eps_loc(3,2) = eps_loc(2,3) + + ! rotate eps_loc to spherical coordinates + eps_loc_new(:,:) = matmul(matmul(nu_3dmovie(:,:,ipoints_3dmovie),eps_loc(:,:)), & + transpose(nu_3dmovie(:,:,ipoints_3dmovie))) + if (MOVIE_VOLUME_TYPE == 3) eps_loc_new(:,:) = eps_loc(:,:)*muv_3dmovie + + store_val3d_NN(ipoints_3dmovie) = eps_loc_new(1,1) + store_val3d_EE(ipoints_3dmovie) = eps_loc_new(2,2) + store_val3d_ZZ(ipoints_3dmovie) = eps_loc_new(3,3) + store_val3d_NE(ipoints_3dmovie) = eps_loc_new(1,2) + store_val3d_NZ(ipoints_3dmovie) = eps_loc_new(1,3) + store_val3d_EZ(ipoints_3dmovie) = eps_loc_new(2,3) + endif + enddo + enddo + enddo + enddo + if (ipoints_3dmovie /= npoints_3dmovie) stop 'did not find the right number of points for 3D movie' + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create group and datasets + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + if (myrank == 0) then + + ! create the file, group and dataset + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + ! create the dataset + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'NN', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'EE', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'ZZ', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'NE', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'NZ', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'EZ', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + + ! close the group + call h5_close_group() + ! close the file + call h5_close_file() + + endif + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NN', store_val3d_NN, (/sum(offset_poin_vol(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EE', store_val3d_EE, (/sum(offset_poin_vol(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'ZZ', store_val3d_ZZ, (/sum(offset_poin_vol(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NE', store_val3d_NE, (/sum(offset_poin_vol(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NZ', store_val3d_NZ, (/sum(offset_poin_vol(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EZ', store_val3d_EZ, (/sum(offset_poin_vol(0:myrank-1))/), if_col) + + call h5_close_group() + call h5_close_file_p() + + deallocate(store_val3d_NN,store_val3d_EE,store_val3d_ZZ, & + store_val3d_NE,store_val3d_NZ,store_val3d_EZ) + + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_strains_hdf5 + + + + +subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_mantle, & + div_displ_outer_core, & + accel_outer_core,kappavstore_outer_core,rhostore_outer_core,ibool_outer_core, & + vnspec_eps_ic,eps_trace_over_3_inner_core, & + vnspec_cm,epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, & + vnspec_ic,epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, & + epsilondev_xz_inner_core,epsilondev_yz_inner_core) + +! outputs divergence and curl: MOVIE_VOLUME_TYPE == 4 + + use constants_solver + use shared_parameters, only: OUTPUT_FILES + use specfem_par, only: it + +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 +#endif + + implicit none + + integer,intent(in) :: vnspec_eps_cm + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_eps_cm) :: eps_trace_over_3_crust_mantle + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE_3DMOVIE) :: div_displ_outer_core + + real(kind=CUSTOM_REAL), dimension(NGLOB_OUTER_CORE) :: accel_outer_core + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE) :: rhostore_outer_core,kappavstore_outer_core + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE) :: ibool_outer_core + + integer,intent(in) :: vnspec_eps_ic + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_eps_ic) :: eps_trace_over_3_inner_core + + integer,intent(in) :: vnspec_cm + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_cm) :: & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle + + integer,intent(in) :: vnspec_ic + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_ic) :: & + epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, & + epsilondev_xz_inner_core,epsilondev_yz_inner_core + +#ifdef USE_HDF5 + + ! local parameters + real(kind=CUSTOM_REAL) :: rhol,kappal + real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: div_s_outer_core + integer :: ispec,iglob,i,j,k,ier + real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: tmp_data + + ! output parameters + logical,parameter :: MOVIE_OUTPUT_DIV = .true. ! divergence + logical,parameter :: MOVIE_OUTPUT_CURL = .true. ! curl + logical,parameter :: MOVIE_OUTPUT_CURLNORM = .true. ! Frobenius norm of curl + + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_eps_cm + integer :: nspec_vol_mov_all_proc_eps_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_eps_ic + integer :: nspec_vol_mov_all_proc_eps_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_cm + integer :: nspec_vol_mov_all_proc_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_oc + integer :: nspec_vol_mov_all_proc_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_ic + integer :: nspec_vol_mov_all_proc_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_cm_norm + integer :: nspec_vol_mov_all_proc_cm_norm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_ic_norm + integer :: nspec_vol_mov_all_proc_ic_norm + + ! prepare offset array + call gather_all_all_singlei(vnspec_eps_cm, offset_nspec_vol_eps_cm, NPROCTOT_VAL) + nspec_vol_mov_all_proc_eps_cm = sum(offset_nspec_vol_cm) + call gather_all_all_singlei(vnspec_eps_ic, offset_nspec_vol_eps_ic, NPROCTOT_VAL) + nspec_vol_mov_all_proc_eps_ic = sum(offset_nspec_vol_ic) + call gather_all_all_singlei(vnspec_cm, offset_nspec_vol_cm, NPROCTOT_VAL) + nspec_vol_mov_all_proc_cm = sum(offset_nspec_vol_cm) + call gather_all_all_singlei(vnspec_ic, offset_nspec_vol_ic, NPROCTOT_VAL) + nspec_vol_mov_all_proc_ic = sum(offset_nspec_vol_ic) + if (NSPEC_OUTER_CORE_3DMOVIE > 1) then + call gather_all_all_singlei(NSPEC_OUTER_CORE_3DMOVIE, offset_nspec_vol_oc, NPROCTOT_VAL) + else + call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_vol_oc, NPROCTOT_VAL) + endif + nspec_vol_mov_all_proc_oc = sum(offset_nspec_vol_oc) + if (MOVIE_OUTPUT_CURLNORM) then + call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_vol_cm_norm, NPROCTOT_VAL) + nspec_vol_mov_all_proc_cm_norm = sum(offset_nspec_vol_cm_norm) + call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_vol_ic_norm, NPROCTOT_VAL) + nspec_vol_mov_all_proc_ic_norm = sum(offset_nspec_vol_ic_norm) + endif + + ! checks + if (vnspec_cm /= NSPEC_CRUST_MANTLE) call exit_MPI(myrank,'Invalid vnspec_cm value for write_movie_volume_divcurl() routine') + + ! create group and datasets + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + if (myrank == 0) then + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + if (MOVIE_OUTPUT_DIV) then + call h5_create_dataset_gen_in_group('reg1_div_displ', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_eps_cm/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg2_div_displ', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_oc/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg3_div_displ', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_eps_ic/), 4, CUSTOM_REAL) + endif + + if (MOVIE_OUTPUT_CURL) then + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xx', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_yy', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xy', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xz', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_yz', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xx', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_yy', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xy', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xz', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_yz', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic/), 4, CUSTOM_REAL) + endif + + if (MOVIE_OUTPUT_CURLNORM) then + call h5_create_dataset_gen_in_group('reg1_epsdev_displ_norm', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm_norm/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg3_epsdev_displ_norm', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic_norm/), 4, CUSTOM_REAL) + endif + + call h5_close_group() + call h5_close_file() + endif + + call synchronize_all() + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + if (MOVIE_OUTPUT_DIV) then + call h5_write_dataset_collect_hyperslab_in_group('reg1_div_displ', eps_trace_over_3_crust_mantle, (/0,0,0,offset_nspec_vol_eps_cm(0:myrank-1)/), if_col) + + if (NSPEC_OUTER_CORE_3DMOVIE > 1) then + call h5_write_dataset_collect_hyperslab_in_group('reg2_div_displ', ONE_THIRD*div_displ_outer_core, (/0,0,0,offset_nspec_vol_oc(0:myrank-1)/), if_col) + else + allocate(div_s_outer_core(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array div_s_outer_core') + do ispec = 1, NSPEC_OUTER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_outer_core(i,j,k,ispec) + rhol = rhostore_outer_core(i,j,k,ispec) + kappal = kappavstore_outer_core(i,j,k,ispec) + div_s_outer_core(i,j,k,ispec) = rhol * accel_outer_core(iglob) / kappal + enddo + enddo + enddo + enddo + call h5_write_dataset_collect_hyperslab_in_group('reg2_div_displ', div_s_outer_core, (/0,0,0,offset_nspec_vol_oc(0:myrank-1)/), if_col) + deallocate(div_s_outer_core) + endif + + call h5_write_dataset_collect_hyperslab_in_group('reg3_div_displ', eps_trace_over_3_inner_core, (/0,0,0,offset_nspec_vol_eps_ic(0:myrank-1)/), if_col) + + endif + + if (MOVIE_OUTPUT_CURL) then + call h5_write_dataset_collect_hyperslab_in_group('crust_mantle_epsdev_displ_xx', epsilondev_xx_crust_mantle, (/0,0,0,offset_nspec_vol_cm(0:myrank-1)/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('crust_mantle_epsdev_displ_yy', epsilondev_yy_crust_mantle, (/0,0,0,offset_nspec_vol_cm(0:myrank-1)/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('crust_mantle_epsdev_displ_xy', epsilondev_xy_crust_mantle, (/0,0,0,offset_nspec_vol_cm(0:myrank-1)/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('crust_mantle_epsdev_displ_xz', epsilondev_xz_crust_mantle, (/0,0,0,offset_nspec_vol_cm(0:myrank-1)/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('crust_mantle_epsdev_displ_yz', epsilondev_yz_crust_mantle, (/0,0,0,offset_nspec_vol_cm(0:myrank-1)/), if_col) + + call h5_write_dataset_collect_hyperslab_in_group('inner_core_epsdev_displ_xx', epsilondev_xx_inner_core, (/0,0,0,offset_nspec_vol_ic(0:myrank-1)/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('inner_core_epsdev_displ_yy', epsilondev_yy_inner_core, (/0,0,0,offset_nspec_vol_ic(0:myrank-1)/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('inner_core_epsdev_displ_xy', epsilondev_xy_inner_core, (/0,0,0,offset_nspec_vol_ic(0:myrank-1)/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('inner_core_epsdev_displ_xz', epsilondev_xz_inner_core, (/0,0,0,offset_nspec_vol_ic(0:myrank-1)/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('inner_core_epsdev_displ_yz', epsilondev_yz_inner_core, (/0,0,0,offset_nspec_vol_ic(0:myrank-1)/), if_col) + endif + + if (MOVIE_OUTPUT_CURLNORM) then + allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + ! Frobenius norm + do ispec = 1, NSPEC_CRUST_MANTLE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + tmp_data(i,j,k,ispec) = sqrt( epsilondev_xx_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_yy_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_xy_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_xz_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_yz_crust_mantle(i,j,k,ispec)**2) + enddo + enddo + enddo + enddo + call h5_write_dataset_collect_hyperslab_in_group('reg1_epsdev_displ_norm', tmp_data, (/0,0,0,offset_nspec_vol_cm_norm(0:myrank-1)/), if_col) + deallocate(tmp_data) + + ! Frobenius norm + allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + do ispec = 1, NSPEC_INNER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + tmp_data(i,j,k,ispec) = sqrt( epsilondev_xx_inner_core(i,j,k,ispec)**2 & + + epsilondev_yy_inner_core(i,j,k,ispec)**2 & + + epsilondev_xy_inner_core(i,j,k,ispec)**2 & + + epsilondev_xz_inner_core(i,j,k,ispec)**2 & + + epsilondev_yz_inner_core(i,j,k,ispec)**2) + enddo + enddo + enddo + enddo + call h5_write_dataset_collect_hyperslab_in_group('reg3_epsdev_displ_norm', tmp_data, (/0,0,0,offset_nspec_vol_ic_norm(0:myrank-1)/), if_col) + deallocate(tmp_data) + endif + + call h5_close_group() + call h5_close_file() + + !if (myrank == 0) call write_xdmf... + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_divcurl_hdf5 + + + + subroutine write_movie_volume_vector_hdf5(npoints_3dmovie, & + ibool_crust_mantle,vector_crust_mantle, & + scalingval,mask_3dmovie,nu_3dmovie) + +! outputs displacement/velocity: MOVIE_VOLUME_TYPE == 5 / 6 + + use constants_solver + use shared_parameters, only: OUTPUT_FILES,MOVIE_VOLUME_TYPE,MOVIE_COARSE + use specfem_par, only: it + +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 +#endif + + implicit none + + ! input + integer :: npoints_3dmovie + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle + + ! displacement or velocity array + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CRUST_MANTLE) :: vector_crust_mantle + logical, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_3DMOVIE) :: mask_3dmovie + + double precision :: scalingval + real(kind=CUSTOM_REAL), dimension(NDIM,NDIM,npoints_3dmovie) :: nu_3dmovie + +#ifdef USE_HDF5 + + ! local variables + real(kind=CUSTOM_REAL), dimension(:), allocatable :: store_val3d_N,store_val3d_E,store_val3d_Z + real(kind=CUSTOM_REAL), dimension(NDIM) :: vector_local,vector_local_new + + integer :: ipoints_3dmovie,i,j,k,ispec,iNIT,iglob,ier + character(len=2) :: movie_prefix + + integer, dimension(0:NPROCTOT_VAL-1) :: offset_npoints_3dmovie + integer :: npoints_3dmovie_all_proc + + + ! check + if (NDIM /= 3) call exit_MPI(myrank,'write_movie_volume requires NDIM = 3') + + ! gather number of points + call gather_all_all_singlei(npoints_3dmovie, offset_npoints_3dmovie, NPROCTOT_VAL) + npoints_3dmovie_all_proc = sum(offset_npoints_3dmovie) + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + ! allocates arrays + allocate(store_val3d_N(npoints_3dmovie), & + store_val3d_E(npoints_3dmovie), & + store_val3d_Z(npoints_3dmovie), & + stat=ier) + if (ier /= 0 ) call exit_mpi(myrank,'Error allocating store_val3d_N,.. movie arrays') + + if (MOVIE_VOLUME_TYPE == 5) then + movie_prefix='DI' ! displacement + else if (MOVIE_VOLUME_TYPE == 6) then + movie_prefix='VE' ! velocity + endif + + if (MOVIE_COARSE) then + iNIT = NGLLX-1 + else + iNIT = 1 + endif + + ipoints_3dmovie = 0 + + ! stores field in crust/mantle region + do ispec = 1,NSPEC_CRUST_MANTLE + do k = 1,NGLLZ,iNIT + do j = 1,NGLLY,iNIT + do i = 1,NGLLX,iNIT + if (mask_3dmovie(i,j,k,ispec)) then + ipoints_3dmovie = ipoints_3dmovie + 1 + iglob = ibool_crust_mantle(i,j,k,ispec) + + ! dimensionalizes field by scaling + vector_local(:) = vector_crust_mantle(:,iglob)*real(scalingval,kind=CUSTOM_REAL) + + ! rotate eps_loc to spherical coordinates + vector_local_new(:) = matmul(nu_3dmovie(:,:,ipoints_3dmovie), vector_local(:)) + + ! stores field + store_val3d_N(ipoints_3dmovie) = vector_local_new(1) + store_val3d_E(ipoints_3dmovie) = vector_local_new(2) + store_val3d_Z(ipoints_3dmovie) = vector_local_new(3) + endif + enddo + enddo + enddo + enddo + close(IOUT) + + ! checks number of processed points + if (ipoints_3dmovie /= npoints_3dmovie) stop 'did not find the right number of points for 3D movie' + + ! create group and datasets + if (myrank == 0) then + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'N', (/npoints_3dmovie_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'E', (/npoints_3dmovie_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'Z', (/npoints_3dmovie_all_proc/), 1, CUSTOM_REAL) + + call h5_close_group() + call h5_close_file() + endif + + call synchronize_all() + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'N', store_val3d_N(1:npoints_3dmovie), (/offset_npoints_3dmovie(0:myrank-1)/), if_col) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'E', store_val3d_E(1:npoints_3dmovie), (/offset_npoints_3dmovie(0:myrank-1)/), if_col) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'Z', store_val3d_Z(1:npoints_3dmovie), (/offset_npoints_3dmovie(0:myrank-1)/), if_col) + + call h5_close_group() + call h5_close_file() + + deallocate(store_val3d_N,store_val3d_E,store_val3d_Z) + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + + end subroutine write_movie_volume_vector_hdf5 + + + subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core,displ_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + +! outputs norm of displacement: MOVIE_VOLUME_TYPE == 7 + + use constants_solver + use shared_parameters, only: OUTPUT_FILES + use specfem_par, only: it, scale_displ + +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 +#endif + + implicit none + + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CRUST_MANTLE) :: displ_crust_mantle + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_INNER_CORE) :: displ_inner_core + real(kind=CUSTOM_REAL), dimension(NGLOB_OUTER_CORE) :: displ_outer_core + + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE) :: ibool_inner_core + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE) :: ibool_outer_core + +#ifdef USE_HDF5 + + ! local parameters + integer :: ispec,iglob,i,j,k,ier + real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: tmp_data + + logical,parameter :: OUTPUT_CRUST_MANTLE = .true. + logical,parameter :: OUTPUT_OUTER_CORE = .true. + logical,parameter :: OUTPUT_INNER_CORE = .true. + + ! offset arrays + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic + integer :: nspec_cm_all_proc,nspec_oc_all_proc,nspec_ic_all_proc + + ! gather number of points + call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_cm, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_oc, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_ic, NPROCTOT_VAL) + nspec_cm_all_proc = sum(offset_nspec_cm) + nspec_oc_all_proc = sum(offset_nspec_oc) + nspec_ic_all_proc = sum(offset_nspec_ic) + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + ! create group and datasets + if (myrank == 0) then + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + if (OUTPUT_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group('reg1_displ', (/NGLLX,NGLLY,NGLLZ,nspec_cm_all_proc/), 4, CUSTOM_REAL) + endif + if (OUTPUT_OUTER_CORE) then + call h5_create_dataset_gen_in_group('reg2_displ', (/NGLLX,NGLLY,NGLLZ,nspec_oc_all_proc/), 4, CUSTOM_REAL) + endif + if (OUTPUT_INNER_CORE) then + call h5_create_dataset_gen_in_group('reg3_displ', (/NGLLX,NGLLY,NGLLZ,nspec_ic_all_proc/), 4, CUSTOM_REAL) + endif + + call h5_close_group() + call h5_close_file() + endif + + call synchronize_all() + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + ! outputs norm of displacement + if (OUTPUT_CRUST_MANTLE) then + ! crust mantle + ! these binary arrays can be converted into mesh format using the utility ./bin/xcombine_vol_data + allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_CRUST_MANTLE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_crust_mantle(i,j,k,ispec) + ! norm + tmp_data(i,j,k,ispec) = real(scale_displ,kind=CUSTOM_REAL) * sqrt( displ_crust_mantle(1,iglob)**2 & + + displ_crust_mantle(2,iglob)**2 & + + displ_crust_mantle(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call h5_write_dataset_collect_hyperslab_in_group('reg1_displ', tmp_data, (/0,0,0,offset_nspec_cm(0:myrank-1)/), if_col) + + deallocate(tmp_data) + endif + + if (OUTPUT_OUTER_CORE) then + ! outer core + allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_OUTER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_outer_core(i,j,k,ispec) + ! norm + ! note: disp_outer_core is potential, this just outputs the potential, + ! not the actual displacement u = grad(rho * Chi) / rho + tmp_data(i,j,k,ispec) = abs(displ_outer_core(iglob)) + enddo + enddo + enddo + enddo + + call h5_write_dataset_collect_hyperslab_in_group('reg2_displ', tmp_data, (/0,0,0,offset_nspec_oc(0:myrank-1)/), if_col) + + deallocate(tmp_data) + endif + + if (OUTPUT_INNER_CORE) then + ! inner core + allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_INNER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_inner_core(i,j,k,ispec) + ! norm + tmp_data(i,j,k,ispec) = real(scale_displ,kind=CUSTOM_REAL) * sqrt( displ_inner_core(1,iglob)**2 & + + displ_inner_core(2,iglob)**2 & + + displ_inner_core(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call h5_write_dataset_collect_hyperslab_in_group('reg3_displ', tmp_data, (/0,0,0,offset_nspec_ic(0:myrank-1)/), if_col) + + deallocate(tmp_data) + endif + + call h5_close_group() + call h5_close_file_p() + + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_displnorm_hdf5 + + +subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,veloc_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + +! outputs norm of velocity: MOVIE_VOLUME_TYPE == 8 + + use constants_solver + use shared_parameters, only: OUTPUT_FILES + use specfem_par, only: it, scale_veloc + +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 +#endif + + implicit none + + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CRUST_MANTLE) :: veloc_crust_mantle + real(kind=CUSTOM_REAL), dimension(NGLOB_OUTER_CORE) :: veloc_outer_core + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_INNER_CORE) :: veloc_inner_core + + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE) :: ibool_inner_core + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE) :: ibool_outer_core + +#ifdef USE_HDF5 + + ! local parameters + integer :: ispec,iglob,i,j,k,ier + real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: tmp_data + + logical,parameter :: OUTPUT_CRUST_MANTLE = .true. + logical,parameter :: OUTPUT_OUTER_CORE = .true. + logical,parameter :: OUTPUT_INNER_CORE = .true. + + ! offset arrays + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic + integer :: nspec_cm_all_proc,nspec_oc_all_proc,nspec_ic_all_proc + + ! gather number of points + call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_cm, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_oc, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_ic, NPROCTOT_VAL) + nspec_cm_all_proc = sum(offset_nspec_cm) + nspec_oc_all_proc = sum(offset_nspec_oc) + nspec_ic_all_proc = sum(offset_nspec_ic) + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + ! create group and datasets + if (myrank == 0) then + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + if (OUTPUT_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group('reg1_veloc', (/NGLLX,NGLLY,NGLLZ,nspec_cm_all_proc/), 4, CUSTOM_REAL) + endif + if (OUTPUT_OUTER_CORE) then + call h5_create_dataset_gen_in_group('reg2_veloc', (/NGLLX,NGLLY,NGLLZ,nspec_oc_all_proc/), 4, CUSTOM_REAL) + endif + if (OUTPUT_INNER_CORE) then + call h5_create_dataset_gen_in_group('reg3_veloc', (/NGLLX,NGLLY,NGLLZ,nspec_ic_all_proc/), 4, CUSTOM_REAL) + endif + + call h5_close_group() + call h5_close_file() + endif + + call synchronize_all() + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + ! outputs norm of velocity + if (OUTPUT_CRUST_MANTLE) then + ! crust mantle + ! these binary arrays can be converted into mesh format using the utility ./bin/xcombine_vol_data + allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_CRUST_MANTLE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_crust_mantle(i,j,k,ispec) + ! norm of velocity + tmp_data(i,j,k,ispec) = real(scale_veloc,kind=CUSTOM_REAL) * sqrt( veloc_crust_mantle(1,iglob)**2 & + + veloc_crust_mantle(2,iglob)**2 & + + veloc_crust_mantle(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call h5_write_dataset_collect_hyperslab_in_group('reg1_veloc', tmp_data, (/0,0,0,offset_nspec_cm(0:myrank-1)/), if_col) + + deallocate(tmp_data) + endif + + if (OUTPUT_OUTER_CORE) then + ! outer core + allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_OUTER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_outer_core(i,j,k,ispec) + ! norm of velocity + ! note: this outputs only the first time derivative of the potential, + ! not the actual velocity v = grad(Chi_dot) + tmp_data(i,j,k,ispec) = abs(veloc_outer_core(iglob)) + enddo + enddo + enddo + enddo + + call h5_write_dataset_collect_hyperslab_in_group('reg2_veloc', tmp_data, (/0,0,0,offset_nspec_oc(0:myrank-1)/), if_col) + + deallocate(tmp_data) + endif + + if (OUTPUT_INNER_CORE) then + ! inner core + allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_INNER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_inner_core(i,j,k,ispec) + ! norm of velocity + tmp_data(i,j,k,ispec) = real(scale_veloc,kind=CUSTOM_REAL) * sqrt( veloc_inner_core(1,iglob)**2 & + + veloc_inner_core(2,iglob)**2 & + + veloc_inner_core(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call h5_write_dataset_collect_hyperslab_in_group('reg3_veloc', tmp_data, (/0,0,0,offset_nspec_ic(0:myrank-1)/), if_col) + + deallocate(tmp_data) + endif + + call h5_close_group() + call h5_close_file_p() + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_velnorm_hdf5 + + + + subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core,accel_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + +! outputs norm of acceleration: MOVIE_VOLUME_TYPE == 1 + + use constants_solver + use shared_parameters, only: OUTPUT_FILES + use specfem_par, only: it, scale_t_inv,scale_veloc + +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 +#endif + + implicit none + + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CRUST_MANTLE) :: accel_crust_mantle + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_INNER_CORE) :: accel_inner_core + real(kind=CUSTOM_REAL), dimension(NGLOB_OUTER_CORE) :: accel_outer_core + + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE) :: ibool_inner_core + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE) :: ibool_outer_core + +#ifdef USE_HDF5 + ! local parameters + integer :: ispec,iglob,i,j,k,ier + real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: tmp_data + real(kind=CUSTOM_REAL) :: scale_accel + + logical,parameter :: OUTPUT_CRUST_MANTLE = .true. + logical,parameter :: OUTPUT_OUTER_CORE = .true. + logical,parameter :: OUTPUT_INNER_CORE = .true. + + ! offset arrays + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic + + integer :: nspec_cm_all_proc,nspec_oc_all_proc,nspec_ic_all_proc + + ! gather number of points + call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_cm, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_oc, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_ic, NPROCTOT_VAL) + nspec_cm_all_proc = sum(offset_nspec_cm) + nspec_oc_all_proc = sum(offset_nspec_oc) + nspec_ic_all_proc = sum(offset_nspec_ic) + + ! dimensionalized scaling + scale_accel = real(scale_veloc * scale_t_inv,kind=CUSTOM_REAL) + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + ! create group and datasets + if (myrank == 0) then + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + if (OUTPUT_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group('reg1_accel', (/NGLLX,NGLLY,NGLLZ,nspec_cm_all_proc/), 4, CUSTOM_REAL) + endif + if (OUTPUT_OUTER_CORE) then + call h5_create_dataset_gen_in_group('reg2_accel', (/NGLLX,NGLLY,NGLLZ,nspec_oc_all_proc/), 4, CUSTOM_REAL) + endif + if (OUTPUT_INNER_CORE) then + call h5_create_dataset_gen_in_group('reg3_accel', (/NGLLX,NGLLY,NGLLZ,nspec_ic_all_proc/), 4, CUSTOM_REAL) + endif + + call h5_close_group() + call h5_close_file() + endif + + call synchronize_all() + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + ! outputs norm of acceleration + if (OUTPUT_CRUST_MANTLE) then + ! acceleration + ! these binary arrays can be converted into mesh format using the utility ./bin/xcombine_vol_data + allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_CRUST_MANTLE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_crust_mantle(i,j,k,ispec) + ! norm + tmp_data(i,j,k,ispec) = scale_accel * sqrt( accel_crust_mantle(1,iglob)**2 & + + accel_crust_mantle(2,iglob)**2 & + + accel_crust_mantle(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call h5_write_dataset_collect_hyperslab_in_group('reg1_accel', tmp_data, (/0,0,0,offset_nspec_cm(0:myrank-1)/), if_col) + + deallocate(tmp_data) + endif + + if (OUTPUT_OUTER_CORE) then + ! outer core acceleration + allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_OUTER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_outer_core(i,j,k,ispec) + ! norm + ! note: this outputs only the second time derivative of the potential, + ! not the actual acceleration or pressure p = - rho * Chi_dot_dot + tmp_data(i,j,k,ispec) = abs(accel_outer_core(iglob)) + enddo + enddo + enddo + enddo + + call h5_write_dataset_collect_hyperslab_in_group('reg2_accel', tmp_data, (/0,0,0,offset_nspec_oc(0:myrank-1)/), if_col) + + deallocate(tmp_data) + endif + + if (OUTPUT_INNER_CORE) then + ! inner core + allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_INNER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_inner_core(i,j,k,ispec) + ! norm of acceleration + tmp_data(i,j,k,ispec) = scale_accel * sqrt( accel_inner_core(1,iglob)**2 & + + accel_inner_core(2,iglob)**2 & + + accel_inner_core(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call h5_write_dataset_collect_hyperslab_in_group('reg3_accel', tmp_data, (/0,0,0,offset_nspec_ic(0:myrank-1)/), if_col) + + deallocate(tmp_data) + endif + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_accelnorm_hdf5 + +! +!------------------------------------------------------------------------------------------------- +! + +#ifdef USE_HDF5 + subroutine get_conn_for_movie(elm_conn,offset,iNIT,npoints_3dmovie,num_ibool_3dmovie,mask_ibool_3dmovie) + + use specfem_par + use specfem_par_crustmantle, only: ibool_crust_mantle + use constants_solver + + + implicit none + + integer, dimension(9,npoints_3dmovie), intent(out) :: elm_conn + integer, intent(in) :: offset ! node id offset (starting global element id of each proc) + integer, intent(in) :: iNIT + integer, intent(in) :: npoints_3dmovie + integer, dimension(NGLOB_CRUST_MANTLE_3DMOVIE),intent(in) :: num_ibool_3dmovie + logical, dimension(NGLOB_CRUST_MANTLE_3DMOVIE),intent(in) :: mask_ibool_3dmovie + + + ! local parameters + integer :: ispec,ii + integer,parameter :: cell_type = 9 + integer :: iglob1,iglob2,iglob3,iglob4,iglob5,iglob6,iglob7,iglob8 + integer :: n1,n2,n3,n4,n5,n6,n7,n8 + integer :: i,j,k + integer :: ispecele, iglob_center + + ispecele = 0 + do ispec = 1,NSPEC_CRUST_MANTLE + + ! checks center of element for movie flag + iglob_center = ibool_crust_mantle((NGLLX+1)/2,(NGLLY+1)/2,(NGLLZ+1)/2,ispec) + + ! checks if movie element + if (mask_ibool_3dmovie(iglob_center)) then + + ! this element is in the movie region + ispecele = ispecele+1 + + do k = 1,NGLLZ-1,iNIT + do j = 1,NGLLY-1,iNIT + do i = 1,NGLLX-1,iNIT + ! defines corners of a vtk element + iglob1 = ibool_crust_mantle(i,j,k,ispec) + iglob2 = ibool_crust_mantle(i+iNIT,j,k,ispec) + iglob3 = ibool_crust_mantle(i+iNIT,j+iNIT,k,ispec) + iglob4 = ibool_crust_mantle(i,j+iNIT,k,ispec) + iglob5 = ibool_crust_mantle(i,j,k+iNIT,ispec) + iglob6 = ibool_crust_mantle(i+iNIT,j,k+iNIT,ispec) + iglob7 = ibool_crust_mantle(i+iNIT,j+iNIT,k+iNIT,ispec) + iglob8 = ibool_crust_mantle(i,j+iNIT,k+iNIT,ispec) + + ! vtk indexing starts at 0 -> adds minus 1 + n1 = num_ibool_3dmovie(iglob1)-1 + n2 = num_ibool_3dmovie(iglob2)-1 + n3 = num_ibool_3dmovie(iglob3)-1 + n4 = num_ibool_3dmovie(iglob4)-1 + n5 = num_ibool_3dmovie(iglob5)-1 + n6 = num_ibool_3dmovie(iglob6)-1 + n7 = num_ibool_3dmovie(iglob7)-1 + n8 = num_ibool_3dmovie(iglob8)-1 + + elm_conn(1, ispecele) = cell_type + elm_conn(2, ispecele) = n1 + offset ! node id starts 0 in xdmf rule + elm_conn(3, ispecele) = n2 + offset + elm_conn(4, ispecele) = n3 + offset + elm_conn(5, ispecele) = n4 + offset + elm_conn(6, ispecele) = n5 + offset + elm_conn(7, ispecele) = n6 + offset + elm_conn(8, ispecele) = n7 + offset + elm_conn(9, ispecele) = n8 + offset + + ! checks indices + if (n1 < 0 .or. n2 < 0 .or. n3 < 0 .or. n4 < 0 .or. n5 < 0 .or. n6 < 0 .or. n7 < 0 .or. n8 < 0) then + print *,'Error: movie element ',ispec,ispecele,'has invalid node index:',n1,n2,n3,n4,n5,n6,n7,n8 + call exit_mpi(myrank,'Error invalid movie element node index') + endif + + enddo !i + enddo !j + enddo !k + endif + + enddo !ispec + + end subroutine get_conn_for_movie + +#endif + + + +! +!------------------------------------------------------------------------------------------------- +! +#ifdef USE_HDF5 + + subroutine write_xdmf_vol_hdf5(pressure_io, divglob_io, div_io, veloc_io, curl_io, stress_io) + + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + logical, intent(in) :: pressure_io, divglob_io, div_io, veloc_io, curl_io, stress_io + + ! local parameters + integer :: i, ii + character(len=20) :: it_str,nelm_str,nglo_str + character(len=MAX_STRING_LEN) :: fname_xdmf_vol + character(len=MAX_STRING_LEN) :: fname_h5_data_vol_xdmf + + ! checks if anything do, only main process writes out xdmf file + if (myrank /= 0) return + + ! writeout xdmf file for volume movie + fname_xdmf_vol = trim(OUTPUT_FILES) // "/movie_volume.xmf" + fname_h5_data_vol_xdmf = "./movie_volume.h5" ! relative to movie_volume.xmf file + ! this seems to point to a wrong director: + ! fname_h5_data_vol_xdmf = trim(OUTPUT_FILES) // "/movie_volume.h5" + + open(unit=xdmf_vol, file=trim(fname_xdmf_vol), recl=256) + + ! definition of topology and geometry + ! refer only control nodes (8 or 27) as a coarse output + ! data array need to be extracted from full data array on GLL points + write(xdmf_vol,'(a)') '' + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + + ! loop for writing information of mesh partitions +! nelm_str = i2c(sum(nelm_par_proc_nio(:))*(NGLLX-1)*(NGLLY-1)*(NGLLZ-1)) +! nglo_str = i2c(sum(nglob_par_proc_nio(:))) +! +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/elm_conn' +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/x' +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/y' +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/z' +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) '' +! +! do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) +! +! ii = i*NTSTEP_BETWEEN_FRAMES +! !write(it_str, "(i6.6)") ii +! it_str = i2c(ii) +! +! write(xdmf_vol,*) '' +! write(xdmf_vol,*) '' +! enddo + + ! file finish + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + + close(xdmf_vol) + + end subroutine write_xdmf_vol_hdf5 + +#endif From 5e9769414c4146b03a72346d37f28a49644aea63 Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Mon, 28 Oct 2024 12:00:12 -0600 Subject: [PATCH 13/19] add surface/volume movie output in hdf5 --- src/meshfem3D/save_arrays_solver_hdf5.F90 | 300 ++-- src/meshfem3D/save_model_meshfiles_hdf5.F90 | 33 +- src/shared/shared_par.f90 | 3 + src/specfem3D/finalize_simulation.F90 | 3 + src/specfem3D/prepare_movie.f90 | 2 +- src/specfem3D/read_arrays_solver_hdf5.F90 | 474 +++--- src/specfem3D/read_forward_arrays_hdf5.F90 | 211 ++- src/specfem3D/save_forward_arrays_hdf5.F90 | 240 ++- src/specfem3D/save_kernels_hdf5.F90 | 137 +- src/specfem3D/specfem3D_par.F90 | 33 +- src/specfem3D/write_movie_surface_hdf5.F90 | 107 +- src/specfem3D/write_movie_volume_hdf5.F90 | 1496 +++++++++++++------ 12 files changed, 1790 insertions(+), 1249 deletions(-) diff --git a/src/meshfem3D/save_arrays_solver_hdf5.F90 b/src/meshfem3D/save_arrays_solver_hdf5.F90 index 058ca220f..154d6e54b 100644 --- a/src/meshfem3D/save_arrays_solver_hdf5.F90 +++ b/src/meshfem3D/save_arrays_solver_hdf5.F90 @@ -7,7 +7,7 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & nspec #ifdef USE_HDF5 - use shared_parameters, only: LOCAL_PATH, ATT_F_C_SOURCE + use shared_parameters, only: LOCAL_PATH, ATT_F_C_SOURCE, H5_COL use meshfem_models_par, only: & OCEANS,TRANSVERSE_ISOTROPY,ANISOTROPIC_3D_MANTLE, & ANISOTROPIC_INNER_CORE,ATTENUATION @@ -62,8 +62,6 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & ! MPI variables integer :: info, comm - logical :: if_collective = .true. - ! processor dependent group names character(len=64) :: gname_region @@ -350,10 +348,10 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & write(gname_region, "('reg',i1)") iregion_code call h5_open_group(gname_region) - call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes", (/offset_nnodes(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes_xy", (/offset_nnodes_xy(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes_oceans", (/offset_nnodes_oceans(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group("offset_nelems", (/offset_nelems(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes", (/offset_nnodes(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes_xy", (/offset_nnodes_xy(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes_oceans", (/offset_nnodes_oceans(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("offset_nelems", (/offset_nelems(myrank)/), (/myrank/), H5_COL) allocate(tmp_array(nglob),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array for mesh topology') @@ -372,7 +370,7 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('xstore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('xstore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) !--- y coordinate tmp_array(:) = 0._CUSTOM_REAL do ispec = 1,nspec @@ -386,7 +384,7 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & enddo enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('ystore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ystore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) !--- z coordinate tmp_array(:) = 0._CUSTOM_REAL do ispec = 1,nspec @@ -400,25 +398,25 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & enddo enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('zstore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('zstore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) deallocate(tmp_array) - call h5_write_dataset_collect_hyperslab_in_group('ibool', ibool, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('idoubling', idoubling, (/sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('ispec_is_tiso', ispec_is_tiso, (/sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('xixstore', xixstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('xiystore', xiystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('xizstore', xizstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('etaxstore', etaxstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('etaystore', etaystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('etazstore', etazstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('gammaxstore', gammaxstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('gammaystore', gammaystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('gammazstore', gammazstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('rhostore', rhostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('kappavstore', kappavstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibool', ibool, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('idoubling', idoubling, (/sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ispec_is_tiso', ispec_is_tiso, (/sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('xixstore', xixstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('xiystore', xiystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('xizstore', xizstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('etaxstore', etaxstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('etaystore', etaystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('etazstore', etazstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('gammaxstore', gammaxstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('gammaystore', gammaystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('gammazstore', gammazstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('rhostore', rhostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('kappavstore', kappavstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) if (iregion_code /= IREGION_OUTER_CORE) then - call h5_write_dataset_collect_hyperslab_in_group('muvstore', muvstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('muvstore', muvstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif select case(iregion_code) @@ -426,55 +424,55 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & ! crusl/mantle region mesh ! save anisotropy in the mantle only if (ANISOTROPIC_3D_MANTLE) then - call h5_write_dataset_collect_hyperslab_in_group('c11store', c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c12store', c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c13store', c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c14store', c14store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c15store', c15store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c16store', c16store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c22store', c22store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c23store', c23store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c24store', c24store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c25store', c25store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c26store', c26store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c33store', c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c34store', c34store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c35store', c35store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c36store', c36store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c44store', c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c45store', c45store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c46store', c46store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c55store', c55store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c56store', c56store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c66store', c66store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c11store', c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c12store', c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c13store', c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c14store', c14store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c15store', c15store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c16store', c16store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c22store', c22store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c23store', c23store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c24store', c24store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c25store', c25store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c26store', c26store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c33store', c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c34store', c34store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c35store', c35store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c36store', c36store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c44store', c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c45store', c45store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c46store', c46store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c55store', c55store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c56store', c56store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c66store', c66store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) else if (TRANSVERSE_ISOTROPY) then - call h5_write_dataset_collect_hyperslab_in_group('kappahstore', kappahstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('muhstore', muhstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('eta_anisostore', eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('kappahstore', kappahstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('muhstore', muhstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('eta_anisostore', eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif endif ! for azimutahl aniso kernels - call h5_write_dataset_collect_hyperslab_in_group('mu0store', mu0store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('mu0store', mu0store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) !! case(IREGION_INNER_CORE) ! inner core region mesh if (ANISOTROPIC_INNER_CORE) then - call h5_write_dataset_collect_hyperslab_in_group('c11store', c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c12store', c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c13store', c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c33store', c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('c44store', c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('c11store', c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c12store', c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c13store', c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c33store', c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c44store', c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif end select !! ! Stacy if (ABSORBING_CONDITIONS) then if (iregion_code == IREGION_CRUST_MANTLE) then - call h5_write_dataset_collect_hyperslab_in_group('rho_vp', rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('rho_vs', rho_vs, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('rho_vp', rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('rho_vs', rho_vs, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) else if (iregion_code == IREGION_OUTER_CORE) then - call h5_write_dataset_collect_hyperslab_in_group('rho_vp', rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('rho_vp', rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif endif !!! @@ -482,23 +480,23 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & if (((NCHUNKS /= 6 .and. ABSORBING_CONDITIONS) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & ((ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & ((ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) .and. iregion_code == IREGION_INNER_CORE)) then - call h5_write_dataset_collect_hyperslab_in_group('rmassx', rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('rmassy', rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('rmassx', rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('rmassy', rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) endif - call h5_write_dataset_collect_hyperslab_in_group('rmassz', rmassz, (/sum(offset_nnodes(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('rmassz', rmassz, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) ! maxx matrices for backward simulation when ROTATION is .true. if (ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) then if (iregion_code == IREGION_CRUST_MANTLE .or. iregion_code == IREGION_INNER_CORE) then - call h5_write_dataset_collect_hyperslab_in_group('b_rmassx', b_rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('b_rmassy', b_rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('b_rmassx', b_rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('b_rmassy', b_rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) endif endif ! ocean load mass matrix if (OCEANS .and. iregion_code == IREGION_CRUST_MANTLE) then - call h5_write_dataset_collect_hyperslab_in_group('rmass_ocean_load', rmass_ocean_load, (/sum(offset_nnodes_oceans(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('rmass_ocean_load', rmass_ocean_load, (/sum(offset_nnodes_oceans(0:myrank-1))/), H5_COL) endif ! close group for iregion_code @@ -515,49 +513,49 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & ! open group for iregion_code call h5_open_group(gname_region) - call h5_write_dataset_collect_hyperslab_in_group('nspec2D_xmin', (/act_arr_nspec2d_xmin(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('nspec2D_xmax', (/act_arr_nspec2d_xmax(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('nspec2D_ymin', (/act_arr_nspec2d_ymin(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('nspec2D_ymax', (/act_arr_nspec2d_ymax(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('nspec2D_bottom', (/act_arr_nspec2d_bottom(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('nspec2D_top', (/act_arr_nspec2d_top(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_xmin', (/act_arr_nspec2d_xmin(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_xmax', (/act_arr_nspec2d_xmax(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_ymin', (/act_arr_nspec2d_ymin(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_ymax', (/act_arr_nspec2d_ymax(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_bottom', (/act_arr_nspec2d_bottom(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_top', (/act_arr_nspec2d_top(myrank)/), (/myrank/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_xmin', (/arr_nspec2d_xmin(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_xmax', (/arr_nspec2d_xmax(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_ymin', (/arr_nspec2d_ymin(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_ymax', (/arr_nspec2d_ymax(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_bottom', (/arr_nspec2d_bottom(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_top', (/arr_nspec2d_top(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_xmin', (/arr_nspec2d_xmin(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_xmax', (/arr_nspec2d_xmax(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_ymin', (/arr_nspec2d_ymin(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_ymax', (/arr_nspec2d_ymax(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_bottom', (/arr_nspec2d_bottom(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_top', (/arr_nspec2d_top(myrank)/), (/myrank/), H5_COL) if (arr_nspec2d_xmin(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmin', ibelm_xmin, (/sum(arr_nspec2d_xmin(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('normal_xmin', normal_xmin, (/0,0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmin', jacobian2D_xmin, (/0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmin', ibelm_xmin, (/sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_xmin', normal_xmin, (/0,0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmin', jacobian2D_xmin, (/0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) endif if (arr_nspec2d_xmax(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmax', ibelm_xmax, (/sum(arr_nspec2d_xmax(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('normal_xmax', normal_xmax, (/0,0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmax', jacobian2D_xmax, (/0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmax', ibelm_xmax, (/sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_xmax', normal_xmax, (/0,0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmax', jacobian2D_xmax, (/0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) endif if (arr_nspec2d_ymin(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymin', ibelm_ymin, (/sum(arr_nspec2d_ymin(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('normal_ymin', normal_ymin, (/0,0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymin', jacobian2D_ymin, (/0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymin', ibelm_ymin, (/sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_ymin', normal_ymin, (/0,0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymin', jacobian2D_ymin, (/0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) endif if (arr_nspec2d_ymax(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymax', ibelm_ymax, (/sum(arr_nspec2d_ymax(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('normal_ymax', normal_ymax, (/0,0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymax', jacobian2D_ymax, (/0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymax', ibelm_ymax, (/sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_ymax', normal_ymax, (/0,0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymax', jacobian2D_ymax, (/0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) endif if (arr_nspec2d_bottom(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_bottom', ibelm_bottom, (/sum(arr_nspec2d_bottom(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('normal_bottom', normal_bottom, (/0,0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_bottom', jacobian2D_bottom, (/0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_bottom', ibelm_bottom, (/sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_bottom', normal_bottom, (/0,0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_bottom', jacobian2D_bottom, (/0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) endif if (arr_nspec2d_top(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_top', ibelm_top, (/sum(arr_nspec2d_top(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('normal_top', normal_top, (/0,0,0,sum(arr_nspec2d_top(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_top', jacobian2D_top, (/0,0,sum(arr_nspec2d_top(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_top', ibelm_top, (/sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_top', normal_top, (/0,0,0,sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_top', jacobian2D_top, (/0,0,sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) endif ! close group for iregion_code @@ -575,10 +573,10 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & call h5_open_group(gname_region) ! create dataset for attenuation - call h5_write_dataset_collect_hyperslab_in_group('tau_s_store', tau_s_store, (/myrank*N_SLS/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('tau_e_store', tau_e_store, (/0,0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('Qmu_store', Qmu_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('att_f_c_source', (/ATT_F_C_SOURCE/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('tau_s_store', tau_s_store, (/myrank*N_SLS/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('tau_e_store', tau_e_store, (/0,0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('Qmu_store', Qmu_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('att_f_c_source', (/ATT_F_C_SOURCE/), (/myrank/), H5_COL) call h5_close_group() call h5_close_file_p() @@ -607,7 +605,7 @@ subroutine save_arrays_boundary_hdf5() ! saves arrays for boundaries such as MOHO, 400 and 670 discontinuities #ifdef USE_HDF5 - use shared_parameters, only: LOCAL_PATH + use shared_parameters, only: LOCAL_PATH, H5_COL use constants, only: myrank,SUPPRESS_CRUSTAL_MESH,CUSTOM_REAL, & NDIM, NGLLX, NGLLY @@ -633,10 +631,6 @@ subroutine save_arrays_boundary_hdf5() #ifdef USE_HDF5 - ! local parameters - logical :: if_collective = .true. - - ! arrays for Moho, 400, 670 integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_moho_top integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_moho_bot @@ -738,25 +732,25 @@ subroutine save_arrays_boundary_hdf5() call h5_open_group(gname_region) ! write datasets - call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_MOHO', (/act_arr_nspec2d_moho_top(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_400', (/act_arr_nspec2d_400_top(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_670', (/act_arr_nspec2d_670_top(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_MOHO_top', (/act_arr_nspec2d_moho_top(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_MOHO_bot', (/act_arr_nspec2d_moho_bot(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_400_top', (/act_arr_nspec2d_400_top(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_400_bot', (/act_arr_nspec2d_400_bot(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_670_top', (/act_arr_nspec2d_670_top(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_670_bot', (/act_arr_nspec2d_670_bot(myrank)/), (/myrank/), if_collective) - - call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_top', ibelm_moho_top, (/sum(arr_nspec2d_moho_top(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_bot', ibelm_moho_bot, (/sum(arr_nspec2d_moho_bot(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('ibelm_400_top', ibelm_400_top, (/sum(arr_nspec2d_400_top(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('ibelm_400_bot', ibelm_400_bot, (/sum(arr_nspec2d_400_bot(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('ibelm_670_top', ibelm_670_top, (/sum(arr_nspec2d_670_top(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('ibelm_670_bot', ibelm_670_bot, (/sum(arr_nspec2d_670_bot(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('normal_moho', normal_moho, (/0,0,0,sum(arr_nspec2d_moho_top(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('normal_400', normal_400, (/0,0,0,sum(arr_nspec2d_400_top(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('normal_670', normal_670, (/0,0,0,sum(arr_nspec2d_670_top(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_MOHO', (/act_arr_nspec2d_moho_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_400', (/act_arr_nspec2d_400_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_670', (/act_arr_nspec2d_670_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_MOHO_top', (/act_arr_nspec2d_moho_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_MOHO_bot', (/act_arr_nspec2d_moho_bot(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_400_top', (/act_arr_nspec2d_400_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_400_bot', (/act_arr_nspec2d_400_bot(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_670_top', (/act_arr_nspec2d_670_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_670_bot', (/act_arr_nspec2d_670_bot(myrank)/), (/myrank/), H5_COL) + + call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_top', ibelm_moho_top, (/sum(arr_nspec2d_moho_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_bot', ibelm_moho_bot, (/sum(arr_nspec2d_moho_bot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_400_top', ibelm_400_top, (/sum(arr_nspec2d_400_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_400_bot', ibelm_400_bot, (/sum(arr_nspec2d_400_bot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_670_top', ibelm_670_top, (/sum(arr_nspec2d_670_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_670_bot', ibelm_670_bot, (/sum(arr_nspec2d_670_bot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_moho', normal_moho, (/0,0,0,sum(arr_nspec2d_moho_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_400', normal_400, (/0,0,0,sum(arr_nspec2d_400_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_670', normal_670, (/0,0,0,sum(arr_nspec2d_670_top(0:myrank-1))/), H5_COL) ! close group for iregion_code call h5_close_group() @@ -789,6 +783,7 @@ subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & num_colors_outer,num_colors_inner, & num_elem_colors) use constants + use shared_parameters, only: H5_COL #ifdef USE_HDF5 use meshfem_par, only: & @@ -822,7 +817,6 @@ subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & #ifdef USE_HDF5 ! local parameters character(len=64) :: gname_region - logical :: if_collective = .true. ! offset arrays integer, dimension(0:NPROCTOT-1) :: offset_num_interfaces @@ -915,41 +909,41 @@ subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & call h5_open_group(gname_region) ! write datasets - call h5_write_dataset_collect_hyperslab_in_group('offset_num_interfaces', (/offset_num_interfaces(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('offset_max_nibool_interfaces', (/offset_max_nibool_interfaces(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('offset_nspec_inner', (/offset_nspec_inner(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('offset_nspec_outer', (/offset_nspec_outer(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('offset_num_phase_ispec', (/offset_num_phase_ispec(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_outer', (/offset_num_colors_outer(myrank)/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_inner', (/offset_num_colors_inner(myrank)/), (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_interfaces', (/offset_num_interfaces(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_max_nibool_interfaces', (/offset_max_nibool_interfaces(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_nspec_inner', (/offset_nspec_inner(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_nspec_outer', (/offset_nspec_outer(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_phase_ispec', (/offset_num_phase_ispec(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_outer', (/offset_num_colors_outer(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_inner', (/offset_num_colors_inner(myrank)/), (/myrank/), H5_COL) if (sum(offset_num_interfaces) > 0) then - call h5_write_dataset_collect_hyperslab_in_group("max_nibool_interfaces", (/max_nibool_interfaces/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group("my_neighbors", my_neighbors, (/sum(offset_num_interfaces(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group("nibool_interfaces", nibool_interfaces, (/sum(offset_num_interfaces(0:myrank-1))/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group("ibool_interfaces", ibool_interfaces, (/0,sum(offset_num_interfaces(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("max_nibool_interfaces", (/max_nibool_interfaces/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("my_neighbors", my_neighbors, (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("nibool_interfaces", nibool_interfaces, (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("ibool_interfaces", ibool_interfaces, (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) else ! dummy - call h5_write_dataset_collect_hyperslab_in_group("max_nibool_interfaces", (/max_nibool_interfaces/), (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group("my_neighbors", my_neighbors, (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group("nibool_interfaces", nibool_interfaces, (/myrank/), if_collective) - call h5_write_dataset_collect_hyperslab_in_group("ibool_interfaces", ibool_interfaces, (/0,myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("max_nibool_interfaces", (/max_nibool_interfaces/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("my_neighbors", my_neighbors, (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("nibool_interfaces", nibool_interfaces, (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("ibool_interfaces", ibool_interfaces, (/0,myrank/), H5_COL) endif ! num_phase_ispec if (sum(offset_num_phase_ispec) > 0) then - call h5_write_dataset_collect_hyperslab_in_group("phase_ispec_inner", phase_ispec_inner, (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("phase_ispec_inner", phase_ispec_inner, (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) else ! dummy - call h5_write_dataset_collect_hyperslab_in_group("phase_ispec_inner", phase_ispec_inner, (/myrank,0/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("phase_ispec_inner", phase_ispec_inner, (/myrank,0/), H5_COL) endif ! mesh coloring if (sum(offset_num_colors_outer) + sum(offset_num_colors_inner) > 0) then - call h5_write_dataset_collect_hyperslab_in_group("num_elem_colors", num_elem_colors, (/sum(offset_num_colors_outer(0:myrank-1)) + sum(offset_num_colors_inner(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("num_elem_colors", num_elem_colors, (/sum(offset_num_colors_outer(0:myrank-1)) + sum(offset_num_colors_inner(0:myrank-1))/), H5_COL) else ! dummy - call h5_write_dataset_collect_hyperslab_in_group("num_elem_colors", num_elem_colors, (/myrank/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group("num_elem_colors", num_elem_colors, (/myrank/), H5_COL) endif ! close group for iregion_code @@ -975,6 +969,7 @@ subroutine get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & #ifdef USE_HDF5 use constants, only: myrank + use shared_parameters, only: H5_COL use meshfem_par, only: LOCAL_PATH, NPROCTOT use manager_hdf5 #endif @@ -1005,7 +1000,6 @@ subroutine get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & ! variables for HDF5 character(len=64) :: gname_region - logical :: if_col = .true. ! collective write ! get MPI parameters call world_get_comm(comm) @@ -1075,21 +1069,21 @@ subroutine get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & call h5_write_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", (/offset_num_abs_boundary_faces(myrank)/), (/myrank/), .true.) if (sum(offset_num_abs_boundary_faces) > 0) then - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) if (iregion == IREGION_CRUST_MANTLE) then - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) endif else ! dummy - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ispec", (/0/), (/myrank/), if_col) - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_npoin", (/0/), (/myrank/), if_col) - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ijk", i3d_dummy, (/0,0,myrank/), if_col) - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", r2d_dummy, (/0,myrank/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ispec", (/0/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_npoin", (/0/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ijk", i3d_dummy, (/0,0,myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", r2d_dummy, (/0,myrank/), H5_COL) if (iregion == IREGION_CRUST_MANTLE) then - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_normal", r3d_dummy, (/0,0,myrank/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_normal", r3d_dummy, (/0,0,myrank/), H5_COL) endif endif diff --git a/src/meshfem3D/save_model_meshfiles_hdf5.F90 b/src/meshfem3D/save_model_meshfiles_hdf5.F90 index df1077d93..399253cbf 100644 --- a/src/meshfem3D/save_model_meshfiles_hdf5.F90 +++ b/src/meshfem3D/save_model_meshfiles_hdf5.F90 @@ -32,7 +32,7 @@ subroutine save_model_meshfiles_hdf5() #ifdef USE_HDF5 ! outputs model files in binary format - use shared_parameters, only: R_PLANET,RHOAV,LOCAL_PATH + use shared_parameters, only: R_PLANET,RHOAV,LOCAL_PATH,H5_COL use meshfem_par, only: nspec,iregion_code,NPROCTOT @@ -63,9 +63,6 @@ subroutine save_model_meshfiles_hdf5() ! offset for nspec integer, dimension(0:NPROCTOT-1) :: offset_nelems - ! if collective write - logical :: if_collective = .true. - ! gather nspec call gather_all_all_singlei(nspec, offset_nelems, NPROCTOT) @@ -176,27 +173,27 @@ subroutine save_model_meshfiles_hdf5() temp_store(:,:,:,:) = sqrt((kappavstore(:,:,:,:) + 4.0_CUSTOM_REAL * muvstore(:,:,:,:)/3.0_CUSTOM_REAL)/rhostore(:,:,:,:)) & * scaleval1 dset_name = 'vpv' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! vph temp_store(:,:,:,:) = sqrt((kappahstore(:,:,:,:) + 4.0_CUSTOM_REAL * muhstore(:,:,:,:)/3.0_CUSTOM_REAL)/rhostore(:,:,:,:)) & * scaleval1 dset_name = 'vph' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! vsv temp_store(:,:,:,:) = sqrt( muvstore(:,:,:,:)/rhostore(:,:,:,:) )*scaleval1 dset_name = 'vsv' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! vsh temp_store(:,:,:,:) = sqrt( muhstore(:,:,:,:)/rhostore(:,:,:,:) )*scaleval1 dset_name = 'vsh' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! rho temp_store(:,:,:,:) = rhostore(:,:,:,:) * scaleval2 dset_name = 'rho' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! eta dset_name = 'eta' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) else ! isotropic model @@ -204,15 +201,15 @@ subroutine save_model_meshfiles_hdf5() temp_store(:,:,:,:) = sqrt((kappavstore(:,:,:,:) + 4.0_CUSTOM_REAL * muvstore(:,:,:,:)/3.0_CUSTOM_REAL)/rhostore(:,:,:,:)) & * scaleval1 dset_name = 'vp' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! vs temp_store(:,:,:,:) = sqrt( muvstore(:,:,:,:)/rhostore(:,:,:,:) )*scaleval1 dset_name = 'vs' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! rho temp_store(:,:,:,:) = rhostore(:,:,:,:) * scaleval2 dset_name = 'rho' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif ! TRANSVERSE_ISOTROPY @@ -224,14 +221,14 @@ subroutine save_model_meshfiles_hdf5() ! Gc_prime dset_name = 'Gc_prime' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, Gc_prime_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, Gc_prime_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! Gs_prime dset_name = 'Gs_prime' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, Gs_prime_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, Gs_prime_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! mu0 temp_store(:,:,:,:) = mu0store(:,:,:,:) * scale_GPa dset_name = 'mu0' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif @@ -252,7 +249,7 @@ subroutine save_model_meshfiles_hdf5() ! Qmu dset_name = 'Qmu' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif ! ATTENUATION @@ -261,7 +258,7 @@ subroutine save_model_meshfiles_hdf5() if (HETEROGEN_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then ! dvp dset_name = 'dvp' - call h5_write_dataset_collect_hyperslab_in_group(dset_name, dvpstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_collective) + call h5_write_dataset_collect_hyperslab_in_group(dset_name, dvpstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif ! close the group diff --git a/src/shared/shared_par.f90 b/src/shared/shared_par.f90 index 4aa1b6294..30fd48d1a 100644 --- a/src/shared/shared_par.f90 +++ b/src/shared/shared_par.f90 @@ -216,6 +216,9 @@ module shared_input_parameters ! number of io dedicated nodes integer :: HDF5_IO_NODES = 0 + ! HDF5 IO mode (collective or independent) + logical :: H5_COL = .true. + ! flag for io-dedicated/compute node. logical :: IO_storage_task = .false. logical :: IO_compute_task = .true. diff --git a/src/specfem3D/finalize_simulation.F90 b/src/specfem3D/finalize_simulation.F90 index 42e277a33..323f7a490 100644 --- a/src/specfem3D/finalize_simulation.F90 +++ b/src/specfem3D/finalize_simulation.F90 @@ -299,6 +299,9 @@ subroutine finalize_simulation_cleanup() if (MOVIE_VOLUME) then deallocate(nu_3dmovie) deallocate(mask_3dmovie,muvstore_crust_mantle_3dmovie) + if (HDF5_ENABLED) then + call movie_volume_finalize_hdf5() + endif endif ! noise simulations diff --git a/src/specfem3D/prepare_movie.f90 b/src/specfem3D/prepare_movie.f90 index 765d5ffa1..32b05a546 100644 --- a/src/specfem3D/prepare_movie.f90 +++ b/src/specfem3D/prepare_movie.f90 @@ -195,7 +195,7 @@ subroutine prepare_movie_volume() if (HDF5_ENABLED) then call write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovie,mask_ibool_3dmovie, & - muvstore_crust_mantle_3dmovie,npoints_3dmovie) + muvstore_crust_mantle_3dmovie,npoints_3dmovie,nspecel_3dmovie) else call write_movie_volume_mesh(nu_3dmovie,num_ibool_3dmovie,mask_3dmovie,mask_ibool_3dmovie, & muvstore_crust_mantle_3dmovie,npoints_3dmovie) diff --git a/src/specfem3D/read_arrays_solver_hdf5.F90 b/src/specfem3D/read_arrays_solver_hdf5.F90 index 2bf02ba02..1ea524a38 100644 --- a/src/specfem3D/read_arrays_solver_hdf5.F90 +++ b/src/specfem3D/read_arrays_solver_hdf5.F90 @@ -18,6 +18,7 @@ subroutine read_arrays_solver_hdf5(iregion_code, & use constants_solver #ifdef USE_HDF5 + use shared_parameters, only: H5_COL use specfem_par, only: & ABSORBING_CONDITIONS, & LOCAL_PATH,ABSORBING_CONDITIONS @@ -78,9 +79,6 @@ subroutine read_arrays_solver_hdf5(iregion_code, & ! MPI variables integer :: info, comm - ! if collective read - logical :: if_col = .true. - ! offset arrays integer, dimension(0:NPROCTOT_VAL-1) :: offset_nnodes integer, dimension(0:NPROCTOT_VAL-1) :: offset_nnodes_xy @@ -102,10 +100,10 @@ subroutine read_arrays_solver_hdf5(iregion_code, & ! open the group call h5_open_group(gname_region) - call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes", offset_nnodes, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes_xy", offset_nnodes_xy, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes_oceans", offset_nnodes_oceans, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("offset_nelems", offset_nelems, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes", offset_nnodes, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes_xy", offset_nnodes_xy, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes_oceans", offset_nnodes_oceans, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("offset_nelems", offset_nelems, (/0/), H5_COL) lnspec = offset_nelems(myrank) lnglob = offset_nnodes(myrank) @@ -128,35 +126,35 @@ subroutine read_arrays_solver_hdf5(iregion_code, & endif ! read xstore - call h5_read_dataset_collect_hyperslab_in_group("xstore", xstore, (/sum(offset_nnodes(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("xstore", xstore, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) ! read ystore - call h5_read_dataset_collect_hyperslab_in_group("ystore", ystore, (/sum(offset_nnodes(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ystore", ystore, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) ! read zstore - call h5_read_dataset_collect_hyperslab_in_group("zstore", zstore, (/sum(offset_nnodes(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("zstore", zstore, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) ! ibool - call h5_read_dataset_collect_hyperslab_in_group("ibool", ibool, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibool", ibool, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! idoubling - call h5_read_dataset_collect_hyperslab_in_group("idoubling", idoubling, (/sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("idoubling", idoubling, (/sum(offset_nelems(0:myrank-1))/), H5_COL) ! ispec_is_tiso - call h5_read_dataset_collect_hyperslab_in_group("ispec_is_tiso", ispec_is_tiso, (/sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ispec_is_tiso", ispec_is_tiso, (/sum(offset_nelems(0:myrank-1))/), H5_COL) ! xix - call h5_read_dataset_collect_hyperslab_in_group("xixstore", xix, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("xixstore", xix, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! xiy - call h5_read_dataset_collect_hyperslab_in_group("xiystore", xiy, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("xiystore", xiy, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! xiz - call h5_read_dataset_collect_hyperslab_in_group("xizstore", xiz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("xizstore", xiz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! etax - call h5_read_dataset_collect_hyperslab_in_group("etaxstore", etax, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("etaxstore", etax, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! etay - call h5_read_dataset_collect_hyperslab_in_group("etaystore", etay, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("etaystore", etay, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! etaz - call h5_read_dataset_collect_hyperslab_in_group("etazstore", etaz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("etazstore", etaz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! gammax - call h5_read_dataset_collect_hyperslab_in_group("gammaxstore", gammax, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("gammaxstore", gammax, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! gammay - call h5_read_dataset_collect_hyperslab_in_group("gammaystore", gammay, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("gammaystore", gammay, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! gammaz - call h5_read_dataset_collect_hyperslab_in_group("gammazstore", gammaz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("gammazstore", gammaz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) if (iregion_code == IREGION_TRINFINITE .or. iregion_code == IREGION_INFINITE) then call h5_close_group() @@ -165,13 +163,13 @@ subroutine read_arrays_solver_hdf5(iregion_code, & endif ! rhostore - call h5_read_dataset_collect_hyperslab_in_group("rhostore", rhostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("rhostore", rhostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! kappavstore - call h5_read_dataset_collect_hyperslab_in_group("kappavstore", kappavstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("kappavstore", kappavstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) if (iregion_code /= IREGION_OUTER_CORE) then ! muvstore - call h5_read_dataset_collect_hyperslab_in_group("muvstore", muvstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("muvstore", muvstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif select case (iregion_code) @@ -179,74 +177,74 @@ subroutine read_arrays_solver_hdf5(iregion_code, & ! crust/mantle if (ANISOTROPIC_3D_MANTLE_VAL) then ! c11store - call h5_read_dataset_collect_hyperslab_in_group("c11store", c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c11store", c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c12store - call h5_read_dataset_collect_hyperslab_in_group("c12store", c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c12store", c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c13store - call h5_read_dataset_collect_hyperslab_in_group("c13store", c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c13store", c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c14store - call h5_read_dataset_collect_hyperslab_in_group("c14store", c14store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c14store", c14store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c15store - call h5_read_dataset_collect_hyperslab_in_group("c15store", c15store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c15store", c15store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c16store - call h5_read_dataset_collect_hyperslab_in_group("c16store", c16store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c16store", c16store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c22store - call h5_read_dataset_collect_hyperslab_in_group("c22store", c22store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c22store", c22store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c23store - call h5_read_dataset_collect_hyperslab_in_group("c23store", c23store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c23store", c23store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c24store - call h5_read_dataset_collect_hyperslab_in_group("c24store", c24store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c24store", c24store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c25store - call h5_read_dataset_collect_hyperslab_in_group("c25store", c25store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c25store", c25store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c26store - call h5_read_dataset_collect_hyperslab_in_group("c26store", c26store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c26store", c26store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c33store - call h5_read_dataset_collect_hyperslab_in_group("c33store", c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c33store", c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c34store - call h5_read_dataset_collect_hyperslab_in_group("c34store", c34store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c34store", c34store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c35store - call h5_read_dataset_collect_hyperslab_in_group("c35store", c35store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c35store", c35store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c36store - call h5_read_dataset_collect_hyperslab_in_group("c36store", c36store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c36store", c36store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c44store - call h5_read_dataset_collect_hyperslab_in_group("c44store", c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c44store", c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c45store - call h5_read_dataset_collect_hyperslab_in_group("c45store", c45store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c45store", c45store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c46store - call h5_read_dataset_collect_hyperslab_in_group("c46store", c46store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c46store", c46store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c55store - call h5_read_dataset_collect_hyperslab_in_group("c55store", c55store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c55store", c55store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c56store - call h5_read_dataset_collect_hyperslab_in_group("c56store", c56store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c56store", c56store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c66store - call h5_read_dataset_collect_hyperslab_in_group("c66store", c66store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c66store", c66store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) else if (TRANSVERSE_ISOTROPY_VAL) then ! kappahstore - call h5_read_dataset_collect_hyperslab_in_group("kappahstore", kappahstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("kappahstore", kappahstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! muhstore - call h5_read_dataset_collect_hyperslab_in_group("muhstore", muhstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("muhstore", muhstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! eta_anisostore - call h5_read_dataset_collect_hyperslab_in_group("eta_anisostore", eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("eta_anisostore", eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif endif ! mu0store - call h5_read_dataset_collect_hyperslab_in_group("mu0store", mu0store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("mu0store", mu0store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) case (IREGION_INNER_CORE) ! inner core if (ANISOTROPIC_INNER_CORE_VAL) then ! c11store - call h5_read_dataset_collect_hyperslab_in_group("c11store", c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c11store", c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c12store - call h5_read_dataset_collect_hyperslab_in_group("c12store", c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c12store", c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c13store - call h5_read_dataset_collect_hyperslab_in_group("c13store", c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c13store", c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c33store - call h5_read_dataset_collect_hyperslab_in_group("c33store", c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c33store", c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! c44store - call h5_read_dataset_collect_hyperslab_in_group("c44store", c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("c44store", c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif end select @@ -254,12 +252,12 @@ subroutine read_arrays_solver_hdf5(iregion_code, & if (ABSORBING_CONDITIONS) then if (iregion_code == IREGION_CRUST_MANTLE) then ! rho_vp - call h5_read_dataset_collect_hyperslab_in_group("rho_vp", rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("rho_vp", rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! rho_vs - call h5_read_dataset_collect_hyperslab_in_group("rho_vs", rho_vs, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("rho_vs", rho_vs, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) else if (iregion_code == IREGION_OUTER_CORE) then ! rho_vp - call h5_read_dataset_collect_hyperslab_in_group("rho_vp", rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("rho_vp", rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif endif @@ -268,26 +266,26 @@ subroutine read_arrays_solver_hdf5(iregion_code, & ((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & ((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_INNER_CORE)) then ! rmassx - call h5_read_dataset_collect_hyperslab_in_group("rmassx", rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("rmassx", rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) ! rmassy - call h5_read_dataset_collect_hyperslab_in_group("rmassy", rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("rmassy", rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) endif ! rmassz - call h5_read_dataset_collect_hyperslab_in_group("rmassz", rmassz, (/sum(offset_nnodes(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("rmassz", rmassz, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) if (((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & ((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_INNER_CORE)) then ! b_rmassx - call h5_read_dataset_collect_hyperslab_in_group("b_rmassx", b_rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("b_rmassx", b_rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) ! b_rmassy - call h5_read_dataset_collect_hyperslab_in_group("b_rmassy", b_rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("b_rmassy", b_rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) endif ! read additional ocean load mass matrix if (OCEANS_VAL .and. iregion_code == IREGION_CRUST_MANTLE) then ! rmass_ocean_load - call h5_read_dataset_collect_hyperslab_in_group("rmass_ocean_load", rmass_ocean_load, (/sum(offset_nnodes_oceans(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("rmass_ocean_load", rmass_ocean_load, (/sum(offset_nnodes_oceans(0:myrank-1))/), H5_COL) endif ! close group @@ -337,8 +335,6 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) character(len=64) :: gname_region ! MPI variables integer :: info, comm - ! if collective read - logical :: if_col = .true. ! offset arrays integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_interfaces @@ -366,13 +362,13 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) ! open the group call h5_open_group(gname_region) - call h5_read_dataset_collect_hyperslab_in_group('offset_num_interfaces', offset_num_interfaces, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group('offset_max_nibool_interfaces', offset_max_nibool_interfaces, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group('offset_num_phase_ispec', offset_num_phase_ispec, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group('offset_nspec_inner', offset_nspec_inner, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group('offset_nspec_outer', offset_nspec_outer, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group('offset_num_colors_outer', offset_num_colors_outer, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group('offset_num_colors_inner', offset_num_colors_inner, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group('offset_num_interfaces', offset_num_interfaces, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_max_nibool_interfaces', offset_max_nibool_interfaces, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_num_phase_ispec', offset_num_phase_ispec, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_nspec_inner', offset_nspec_inner, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_nspec_outer', offset_nspec_outer, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_num_colors_outer', offset_num_colors_outer, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_num_colors_inner', offset_num_colors_inner, (/0/), H5_COL) num_interfaces = offset_num_interfaces(myrank) max_nibool_interfaces = offset_max_nibool_interfaces(myrank) @@ -399,10 +395,10 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) if (num_interfaces > 0) then call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & my_neighbors_crust_mantle(1:num_interfaces), & - (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & nibool_interfaces_crust_mantle(1:num_interfaces), & - (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) end if allocate(ibool_interfaces_crust_mantle(max_nibool_interfaces,num_interfaces), stat=ierr) @@ -411,7 +407,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) ibool_interfaces_crust_mantle(:,:) = 0 call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & ibool_interfaces_crust_mantle(1:max_nibool_interfaces,1:num_interfaces), & - (/0,sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) end if allocate(phase_ispec_inner_crust_mantle(num_phase_ispec,2), stat=ierr) @@ -420,7 +416,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) phase_ispec_inner_crust_mantle(:,:) = 0 call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & phase_ispec_inner_crust_mantle(1:num_phase_ispec,1:2), & - (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_col) + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) end if allocate(num_elem_colors_crust_mantle(num_colors_outer+num_colors_inner), stat=ierr) @@ -428,7 +424,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) if (USE_MESH_COLORING_GPU) then call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & num_elem_colors_crust_mantle(1:(num_colors_outer+num_colors_inner)), & - (/sum(offset_num_colors_outer(0:myrank-1))/), if_col) + (/sum(offset_num_colors_outer(0:myrank-1))/), H5_COL) end if case (IREGION_OUTER_CORE) @@ -447,10 +443,10 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) if (num_interfaces > 0) then call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & my_neighbors_outer_core(1:num_interfaces), & - (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & nibool_interfaces_outer_core(1:num_interfaces), & - (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) end if allocate(ibool_interfaces_outer_core(max_nibool_interfaces,num_interfaces), stat=ierr) @@ -459,7 +455,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) ibool_interfaces_outer_core(:,:) = 0 call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & ibool_interfaces_outer_core(1:max_nibool_interfaces,1:num_interfaces), & - (/0,sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) end if allocate(phase_ispec_inner_outer_core(num_phase_ispec,2), stat=ierr) @@ -468,7 +464,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) phase_ispec_inner_outer_core(:,:) = 0 call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & phase_ispec_inner_outer_core(1:num_phase_ispec,1:2), & - (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_col) + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) end if allocate(num_elem_colors_outer_core(num_colors_outer+num_colors_inner), stat=ierr) @@ -476,7 +472,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) if (USE_MESH_COLORING_GPU) then call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & num_elem_colors_outer_core(1:(num_colors_outer+num_colors_inner)), & - (/sum(offset_num_colors_outer(0:myrank-1))/), if_col) + (/sum(offset_num_colors_outer(0:myrank-1))/), H5_COL) end if case (IREGION_INNER_CORE) @@ -495,10 +491,10 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) if (num_interfaces > 0) then call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & my_neighbors_inner_core(1:num_interfaces), & - (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & nibool_interfaces_inner_core(1:num_interfaces), & - (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) end if allocate(ibool_interfaces_inner_core(max_nibool_interfaces,num_interfaces), stat=ierr) @@ -507,7 +503,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) ibool_interfaces_inner_core(:,:) = 0 call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & ibool_interfaces_inner_core(1:max_nibool_interfaces,1:num_interfaces), & - (/0,sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) end if allocate(phase_ispec_inner_inner_core(num_phase_ispec,2), stat=ierr) @@ -516,7 +512,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) phase_ispec_inner_inner_core(:,:) = 0 call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & phase_ispec_inner_inner_core(1:num_phase_ispec,1:2), & - (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_col) + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) end if allocate(num_elem_colors_inner_core(num_colors_outer+num_colors_inner), stat=ierr) @@ -524,7 +520,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) if (USE_MESH_COLORING_GPU) then call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & num_elem_colors_inner_core(1:(num_colors_outer+num_colors_inner)), & - (/sum(offset_num_colors_outer(0:myrank-1))/), if_col) + (/sum(offset_num_colors_outer(0:myrank-1))/), H5_COL) end if case (IREGION_TRINFINITE) @@ -543,10 +539,10 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) if (num_interfaces > 0) then call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & my_neighbors_trinfinite(1:num_interfaces), & - (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & nibool_interfaces_trinfinite(1:num_interfaces), & - (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) end if allocate(ibool_interfaces_trinfinite(max_nibool_interfaces,num_interfaces), stat=ierr) @@ -555,7 +551,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) ibool_interfaces_trinfinite(:,:) = 0 call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & ibool_interfaces_trinfinite(1:max_nibool_interfaces,1:num_interfaces), & - (/0,sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) end if allocate(phase_ispec_inner_trinfinite(num_phase_ispec,2), stat=ierr) @@ -564,7 +560,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) phase_ispec_inner_trinfinite(:,:) = 0 call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & phase_ispec_inner_trinfinite(1:num_phase_ispec,1:2), & - (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_col) + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) end if allocate(num_elem_colors_trinfinite(num_colors_outer+num_colors_inner), stat=ierr) @@ -572,7 +568,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) if (USE_MESH_COLORING_GPU) then call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & num_elem_colors_trinfinite(1:(num_colors_outer+num_colors_inner)), & - (/sum(offset_num_colors_outer(0:myrank-1))/), if_col) + (/sum(offset_num_colors_outer(0:myrank-1))/), H5_COL) end if case (IREGION_INFINITE) @@ -591,10 +587,10 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) if (num_interfaces > 0) then call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & my_neighbors_infinite(1:num_interfaces), & - (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & nibool_interfaces_infinite(1:num_interfaces), & - (/sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) end if allocate(ibool_interfaces_infinite(max_nibool_interfaces,num_interfaces), stat=ierr) @@ -603,7 +599,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) ibool_interfaces_infinite(:,:) = 0 call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & ibool_interfaces_infinite(1:max_nibool_interfaces,1:num_interfaces), & - (/0,sum(offset_num_interfaces(0:myrank-1))/), if_col) + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) end if allocate(phase_ispec_inner_infinite(num_phase_ispec,2), stat=ierr) @@ -612,7 +608,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) phase_ispec_inner_infinite(:,:) = 0 call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & phase_ispec_inner_infinite(1:num_phase_ispec,1:2), & - (/sum(offset_num_phase_ispec(0:myrank-1)),0/), if_col) + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) end if allocate(num_elem_colors_infinite(num_colors_outer+num_colors_inner), stat=ierr) @@ -620,7 +616,7 @@ subroutine read_mesh_databases_MPI_hdf5(iregion_code) if (USE_MESH_COLORING_GPU) then call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & num_elem_colors_infinite(1:(num_colors_outer+num_colors_inner)), & - (/sum(offset_num_colors_outer(0:myrank-1))/), if_col) + (/sum(offset_num_colors_outer(0:myrank-1))/), H5_COL) end if case default @@ -682,7 +678,6 @@ subroutine read_mesh_databases_coupling_hdf5() #ifdef USE_HDF5 character(len=64) :: gname_region - logical :: if_col = .true. ! MPI variables integer :: info, comm @@ -725,60 +720,60 @@ subroutine read_mesh_databases_coupling_hdf5() ! read actual number of elements ! nspec2D_xmin - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_crust_mantle, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_crust_mantle, (/myrank/), H5_COL) ! nspec2D_xmax - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_crust_mantle, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_crust_mantle, (/myrank/), H5_COL) ! nspec2D_ymin - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_crust_mantle, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_crust_mantle, (/myrank/), H5_COL) ! nspec2D_ymax - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_crust_mantle, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_crust_mantle, (/myrank/), H5_COL) ! read offset arrays (stored length) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) ! ibelm_xmin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_crust_mantle, (/sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_crust_mantle, (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! ibelm_xmax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_crust_mantle, (/sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_crust_mantle, (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! ibelm_ymin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_crust_mantle, (/sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_crust_mantle, (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! ibelm_ymax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_crust_mantle, (/sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_crust_mantle, (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! ibelm_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_crust_mantle, (/sum(offset_nspec2D_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_crust_mantle, (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! ibelm_bottom - call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_crust_mantle, (/sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_crust_mantle, (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! normal_xmin - call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_crust_mantle, (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_crust_mantle, (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! normal_xmax - call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_crust_mantle, (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_crust_mantle, (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! normal_ymin - call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_crust_mantle, (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_crust_mantle, (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! normal_ymax - call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_crust_mantle, (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_crust_mantle, (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! normal_top - call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_crust_mantle, (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_crust_mantle, (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! normal_bottom - call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_crust_mantle, (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_crust_mantle, (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! jacobian_xmin - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_crust_mantle, (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_crust_mantle, (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! jacobian_xmax - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_crust_mantle, (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_crust_mantle, (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! jacobian_ymin - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_crust_mantle, (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_crust_mantle, (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! jacobian_ymax - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_crust_mantle, (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_crust_mantle, (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! jacobian_top - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_crust_mantle, (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_crust_mantle, (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! jacobian_bottom - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_crust_mantle, (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_crust_mantle, (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -791,60 +786,60 @@ subroutine read_mesh_databases_coupling_hdf5() ! read actual number of elements ! nspec2D_xmin - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_outer_core, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_outer_core, (/myrank/), H5_COL) ! nspec2D_xmax - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_outer_core, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_outer_core, (/myrank/), H5_COL) ! nspec2D_ymin - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_outer_core, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_outer_core, (/myrank/), H5_COL) ! nspec2D_ymax - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_outer_core, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_outer_core, (/myrank/), H5_COL) ! read offset arrays (stored length) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) ! ibelm_xmin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_outer_core, (/sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_outer_core, (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! ibelm_xmax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_outer_core, (/sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_outer_core, (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! ibelm_ymin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_outer_core, (/sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_outer_core, (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! ibelm_ymax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_outer_core, (/sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_outer_core, (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! ibelm_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_outer_core, (/sum(offset_nspec2D_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_outer_core, (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! ibelm_bottom - call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_outer_core, (/sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_outer_core, (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! normal_xmin - call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_outer_core, (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_outer_core, (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! normal_xmax - call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_outer_core, (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_outer_core, (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! normal_ymin - call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_outer_core, (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_outer_core, (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! normal_ymax - call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_outer_core, (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_outer_core, (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! normal_top - call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_outer_core, (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_outer_core, (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! normal_bot - call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_outer_core, (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_outer_core, (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! jacobian_xmin - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_outer_core, (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_outer_core, (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! jacobian_xmax - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_outer_core, (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_outer_core, (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! jacobian_ymin - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_outer_core, (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_outer_core, (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! jacobian_ymax - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_outer_core, (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_outer_core, (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! jacobian_top - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_outer_core, (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_outer_core, (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! jacobian_bottom - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_outer_core, (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_outer_core, (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -858,34 +853,34 @@ subroutine read_mesh_databases_coupling_hdf5() ! read actual number of elements ! nspec2D_xmin - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_inner_core, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_inner_core, (/myrank/), H5_COL) ! nspec2D_xmax - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_inner_core, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_inner_core, (/myrank/), H5_COL) ! nspec2D_ymin - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_inner_core, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_inner_core, (/myrank/), H5_COL) ! nspec2D_ymax - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_inner_core, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_inner_core, (/myrank/), H5_COL) ! read offset arrays (stored length) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) ! ibelm_xmin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_inner_core, (/sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_inner_core, (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! ibelm_xmax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_inner_core, (/sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_inner_core, (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! ibelm_ymin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_inner_core, (/sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_inner_core, (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! ibelm_ymax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_inner_core, (/sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_inner_core, (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! ibelm_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_inner_core, (/sum(offset_nspec2D_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_inner_core, (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! ibelm_bottom - call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_inner_core, (/sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_inner_core, (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -900,34 +895,34 @@ subroutine read_mesh_databases_coupling_hdf5() ! read actual number of elements ! nspec2D_xmin - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_trinfinite, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_trinfinite, (/myrank/), H5_COL) ! nspec2D_xmax - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_trinfinite, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_trinfinite, (/myrank/), H5_COL) ! nspec2D_ymin - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_trinfinite, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_trinfinite, (/myrank/), H5_COL) ! nspec2D_ymax - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_trinfinite, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_trinfinite, (/myrank/), H5_COL) ! read offset arrays (stored length) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) ! ibelm_xmin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_trinfinite, (/sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_trinfinite, (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! ibelm_xmax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_trinfinite, (/sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_trinfinite, (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! ibelm_ymin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_trinfinite, (/sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_trinfinite, (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! ibelm_ymax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_trinfinite, (/sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_trinfinite, (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! ibelm_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_trinfinite, (/sum(offset_nspec2D_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_trinfinite, (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! ibelm_bottom - call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_trinfinite, (/sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_trinfinite, (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -941,34 +936,34 @@ subroutine read_mesh_databases_coupling_hdf5() ! read actual number of elements ! nspec2D_xmin - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_infinite, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_infinite, (/myrank/), H5_COL) ! nspec2D_xmax - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_infinite, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_infinite, (/myrank/), H5_COL) ! nspec2D_ymin - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_infinite, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_infinite, (/myrank/), H5_COL) ! nspec2D_ymax - call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_infinite, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_infinite, (/myrank/), H5_COL) ! read offset arrays (stored length) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), if_col) - call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) ! ibelm_xmin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_infinite, (/sum(offset_nspec2D_xmin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_infinite, (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! ibelm_xmax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_infinite, (/sum(offset_nspec2D_xmax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_infinite, (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! ibelm_ymin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_infinite, (/sum(offset_nspec2D_ymin(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_infinite, (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! ibelm_ymax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_infinite, (/sum(offset_nspec2D_ymax(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_infinite, (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! ibelm_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_infinite, (/sum(offset_nspec2D_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_infinite, (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! ibelm_bottom - call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_infinite, (/sum(offset_nspec2D_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_infinite, (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -989,11 +984,11 @@ subroutine read_mesh_databases_coupling_hdf5() call h5_open_group(gname_region) ! read NSPEC2D_MOHO - call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_MOHO", tmp_nspec2d_moho, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_MOHO", tmp_nspec2d_moho, (/myrank/), H5_COL) ! read NSPEC2D_400 - call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_400", tmp_nspec2d_400, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_400", tmp_nspec2d_400, (/myrank/), H5_COL) ! read NSPEC2D_670 - call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_670", tmp_nspec2d_670, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_670", tmp_nspec2d_670, (/myrank/), H5_COL) ! checks setup if (tmp_nspec2d_moho /= NSPEC2D_MOHO .or. tmp_nspec2d_400 /= NSPEC2D_400 .or. tmp_nspec2d_670 /= NSPEC2D_670) then @@ -1004,36 +999,36 @@ subroutine read_mesh_databases_coupling_hdf5() ! read the actual number of elements ! sub_NSPEC2D_MOHO_top - call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_MOHO_top", offset_nspec2D_moho_top, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_MOHO_top", offset_nspec2D_moho_top, (/0/), H5_COL) ! sub_NSPEC2D_MOHO_bottom - call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_MOHO_bottom", offset_nspec2D_moho_bottom, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_MOHO_bottom", offset_nspec2D_moho_bottom, (/0/), H5_COL) ! sub_NSPEC2D_400_top - call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_400_top", offset_nspec2D_400_top, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_400_top", offset_nspec2D_400_top, (/0/), H5_COL) ! sub_NSPEC2D_400_bottom - call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_400_bottom", offset_nspec2D_400_bottom, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_400_bottom", offset_nspec2D_400_bottom, (/0/), H5_COL) ! sub_NSPEC2D_670_top - call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_670_top", offset_nspec2D_670_top, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_670_top", offset_nspec2D_670_top, (/0/), H5_COL) ! sub_NSPEC2D_670_bottom - call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_670_bot", offset_nspec2D_670_bottom, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_670_bot", offset_nspec2D_670_bottom, (/0/), H5_COL) ! ibelm_moho_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_top", ibelm_moho_top, (/sum(offset_nspec2D_moho_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_top", ibelm_moho_top, (/sum(offset_nspec2D_moho_top(0:myrank-1))/), H5_COL) ! ibelm_moho_bot - call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_bot", ibelm_moho_bot, (/sum(offset_nspec2D_moho_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_bot", ibelm_moho_bot, (/sum(offset_nspec2D_moho_bottom(0:myrank-1))/), H5_COL) ! ibelm_400_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_top", ibelm_400_top, (/sum(offset_nspec2D_400_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_top", ibelm_400_top, (/sum(offset_nspec2D_400_top(0:myrank-1))/), H5_COL) ! ibelm_400_bot - call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_bot", ibelm_400_bot, (/sum(offset_nspec2D_400_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_bot", ibelm_400_bot, (/sum(offset_nspec2D_400_bottom(0:myrank-1))/), H5_COL) ! ibelm_670_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_top", ibelm_670_top, (/sum(offset_nspec2D_670_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_top", ibelm_670_top, (/sum(offset_nspec2D_670_top(0:myrank-1))/), H5_COL) ! ibelm_670_bot - call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_bot", ibelm_670_bot, (/sum(offset_nspec2D_670_bottom(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_bot", ibelm_670_bot, (/sum(offset_nspec2D_670_bottom(0:myrank-1))/), H5_COL) ! normal_moho - call h5_read_dataset_collect_hyperslab_in_group("normal_moho", normal_moho, (/0,0,0,sum(offset_nspec2D_moho_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_moho", normal_moho, (/0,0,0,sum(offset_nspec2D_moho_top(0:myrank-1))/), H5_COL) ! normal_400 - call h5_read_dataset_collect_hyperslab_in_group("normal_400", normal_400, (/0,0,0,sum(offset_nspec2D_400_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_400", normal_400, (/0,0,0,sum(offset_nspec2D_400_top(0:myrank-1))/), H5_COL) ! normal_670 - call h5_read_dataset_collect_hyperslab_in_group("normal_670", normal_670, (/0,0,0,sum(offset_nspec2D_670_top(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("normal_670", normal_670, (/0,0,0,sum(offset_nspec2D_670_top(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -1075,7 +1070,6 @@ subroutine read_mesh_databases_stacey_hdf5 ! hdf5 variables integer :: comm, info, ier character(len=64) :: gname_region - logical :: if_col = .true. ! read in collective mode call world_get_comm(comm) @@ -1098,7 +1092,7 @@ subroutine read_mesh_databases_stacey_hdf5 call h5_open_group(gname_region) ! read the offset_num_abs_boundary_faces - call h5_read_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", offset_num_abs_boundary_faces, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", offset_num_abs_boundary_faces, (/0/), H5_COL) ! for this process num_abs_boundary_faces_crust_mantle = offset_num_abs_boundary_faces(myrank) @@ -1117,15 +1111,15 @@ subroutine read_mesh_databases_stacey_hdf5 if (ier /= 0) stop 'Error allocating array abs_boundary_ispec etc.' ! abs_boundary_ispec - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_crust_mantle, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_crust_mantle, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_npoin - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_crust_mantle, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_crust_mantle, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_ijk - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_crust_mantle, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_crust_mantle, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_jacobian2Dw - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_crust_mantle, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_crust_mantle, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_normal - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_crust_mantle, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_crust_mantle, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) else ! dummy ! dummy arrays @@ -1155,7 +1149,7 @@ subroutine read_mesh_databases_stacey_hdf5 call h5_open_group(gname_region) ! read the offset_num_abs_boundary_faces - call h5_read_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", offset_num_abs_boundary_faces, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", offset_num_abs_boundary_faces, (/0/), H5_COL) ! for this process num_abs_boundary_faces_outer_core = offset_num_abs_boundary_faces(myrank) @@ -1174,15 +1168,15 @@ subroutine read_mesh_databases_stacey_hdf5 if (ier /= 0) stop 'Error allocating array abs_boundary_ispec etc.' ! abs_boundary_ispec - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_outer_core, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_outer_core, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_npoin - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_outer_core, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_outer_core, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_ijk - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_outer_core, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_outer_core, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_jacobian2Dw - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_outer_core, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_outer_core, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_normal - !call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_outer_core, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), if_col) + !call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_outer_core, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) else ! dummy ! dummy arrays @@ -1228,6 +1222,7 @@ subroutine read_attenuation_hdf5(iregion_code, factor_common, scale_factor, tau_ use constants_solver #ifdef USE_HDF5 + use shared_parameters, only: H5_COL use specfem_par, only: ATTENUATION_VAL,LOCAL_PATH use manager_hdf5 #endif @@ -1246,7 +1241,6 @@ subroutine read_attenuation_hdf5(iregion_code, factor_common, scale_factor, tau_ ! offset integer, dimension(0:NPROCTOT_VAL-1) :: offset_nelems ! hdf5 variables - logical :: if_col = .true. ! read in collective mode character(len=64) :: gname_region integer :: comm, info double precision, dimension(0:NPROCTOT_VAL-1) :: tmp_dp_arr @@ -1271,7 +1265,7 @@ subroutine read_attenuation_hdf5(iregion_code, factor_common, scale_factor, tau_ ! open the group call h5_open_group(gname_region) ! read the offset_nelems - call h5_read_dataset_collect_hyperslab_in_group("offset_nelems", offset_nelems, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("offset_nelems", offset_nelems, (/0/), H5_COL) ! close group and file call h5_close_group() call h5_close_file_p() @@ -1284,13 +1278,13 @@ subroutine read_attenuation_hdf5(iregion_code, factor_common, scale_factor, tau_ call h5_open_group(gname_region) ! tau_s - call h5_read_dataset_collect_hyperslab_in_group("tau_s_store", tau_s, (/myrank*N_SLS/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("tau_s_store", tau_s, (/myrank*N_SLS/), H5_COL) ! factor_common (tau_e_store) - call h5_read_dataset_collect_hyperslab_in_group("tau_e_store", factor_common, (/0,0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("tau_e_store", factor_common, (/0,0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! scale_factor (Qmu_store) - call h5_read_dataset_collect_hyperslab_in_group("Qmu_store", scale_factor, (/0,0,0,sum(offset_nelems(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("Qmu_store", scale_factor, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! f_c_source - call h5_read_dataset_collect_hyperslab_in_group("att_f_c_source", tmp_dp_arr, (/0/), if_col) + call h5_read_dataset_collect_hyperslab_in_group("att_f_c_source", tmp_dp_arr, (/0/), H5_COL) f_c_source = tmp_dp_arr(myrank) ! close group diff --git a/src/specfem3D/read_forward_arrays_hdf5.F90 b/src/specfem3D/read_forward_arrays_hdf5.F90 index 04954ff9e..e23c4c5de 100644 --- a/src/specfem3D/read_forward_arrays_hdf5.F90 +++ b/src/specfem3D/read_forward_arrays_hdf5.F90 @@ -46,9 +46,6 @@ subroutine read_intermediate_forward_arrays_hdf5() ! MPI variables integer :: info, comm - ! collective write - logical :: if_col = .true. - ! TODO HDF5: put offset array creation in a initialization process ! offset array integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm @@ -95,48 +92,48 @@ subroutine read_intermediate_forward_arrays_hdf5() call h5_open_file_p_collect(file_name) ! read the arrays - call h5_read_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_read_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('A_array_rotation', B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif if (FULL_GRAVITY_VAL) then - call h5_read_dataset_scalar_collect_hyperslab('neq', neq_read, (/myrank/), if_col) - call h5_read_dataset_scalar_collect_hyperslab('neq1', neq1_read, (/myrank/), if_col) - call h5_read_dataset_collect_hyperslab('pgrav1', pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), if_col) + call h5_read_dataset_scalar_collect_hyperslab('neq', neq_read, (/myrank/), H5_COL) + call h5_read_dataset_scalar_collect_hyperslab('neq1', neq1_read, (/myrank/), H5_COL) + call h5_read_dataset_collect_hyperslab('pgrav1', pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), H5_COL) endif ! close the file @@ -172,8 +169,6 @@ subroutine read_forward_arrays_hdf5() ! MPI variables integer :: info, comm - ! collective write - logical :: if_col = .true. ! TODO HDF5: put offset array creation in a initialization process ! offset array @@ -221,47 +216,47 @@ subroutine read_forward_arrays_hdf5() call h5_open_file_p_collect(file_name) ! read the arrays - call h5_read_dataset_collect_hyperslab('displ_crust_mantle', b_displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('displ_inner_core', b_displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', b_veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('veloc_inner_core', b_veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('displ_crust_mantle', b_displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_inner_core', b_displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', b_veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_inner_core', b_veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif if (FULL_GRAVITY_VAL) then - call h5_read_dataset_scalar_collect_hyperslab('neq', b_neq_read, (/myrank/), if_col) - call h5_read_dataset_scalar_collect_hyperslab('neq1', b_neq1_read, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab('neq', b_neq_read, (/myrank/), H5_COL) + call h5_read_dataset_scalar_collect_hyperslab('neq1', b_neq1_read, (/myrank/), H5_COL) ! check if array sizes match if (b_neq_read /= neq) then @@ -273,7 +268,7 @@ subroutine read_forward_arrays_hdf5() call exit_MPI(myrank,'Invalid forward array neq1 for startrun') endif - call h5_read_dataset_collect_hyperslab('pgrav1', b_pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('pgrav1', b_pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), H5_COL) endif ! close the file @@ -312,8 +307,6 @@ subroutine read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) ! MPI variables integer :: info, comm - ! collective write - logical :: if_col = .true. ! TODO HDF5: put offset array creation in a initialization process ! offset array @@ -362,47 +355,47 @@ subroutine read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) call h5_open_file_p_collect(file_name) ! read the arrays - call h5_read_dataset_collect_hyperslab('displ_crust_mantle', b_displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('displ_inner_core', b_displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', b_veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('veloc_inner_core', b_veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('displ_crust_mantle', b_displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_inner_core', b_displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', b_veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_inner_core', b_veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif if (FULL_GRAVITY_VAL) then - call h5_read_dataset_scalar_collect_hyperslab('neq', b_neq_read, (/myrank/), if_col) - call h5_read_dataset_scalar_collect_hyperslab('neq1', b_neq1_read, (/myrank/), if_col) + call h5_read_dataset_scalar_collect_hyperslab('neq', b_neq_read, (/myrank/), H5_COL) + call h5_read_dataset_scalar_collect_hyperslab('neq1', b_neq1_read, (/myrank/), H5_COL) ! check if array sizes match if (b_neq_read /= neq) then @@ -414,7 +407,7 @@ subroutine read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) call exit_MPI(myrank,'Invalid forward array neq1 for startrun') endif - call h5_read_dataset_collect_hyperslab('pgrav1', b_pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), if_col) + call h5_read_dataset_collect_hyperslab('pgrav1', b_pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), H5_COL) endif ! close the file diff --git a/src/specfem3D/save_forward_arrays_hdf5.F90 b/src/specfem3D/save_forward_arrays_hdf5.F90 index 65ed6aa21..08b149946 100644 --- a/src/specfem3D/save_forward_arrays_hdf5.F90 +++ b/src/specfem3D/save_forward_arrays_hdf5.F90 @@ -45,9 +45,6 @@ subroutine save_intermediate_forward_arrays_hdf5() ! MPI variables integer :: info, comm - ! collective write - logical :: if_col = .true. - ! TODO HDF5: put offset array creation in a initialization process ! offset array integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm @@ -150,48 +147,48 @@ subroutine save_intermediate_forward_arrays_hdf5() call h5_open_file_p_collect(file_name) ! write datasets - call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif ! ATTENUATION_VAL if (FULL_GRAVITY_VAL) then - call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), if_col) - call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), if_col) - call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), H5_COL) endif ! FULL_GRAVITY_VAL ! close file @@ -227,8 +224,6 @@ subroutine save_forward_arrays_hdf5() ! MPI variables integer :: info, comm - ! collective write - logical :: if_col = .true. ! TODO HDF5: put offset array creation in a initialization process ! offset array @@ -332,46 +327,46 @@ subroutine save_forward_arrays_hdf5() call h5_open_file_p_collect(file_name) ! write datasets - call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif ! ATTENUATION_VAL if (FULL_GRAVITY_VAL) then - call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), if_col) - call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), if_col) - call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), H5_COL) endif ! FULL_GRAVITY_VAL ! close file @@ -408,8 +403,6 @@ subroutine save_forward_arrays_undoatt_hdf5() ! MPI variables integer :: info, comm - ! collective write - logical :: if_col = .true. ! TODO HDF5: put offset array creation in a initialization process ! offset array @@ -514,48 +507,48 @@ subroutine save_forward_arrays_undoatt_hdf5() call h5_open_file_p_collect(file_name) ! write datasets - call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif ! ATTENUATION_VAL if (FULL_GRAVITY_VAL) then - call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), if_col) - call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), if_col) - call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), H5_COL) endif ! FULL_GRAVITY_VAL ! close file @@ -580,7 +573,7 @@ subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_cru use specfem_par_innercore #ifdef USE_HDF5 - use shared_parameters, only: R_PLANET,RHOAV,LOCAL_PATH,TRANSVERSE_ISOTROPY + use shared_parameters, only: R_PLANET,RHOAV,LOCAL_PATH,TRANSVERSE_ISOTROPY,H5_COL use manager_hdf5 #endif @@ -598,9 +591,6 @@ subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_cru real(kind=CUSTOM_REAL),dimension(:,:,:,:),allocatable :: muv_shifted,muh_shifted integer :: i,j,k,ispec - ! collective write - logical :: if_col = .true. - ! debug logical, parameter :: OUTPUT_RELAXED_MODEL = .false. @@ -761,27 +751,27 @@ subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_cru temp_store(:,:,:,:) = sqrt((kappavstore_crust_mantle(:,:,:,:) & + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & * scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vpv_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('vpv_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! vph temp_store(:,:,:,:) = sqrt((kappahstore_crust_mantle(:,:,:,:) & + FOUR_THIRDS * muh_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & * scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vph_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('vph_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! vsv temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vsv_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('vsv_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! vsh temp_store(:,:,:,:) = sqrt( muh_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vsh_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('vsh_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) else ! isotropic ! vp temp_store(:,:,:,:) = sqrt((kappavstore_crust_mantle(:,:,:,:) & + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & * scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! vs temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) endif @@ -835,10 +825,10 @@ subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_cru temp_store(:,:,:,:) = sqrt((kappavstore_inner_core(:,:,:,:) & + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_inner_core(:,:,:,:)) & * scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), H5_COL) ! vs temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_inner_core(:,:,:,:) )*scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), H5_COL) endif deallocate(temp_store,muv_shifted) @@ -867,13 +857,13 @@ subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_cru call h5_open_group(group_name) ! muv_relaxed - call h5_write_dataset_collect_hyperslab_in_group('muv_relaxed', muvstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('muv_relaxed', muvstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! muh_relaxed - call h5_write_dataset_collect_hyperslab_in_group('muh_relaxed', muhstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('muh_relaxed', muhstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! kappav_relaxed - call h5_write_dataset_collect_hyperslab_in_group('kappav_relaxed', kappavstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('kappav_relaxed', kappavstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! kappah_relaxed - call h5_write_dataset_collect_hyperslab_in_group('kappah_relaxed', kappahstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group('kappah_relaxed', kappahstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! close group call h5_close_group() diff --git a/src/specfem3D/save_kernels_hdf5.F90 b/src/specfem3D/save_kernels_hdf5.F90 index 0dfdfc49e..17dea71f9 100644 --- a/src/specfem3D/save_kernels_hdf5.F90 +++ b/src/specfem3D/save_kernels_hdf5.F90 @@ -44,7 +44,6 @@ subroutine write_kernels_strength_noise_hdf5() ! local parameters character(len=MAX_STRING_LEN) :: file_name integer :: info, comm - logical :: if_col = .true. ! gather the number of elements in each region call gather_all_all_singlei(size(sigma_kl_crust_mantle,4), offset_nspec_cm_adj, NPROCTOT_VAL) @@ -72,7 +71,7 @@ subroutine write_kernels_strength_noise_hdf5() call h5_open_file_p_collect(file_name) ! write data - call h5_write_dataset_collect_hyperslab('sigma_kernel', sigma_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('sigma_kernel', sigma_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) ! close hdf5 call h5_close_file_p() @@ -122,7 +121,6 @@ subroutine write_kernels_cm_ani_hdf5(alphav_kl_crust_mantle,alphah_kl_crust_mant ! local parameters character(len=MAX_STRING_LEN) :: file_name integer :: info, comm - logical :: if_col = .true. ! offset array integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_adj @@ -222,67 +220,67 @@ subroutine write_kernels_cm_ani_hdf5(alphav_kl_crust_mantle,alphah_kl_crust_mant ! write data if (SAVE_TRANSVERSE_KL_ONLY) then - call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) else if (SAVE_AZIMUTHAL_ANISO_KL_ONLY) then - call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('Gc_prime_kernel', Gc_prime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('Gs_prime_kernel', Gs_prime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('Gc_prime_kernel', Gc_prime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('Gs_prime_kernel', Gs_prime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) ! check isotropic kernel if (.false.) then - call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) endif ! check anisotropic kernels !if (.false.) then - ! call h5_write_dataset_collect_hyperslab('A_kernel', A_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('C_kernel', C_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('L_kernel', L_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('N_kernel', N_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('F_kernel', F_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('Gc_kernel', Gc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('Gs_kernel', Gs_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('Jc_kernel', Jc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('Kc_kernel', Kc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('Mc_kernel', Mc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('Bc_kernel', Bc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('Hc_kernel', Hc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('Ec_kernel', Ec_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - ! call h5_write_dataset_collect_hyperslab('Dc_kernel', Dc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + ! call h5_write_dataset_collect_hyperslab('A_kernel', A_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('C_kernel', C_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('L_kernel', L_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('N_kernel', N_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('F_kernel', F_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Gc_kernel', Gc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Gs_kernel', Gs_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Jc_kernel', Jc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Kc_kernel', Kc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Mc_kernel', Mc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Bc_kernel', Bc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Hc_kernel', Hc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Ec_kernel', Ec_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Dc_kernel', Dc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) !endif else ! fully anisotropic kernels - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('cijkl_kernel', cijkl_kl_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('cijkl_kernel', cijkl_kl_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) endif @@ -324,7 +322,6 @@ subroutine write_kernels_cm_iso_hdf5(mu_kl_crust_mantle, kappa_kl_crust_mantle, ! local parameters character(len=MAX_STRING_LEN) :: file_name integer :: info, comm - logical :: if_col = .true. ! checks if anything to do if (ANISOTROPIC_KL) return @@ -365,16 +362,16 @@ subroutine write_kernels_cm_iso_hdf5(mu_kl_crust_mantle, kappa_kl_crust_mantle, call h5_open_file_p_collect(file_name) ! write data - call h5_write_dataset_collect_hyperslab('rhonotprime_kernel', rhonotprime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('kappa_kernel', kappa_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('mu_kernel', mu_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('rhonotprime_kernel', rhonotprime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('kappa_kernel', kappa_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('mu_kernel', mu_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) ! close hdf5 call h5_close_file_p() @@ -408,7 +405,6 @@ subroutine write_kernels_oc_hdf5() ! local parameters character(len=MAX_STRING_LEN) :: file_name integer :: info, comm - logical :: if_col = .true. ! gather the number of elements in each region call gather_all_all_singlei(NSPEC_OUTER_CORE_ADJOINT, offset_nspec_oc_adj, NPROCTOT_VAL) @@ -437,8 +433,8 @@ subroutine write_kernels_oc_hdf5() call h5_open_file_p_collect(file_name) ! write data - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_outer_core, (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_outer_core, (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_outer_core, (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_outer_core, (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), H5_COL) ! close hdf5 call h5_close_file_p() @@ -472,7 +468,6 @@ subroutine write_kernels_ic_hdf5() ! local parameters character(len=MAX_STRING_LEN) :: file_name integer :: info, comm - logical :: if_col = .true. ! gather the number of elements in each region call gather_all_all_singlei(NSPEC_INNER_CORE_ADJOINT, offset_nspec_ic_adj, NPROCTOT_VAL) @@ -502,9 +497,9 @@ subroutine write_kernels_ic_hdf5() call h5_open_file_p_collect(file_name) ! write data - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) ! close hdf5 call h5_close_file_p() @@ -544,7 +539,6 @@ subroutine write_kernels_boundary_kl_hdf5() ! local parameters character(len=MAX_STRING_LEN) :: file_name integer :: info, comm - logical :: if_col = .true. if (.not. SAVE_KERNELS_BOUNDARY) return @@ -586,12 +580,12 @@ subroutine write_kernels_boundary_kl_hdf5() ! write data if (.not. SUPPRESS_CRUSTAL_MESH .and. HONOR_1D_SPHERICAL_MOHO) then - call h5_write_dataset_collect_hyperslab('moho_kernel', moho_kl, (/0,0,sum(offset_nspec2d_moho(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('moho_kernel', moho_kl, (/0,0,sum(offset_nspec2d_moho(0:myrank-1))/), H5_COL) endif - call h5_write_dataset_collect_hyperslab('d400_kernel', d400_kl, (/0,0,sum(offset_nspec2d_400(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('d670_kernel', d670_kl, (/0,0,sum(offset_nspec2d_670(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('CMB_kernel', CMB_kl, (/0,0,sum(offset_nspec2d_CMB(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('ICB_kernel', ICB_kl, (/0,0,sum(offset_nspec2d_ICB(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('d400_kernel', d400_kl, (/0,0,sum(offset_nspec2d_400(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('d670_kernel', d670_kl, (/0,0,sum(offset_nspec2d_670(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('CMB_kernel', CMB_kl, (/0,0,sum(offset_nspec2d_CMB(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('ICB_kernel', ICB_kl, (/0,0,sum(offset_nspec2d_ICB(0:myrank-1))/), H5_COL) ! close hdf5 call h5_close_file_p() @@ -625,7 +619,6 @@ subroutine write_kernels_Hessian_hdf5() ! local parameters character(len=MAX_STRING_LEN) :: file_name integer :: info, comm - logical :: if_col = .true. ! gather the number of elements in each region call gather_all_all_singlei(NSPEC_CRUST_MANTLE_ADJOINT, offset_nspec_cm_adj, NPROCTOT_VAL) @@ -656,10 +649,10 @@ subroutine write_kernels_Hessian_hdf5() call h5_open_file_p_collect(file_name) ! write data - call h5_write_dataset_collect_hyperslab('hess_kernel', hess_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('hess_rho_kernel', hess_rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('hess_kappa_kernel', hess_kappa_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab('hess_mu_kernel', hess_mu_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab('hess_kernel', hess_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('hess_rho_kernel', hess_rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('hess_kappa_kernel', hess_kappa_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('hess_mu_kernel', hess_mu_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) ! close hdf5 call h5_close_file_p() diff --git a/src/specfem3D/specfem3D_par.F90 b/src/specfem3D/specfem3D_par.F90 index ccee18320..2d874fec6 100644 --- a/src/specfem3D/specfem3D_par.F90 +++ b/src/specfem3D/specfem3D_par.F90 @@ -1531,7 +1531,6 @@ module specfem_par_movie_hdf5 use manager_hdf5 - logical :: if_col = .true. integer :: info, comm character(len=MAX_STRING_LEN) :: file_name, group_name @@ -1540,7 +1539,37 @@ module specfem_par_movie_hdf5 integer, dimension(:), allocatable :: offset_poin ! volume movie - + ! output parameters + logical, parameter :: MOVIE_OUTPUT_DIV = .true. ! divergence + logical, parameter :: MOVIE_OUTPUT_CURL = .true. ! curl + logical, parameter :: MOVIE_OUTPUT_CURLNORM = .true. ! Frobenius norm of curl + logical, parameter :: OUTPUT_CRUST_MANTLE = .true. + logical, parameter :: OUTPUT_OUTER_CORE = .true. + logical, parameter :: OUTPUT_INNER_CORE = .true. + + ! flags for check which region is output for movie + logical :: output_sv = .false. ! strain or vector output + logical :: output_cm = .false. + logical :: output_oc = .false. + logical :: output_ic = .false. + + integer :: npoints_vol_mov_all_proc + integer :: npoints_vol_mov_all_proc_cm + integer :: npoints_vol_mov_all_proc_oc + integer :: npoints_vol_mov_all_proc_ic + + integer :: nspec_vol_mov_all_proc + integer :: nspec_vol_mov_all_proc_cm + integer :: nspec_vol_mov_all_proc_oc + integer :: nspec_vol_mov_all_proc_ic + + ! number of elements for visualization (nspec * (NGLLX-1) * (NGLLY-1) * (NGLLZ-1)) + integer :: nspec_vol_mov_all_proc_cm_conn, nspec_vol_mov_all_proc_oc_conn, nspec_vol_mov_all_proc_ic_conn + + integer, dimension(:), allocatable :: offset_poin_vol, offset_nspec_vol + integer, dimension(:), allocatable :: offset_poin_vol_oc, offset_nspec_vol_oc + integer, dimension(:), allocatable :: offset_poin_vol_ic, offset_nspec_vol_ic + integer, dimension(:), allocatable :: offset_poin_vol_cm, offset_nspec_vol_cm ! xdmf integer :: xdmf_surf = 30000 diff --git a/src/specfem3D/write_movie_surface_hdf5.F90 b/src/specfem3D/write_movie_surface_hdf5.F90 index c856a10a8..28074e83a 100644 --- a/src/specfem3D/write_movie_surface_hdf5.F90 +++ b/src/specfem3D/write_movie_surface_hdf5.F90 @@ -25,7 +25,7 @@ ! !===================================================================== -subroutine movie_surface_init() +subroutine movie_surface_init_hdf5() #ifdef USE_HDF5 use specfem_par use specfem_par_movie_hdf5 @@ -45,7 +45,7 @@ subroutine movie_surface_init() write(*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' stop #endif -end subroutine movie_surface_init +end subroutine movie_surface_init_hdf5 subroutine movie_surface_finalize_hdf5() @@ -78,8 +78,9 @@ subroutine write_movie_surface_mesh_hdf5() ! local parameters real(kind=CUSTOM_REAL), dimension(:), allocatable :: store_val_x,store_val_y,store_val_z integer :: ipoin,ispec2D,ispec,i,j,k,ier,iglob1,iglob2,iglob3,iglob4,npoin + real(kind=CUSTOM_REAL) :: rval,thetaval,phival,xval,yval,zval - call movie_surface_init() + call movie_surface_init_hdf5() ! gather npoints on each process call gather_all_all_singlei(nmovie_points,offset_poin,NPROCTOT_VAL) @@ -137,52 +138,84 @@ subroutine write_movie_surface_mesh_hdf5() iglob3 = ibool_crust_mantle(i+1,j+1,k,ispec) iglob4 = ibool_crust_mantle(i,j+1,k,ispec) ! iglob1 - ipoin = ipoin + 1 - store_val_x(ipoin) = rstore_crust_mantle(1,iglob1) ! radius r (normalized) - store_val_y(ipoin) = rstore_crust_mantle(2,iglob1) ! colatitude theta (in radian) - store_val_z(ipoin) = rstore_crust_mantle(3,iglob1) ! longitude phi (in radian) + ipoin = ipoin + 1 + rval = rstore_crust_mantle(1,iglob1) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob1) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob1) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval ! iglob2 ipoin = ipoin + 1 - store_val_x(ipoin) = rstore_crust_mantle(1,iglob2) ! radius r (normalized) - store_val_y(ipoin) = rstore_crust_mantle(2,iglob2) ! colatitude theta (in radian) - store_val_z(ipoin) = rstore_crust_mantle(3,iglob2) ! longitude phi (in radian) + rval = rstore_crust_mantle(1,iglob2) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob2) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob2) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval ! iglob3 ipoin = ipoin + 1 - store_val_x(ipoin) = rstore_crust_mantle(1,iglob3) ! radius r (normalized) - store_val_y(ipoin) = rstore_crust_mantle(2,iglob3) ! colatitude theta (in radian) - store_val_z(ipoin) = rstore_crust_mantle(3,iglob3) ! longitude phi (in radian) + rval = rstore_crust_mantle(1,iglob3) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob3) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob3) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval ! iglob4 ipoin = ipoin + 1 - store_val_x(ipoin) = rstore_crust_mantle(1,iglob4) ! radius r (normalized) - store_val_y(ipoin) = rstore_crust_mantle(2,iglob4) ! colatitude theta (in radian) - store_val_z(ipoin) = rstore_crust_mantle(3,iglob4) ! longitude phi (in radian) + rval = rstore_crust_mantle(1,iglob4) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob4) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob4) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval enddo enddo else ! MOVIE_COARSE iglob1 = ibool_crust_mantle(1,1,k,ispec) - iglob2 = ibool_crust_mantle(NGLLX-1,1,k,ispec) - iglob3 = ibool_crust_mantle(NGLLX-1,NGLLY-1,k,ispec) - iglob4 = ibool_crust_mantle(1,NGLLY-1,k,ispec) + iglob2 = ibool_crust_mantle(NGLLX,1,k,ispec) + iglob3 = ibool_crust_mantle(NGLLX,NGLLY,k,ispec) + iglob4 = ibool_crust_mantle(1,NGLLY,k,ispec) ipoin = ipoin + 1 - store_val_x(ipoin) = rstore_crust_mantle(1,iglob1) ! radius r (normalized) - store_val_y(ipoin) = rstore_crust_mantle(2,iglob1) ! colatitude theta (in radian) - store_val_z(ipoin) = rstore_crust_mantle(3,iglob1) ! longitude phi (in radian) + rval = rstore_crust_mantle(1,iglob1) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob1) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob1) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval ipoin = ipoin + 1 - store_val_x(ipoin) = rstore_crust_mantle(1,iglob2) ! radius r (normalized) - store_val_y(ipoin) = rstore_crust_mantle(2,iglob2) ! colatitude theta (in radian) - store_val_z(ipoin) = rstore_crust_mantle(3,iglob2) ! longitude phi (in radian) + rval = rstore_crust_mantle(1,iglob2) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob2) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob2) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval ipoin = ipoin + 1 - store_val_x(ipoin) = rstore_crust_mantle(1,iglob3) ! radius r (normalized) - store_val_y(ipoin) = rstore_crust_mantle(2,iglob3) ! colatitude theta (in radian) - store_val_z(ipoin) = rstore_crust_mantle(3,iglob3) ! longitude phi (in radian) + rval = rstore_crust_mantle(1,iglob3) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob3) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob3) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval ipoin = ipoin + 1 - store_val_x(ipoin) = rstore_crust_mantle(1,iglob4) ! radius r (normalized) - store_val_y(ipoin) = rstore_crust_mantle(2,iglob4) ! colatitude theta (in radian) - store_val_z(ipoin) = rstore_crust_mantle(3,iglob4) ! longitude phi (in radian) + rval = rstore_crust_mantle(1,iglob4) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob4) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob4) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval endif enddo @@ -196,9 +229,9 @@ subroutine write_movie_surface_mesh_hdf5() call h5_open_group("surf_coord") ! write x, y, z - call h5_write_dataset_collect_hyperslab_in_group("x", store_val_x, (/sum(offset_poin(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group("y", store_val_y, (/sum(offset_poin(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group("z", store_val_z, (/sum(offset_poin(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("x", store_val_x, (/sum(offset_poin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("y", store_val_y, (/sum(offset_poin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("z", store_val_z, (/sum(offset_poin(0:myrank-1))/), H5_COL) ! close group and file call h5_close_group() @@ -394,9 +427,9 @@ subroutine write_movie_surface_hdf5() call h5_open_group(group_name) ! write ux, uy, uz - call h5_write_dataset_collect_hyperslab_in_group("ux", store_val_ux, (/sum(offset_poin(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group("uy", store_val_uy, (/sum(offset_poin(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group("uz", store_val_uz, (/sum(offset_poin(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group("ux", store_val_ux, (/sum(offset_poin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("uy", store_val_uy, (/sum(offset_poin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("uz", store_val_uz, (/sum(offset_poin(0:myrank-1))/), H5_COL) ! close group and file call h5_close_group() diff --git a/src/specfem3D/write_movie_volume_hdf5.F90 b/src/specfem3D/write_movie_volume_hdf5.F90 index 06a6e7cdd..6920d0350 100644 --- a/src/specfem3D/write_movie_volume_hdf5.F90 +++ b/src/specfem3D/write_movie_volume_hdf5.F90 @@ -25,11 +25,118 @@ ! !===================================================================== + subroutine movie_volume_init_hdf5() +#ifdef USE_HDF5 + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + integer :: ier + + ! nglobs + allocate(offset_poin_vol(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_poin_vol') + allocate(offset_poin_vol_cm(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_poin_vol_cm') + allocate(offset_poin_vol_oc(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_poin_vol_oc') + allocate(offset_poin_vol_ic(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_poin_vol_ic') + allocate(offset_nspec_vol(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_nspec_vol') + allocate(offset_nspec_vol_cm(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_nspec_vol_cm') + allocate(offset_nspec_vol_oc(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_nspec_vol_oc') + allocate(offset_nspec_vol_ic(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_nspec_vol_ic') + + npoints_vol_mov_all_proc = 0 + npoints_vol_mov_all_proc_cm = 0 + npoints_vol_mov_all_proc_oc = 0 + npoints_vol_mov_all_proc_ic = 0 + + nspec_vol_mov_all_proc = 0 + nspec_vol_mov_all_proc_cm = 0 + nspec_vol_mov_all_proc_oc = 0 + nspec_vol_mov_all_proc_ic = 0 + + nspec_vol_mov_all_proc_cm_conn = 0 + nspec_vol_mov_all_proc_oc_conn = 0 + nspec_vol_mov_all_proc_ic_conn = 0 + + ! if strain or vector output is requested + if ((MOVIE_VOLUME_TYPE == 1 .or. MOVIE_VOLUME_TYPE == 2 .or. MOVIE_VOLUME_TYPE == 3 & + .or. MOVIE_VOLUME_TYPE == 5 .or. MOVIE_VOLUME_TYPE == 6 )) then + output_sv = .true. + endif + ! check if crust mantle region is used for movie + if (MOVIE_VOLUME_TYPE == 4 .and. OUTPUT_CRUST_MANTLE) then + output_cm = .true. + endif + ! check if outer core region is used for movie + if ( (MOVIE_VOLUME_TYPE == 4 .or. MOVIE_VOLUME_TYPE == 7 .or. MOVIE_VOLUME_TYPE == 8 .or. MOVIE_VOLUME_TYPE == 9) & + .and. OUTPUT_OUTER_CORE) then + output_oc = .true. + endif + ! check if inner core region is used for movie + if ( (MOVIE_VOLUME_TYPE == 4 .or. MOVIE_VOLUME_TYPE == 7 .or. MOVIE_VOLUME_TYPE == 8 .or. MOVIE_VOLUME_TYPE == 9) & + .and. OUTPUT_INNER_CORE) then + output_ic = .true. + endif + + ! force false if number of elements is zero + if (NSPEC_CRUST_MANTLE == 0) output_cm = .false. + if (NSPEC_OUTER_CORE == 0) output_oc = .false. + if (NSPEC_INNER_CORE == 0) output_ic = .false. + + ! print + !print*, 'output_sv = ',output_sv + !print*, 'output_cm = ',output_cm + !print*, 'output_oc = ',output_oc + !print*, 'output_ic = ',output_ic + !print*, 'NSPEC_CRUST_MANTLE = ',NSPEC_CRUST_MANTLE + !print*, 'NSPEC_CRUST_MANTLE_STRAIN_ONLY = ',NSPEC_CRUST_MANTLE_STRAIN_ONLY + !print*, 'NSPEC_CRUST_MANTLE_STR_OR_ATT = ',NSPEC_CRUST_MANTLE_STR_OR_ATT + !print*, 'NSPEC_OUTER_CORE = ',NSPEC_OUTER_CORE + !print*, 'NSPEC_OUTER_CORE_3DMOVIE = ',NSPEC_OUTER_CORE_3DMOVIE + !print*, 'NSPEC_INNER_CORE = ',NSPEC_INNER_CORE + +#else + write (*,*) 'Error: HDF5 is not enabled in this version of the code.' + write (*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop +#endif + end subroutine movie_volume_init_hdf5 + + subroutine movie_volume_finalize_hdf5() +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 + + deallocate(offset_poin_vol) + deallocate(offset_poin_vol_cm) + deallocate(offset_poin_vol_oc) + deallocate(offset_poin_vol_ic) + deallocate(offset_nspec_vol) + deallocate(offset_nspec_vol_cm) + deallocate(offset_nspec_vol_oc) + deallocate(offset_nspec_vol_ic) + +#else + write (*,*) 'Error: HDF5 is not enabled in this version of the code.' + write (*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop +#endif + end subroutine movie_volume_finalize_hdf5 + subroutine write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovie,mask_ibool_3dmovie, & - muvstore_crust_mantle_3dmovie,npoints_3dmovie) + muvstore_crust_mantle_3dmovie,npoints_3dmovie,nelems_3dmovie_in) use specfem_par use specfem_par_crustmantle, only: ibool_crust_mantle,rstore_crust_mantle + use specfem_par_outercore, only: ibool_outer_core,rstore_outer_core + use specfem_par_innercore, only: ibool_inner_core,rstore_inner_core #ifdef USE_HDF5 use specfem_par_movie_hdf5 @@ -37,7 +144,7 @@ subroutine write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovi implicit none - integer,intent(in) :: npoints_3dmovie + integer,intent(in) :: npoints_3dmovie,nelems_3dmovie_in integer, dimension(NGLOB_CRUST_MANTLE_3DMOVIE),intent(in) :: num_ibool_3dmovie real(kind=CUSTOM_REAL), dimension(3,3,npoints_3dmovie),intent(inout) :: nu_3dmovie @@ -48,87 +155,272 @@ subroutine write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovi #ifdef USE_HDF5 ! local parameters - integer :: ipoints_3dmovie,ispec,i,j,k,iNIT + integer :: ipoints_3dmovie,ispec,i,j,k,iNIT,nelems_3dmovie integer :: iglob,iglob_center real(kind=CUSTOM_REAL) :: rval,thetaval,phival,xval,yval,zval,st,ct,sp,cp - real(kind=CUSTOM_REAL), dimension(npoints_3dmovie) :: store_val3D_x,store_val3D_y, store_val3D_z - real(kind=CUSTOM_REAL), dimension(npoints_3dmovie) :: store_val3D_mu - integer, dimension(0:NPROCTOT_VAL-1) :: offset_poin_vol - integer :: npoints_vol_mov_all_proc - integer, dimension(9,npoints_3dmovie) :: elm_conn + real(kind=CUSTOM_REAL), dimension(npoints_3dmovie) :: store_val3D_x, store_val3D_y, store_val3D_z + real(kind=CUSTOM_REAL), dimension(NGLOB_CRUST_MANTLE) :: store_val3D_x_cm, store_val3D_y_cm, store_val3D_z_cm + real(kind=CUSTOM_REAL), dimension(NGLOB_OUTER_CORE) :: store_val3D_x_oc, store_val3D_y_oc, store_val3D_z_oc + real(kind=CUSTOM_REAL), dimension(NGLOB_INNER_CORE) :: store_val3D_x_ic, store_val3D_y_ic, store_val3D_z_ic + real(kind=CUSTOM_REAL), dimension(npoints_3dmovie) :: store_val3D_mu + ! dummy num_ibool_3dmovie for cm oc ic + integer, dimension(NGLOB_CRUST_MANTLE) :: num_ibool_3dmovie_cm + integer, dimension(NGLOB_OUTER_CORE) :: num_ibool_3dmovie_oc + integer, dimension(NGLOB_INNER_CORE) :: num_ibool_3dmovie_ic + ! dummy mask_ibool_3dmovie for cm oc ic + logical, dimension(NGLOB_CRUST_MANTLE) :: mask_ibool_3dmovie_cm + logical, dimension(NGLOB_OUTER_CORE) :: mask_ibool_3dmovie_oc + logical, dimension(NGLOB_INNER_CORE) :: mask_ibool_3dmovie_ic + + integer, dimension(:,:), allocatable :: elm_conn, elm_conn_cm, elm_conn_oc, elm_conn_ic + + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_cm_conn, offset_nspec_vol_oc_conn, offset_nspec_vol_ic_conn + + integer :: nelems_3dmovie_cm, nelems_3dmovie_oc, nelems_3dmovie_ic + + ! initialize arrays for hdf5 volume movie output + call movie_volume_init_hdf5() ! safety check if (NDIM /= 3) stop 'movie volume output requires NDIM = 3' - ! prepare offset array - call gather_all_all_singlei(npoints_3dmovie, offset_poin_vol, NPROCTOT_VAL) - npoints_vol_mov_all_proc = sum(offset_poin_vol) - - ! stepping + ! output resolution if (MOVIE_COARSE) then iNIT = NGLLX-1 + nelems_3dmovie = nelems_3dmovie_in else iNIT = 1 + nelems_3dmovie = nelems_3dmovie_in * (NGLLX-1) * (NGLLY-1) * (NGLLZ-1) endif - ! loops over all elements - ipoints_3dmovie = 0 - do ispec = 1,NSPEC_CRUST_MANTLE + ! outer core and inner core is always fine mesh + nelems_3dmovie_cm = NSPEC_CRUST_MANTLE * (NGLLX-1) * (NGLLY-1) * (NGLLZ-1) + nelems_3dmovie_oc = NSPEC_OUTER_CORE * (NGLLX-1) * (NGLLY-1) * (NGLLZ-1) + nelems_3dmovie_ic = NSPEC_INNER_CORE * (NGLLX-1) * (NGLLY-1) * (NGLLZ-1) - ! checks center of element for movie flag - iglob_center = ibool_crust_mantle((NGLLX+1)/2,(NGLLY+1)/2,(NGLLZ+1)/2,ispec) + ! allocate elm_conn + allocate(elm_conn(9,nelems_3dmovie)) + allocate(elm_conn_cm(9,nelems_3dmovie_cm)) + allocate(elm_conn_oc(9,nelems_3dmovie_oc)) + allocate(elm_conn_ic(9,nelems_3dmovie_ic)) - ! checks if movie element - if (mask_ibool_3dmovie(iglob_center)) then + ! prepare offset array + call gather_all_all_singlei(npoints_3dmovie, offset_poin_vol, NPROCTOT_VAL) + call gather_all_all_singlei(NGLOB_CRUST_MANTLE, offset_poin_vol_cm, NPROCTOT_VAL) + call gather_all_all_singlei(NGLOB_OUTER_CORE, offset_poin_vol_oc, NPROCTOT_VAL) + call gather_all_all_singlei(NGLOB_INNER_CORE, offset_poin_vol_ic, NPROCTOT_VAL) + + call gather_all_all_singlei(nelems_3dmovie, offset_nspec_vol, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_vol_cm, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_vol_oc, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_vol_ic, NPROCTOT_VAL) + + ! offset arrays for element connectivity + call gather_all_all_singlei(nelems_3dmovie_cm, offset_nspec_vol_cm_conn, NPROCTOT_VAL) + call gather_all_all_singlei(nelems_3dmovie_oc, offset_nspec_vol_oc_conn, NPROCTOT_VAL) + call gather_all_all_singlei(nelems_3dmovie_ic, offset_nspec_vol_ic_conn, NPROCTOT_VAL) + + npoints_vol_mov_all_proc = sum(offset_poin_vol) + npoints_vol_mov_all_proc_cm = sum(offset_poin_vol_cm) + npoints_vol_mov_all_proc_oc = sum(offset_poin_vol_oc) + npoints_vol_mov_all_proc_ic = sum(offset_poin_vol_ic) + + nspec_vol_mov_all_proc = sum(offset_nspec_vol) + nspec_vol_mov_all_proc_cm = sum(offset_nspec_vol_cm) + nspec_vol_mov_all_proc_oc = sum(offset_nspec_vol_oc) + nspec_vol_mov_all_proc_ic = sum(offset_nspec_vol_ic) - ! stores element coordinates - do k = 1,NGLLZ,iNIT - do j = 1,NGLLY,iNIT - do i = 1,NGLLX,iNIT - ! only store points once - if (mask_3dmovie(i,j,k,ispec)) then - ! point increment - ipoints_3dmovie = ipoints_3dmovie + 1 - - ! gets point position - iglob = ibool_crust_mantle(i,j,k,ispec) - - rval = rstore_crust_mantle(1,iglob) - thetaval = rstore_crust_mantle(2,iglob) - phival = rstore_crust_mantle(3,iglob) - - !x,y,z store have been converted to r theta phi already, need to revert back for xyz output - call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) - - store_val3D_x(ipoints_3dmovie) = xval - store_val3D_y(ipoints_3dmovie) = yval - store_val3D_z(ipoints_3dmovie) = zval - store_val3D_mu(ipoints_3dmovie) = muvstore_crust_mantle_3dmovie(i,j,k,ispec) - - st = sin(thetaval) - ct = cos(thetaval) - sp = sin(phival) - cp = cos(phival) - - nu_3dmovie(1,1,ipoints_3dmovie) = -ct*cp - nu_3dmovie(1,2,ipoints_3dmovie) = -ct*sp - nu_3dmovie(1,3,ipoints_3dmovie) = st - nu_3dmovie(2,1,ipoints_3dmovie) = -sp - nu_3dmovie(2,2,ipoints_3dmovie) = cp - nu_3dmovie(2,3,ipoints_3dmovie) = 0.d0 - nu_3dmovie(3,1,ipoints_3dmovie) = st*cp - nu_3dmovie(3,2,ipoints_3dmovie) = st*sp - nu_3dmovie(3,3,ipoints_3dmovie) = ct - endif !mask_3dmovie - enddo !i - enddo !j - enddo !k + nspec_vol_mov_all_proc_cm_conn = sum(offset_nspec_vol_cm_conn) + nspec_vol_mov_all_proc_oc_conn = sum(offset_nspec_vol_oc_conn) + nspec_vol_mov_all_proc_ic_conn = sum(offset_nspec_vol_ic_conn) + + !if (myrank == 0) then + ! print*, 'npoints_vol_mov_all_proc = ', npoints_vol_mov_all_proc + ! print*, 'npoints_vol_mov_all_proc_cm = ', npoints_vol_mov_all_proc_cm + ! print*, 'npoints_vol_mov_all_proc_oc = ', npoints_vol_mov_all_proc_oc + ! print*, 'npoints_vol_mov_all_proc_ic = ', npoints_vol_mov_all_proc_ic + + ! print*, 'nspec_vol_mov_all_proc = ', nspec_vol_mov_all_proc + ! print*, 'nspec_vol_mov_all_proc_cm = ', nspec_vol_mov_all_proc_cm + ! print*, 'nspec_vol_mov_all_proc_oc = ', nspec_vol_mov_all_proc_oc + ! print*, 'nspec_vol_mov_all_proc_ic = ', nspec_vol_mov_all_proc_ic + + ! print*, 'nspec_vol_mov_all_proc_cm_conn = ', nspec_vol_mov_all_proc_cm_conn + ! print*, 'nspec_vol_mov_all_proc_oc_conn = ', nspec_vol_mov_all_proc_oc_conn + ! print*, 'nspec_vol_mov_all_proc_ic_conn = ', nspec_vol_mov_all_proc_ic_conn + !endif + + ! + ! create the xyz arrays for crust and mantle and strain and vector output + ! + if (output_sv) then + ! loops over all elements + ipoints_3dmovie = 0 + do ispec = 1,NSPEC_CRUST_MANTLE + + ! checks center of element for movie flag + iglob_center = ibool_crust_mantle((NGLLX+1)/2,(NGLLY+1)/2,(NGLLZ+1)/2,ispec) + + ! checks if movie element + if (mask_ibool_3dmovie(iglob_center)) then + + ! stores element coordinates + do k = 1,NGLLZ,iNIT + do j = 1,NGLLY,iNIT + do i = 1,NGLLX,iNIT + ! only store points once + if (mask_3dmovie(i,j,k,ispec)) then + ! point increment + ipoints_3dmovie = ipoints_3dmovie + 1 + + ! gets point position + iglob = ibool_crust_mantle(i,j,k,ispec) + + rval = rstore_crust_mantle(1,iglob) + thetaval = rstore_crust_mantle(2,iglob) + phival = rstore_crust_mantle(3,iglob) + + !x,y,z store have been converted to r theta phi already, need to revert back for xyz output + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + + store_val3D_x(ipoints_3dmovie) = xval + store_val3D_y(ipoints_3dmovie) = yval + store_val3D_z(ipoints_3dmovie) = zval + store_val3D_mu(ipoints_3dmovie) = muvstore_crust_mantle_3dmovie(i,j,k,ispec) + + st = sin(thetaval) + ct = cos(thetaval) + sp = sin(phival) + cp = cos(phival) + + nu_3dmovie(1,1,ipoints_3dmovie) = -ct*cp + nu_3dmovie(1,2,ipoints_3dmovie) = -ct*sp + nu_3dmovie(1,3,ipoints_3dmovie) = st + nu_3dmovie(2,1,ipoints_3dmovie) = -sp + nu_3dmovie(2,2,ipoints_3dmovie) = cp + nu_3dmovie(2,3,ipoints_3dmovie) = 0.d0 + nu_3dmovie(3,1,ipoints_3dmovie) = st*cp + nu_3dmovie(3,2,ipoints_3dmovie) = st*sp + nu_3dmovie(3,3,ipoints_3dmovie) = ct + endif !mask_3dmovie + enddo !i + enddo !j + enddo !k + endif + + enddo !ispec + + ! check if counters are correct + if (ipoints_3dmovie /= npoints_3dmovie) then + print*, 'Error: did not find the right number of points for 3D movie' + print*, 'ipoints_3dmovie = ',ipoints_3dmovie,' npoints_3dmovie = ',npoints_3dmovie + stop endif - enddo !ispec + endif + + ! + ! create the xyz arrays for crust and mantle (not strain or vector output) + ! + if (output_cm) then + + do ispec = 1,NSPEC_CRUST_MANTLE + do k = 1,NGLLZ,1 + do j = 1,NGLLY,1 + do i = 1,NGLLX,1 + iglob = ibool_crust_mantle(i,j,k,ispec) + rval = rstore_crust_mantle(1,iglob) + thetaval = rstore_crust_mantle(2,iglob) + phival = rstore_crust_mantle(3,iglob) + + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + + store_val3D_x_cm(iglob) = xval + store_val3D_y_cm(iglob) = yval + store_val3D_z_cm(iglob) = zval + + ! dummy num_ibool_3dmovie for cm + num_ibool_3dmovie_cm(iglob) = iglob + ! all the mask_ibool_3dmovie_cm are true + mask_ibool_3dmovie_cm(iglob) = .true. + enddo + enddo + enddo + enddo + + endif + + ! + ! create the xyz arrays for outer core + ! + if (output_oc) then + + do ispec = 1, NSPEC_OUTER_CORE + do k = 1,NGLLZ,1 + do j = 1,NGLLY,1 + do i = 1,NGLLX,1 + iglob = ibool_outer_core(i,j,k,ispec) + rval = rstore_outer_core(1,iglob) + thetaval = rstore_outer_core(2,iglob) + phival = rstore_outer_core(3,iglob) + + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + + store_val3D_x_oc(iglob) = xval + store_val3D_y_oc(iglob) = yval + store_val3D_z_oc(iglob) = zval + + ! dummy num_ibool_3dmovie for oc + num_ibool_3dmovie_oc(iglob) = iglob + ! all the mask_ibool_3dmovie_oc are true + mask_ibool_3dmovie_oc(iglob) = .true. + enddo + enddo + enddo + enddo + + endif + + ! + ! create the xyz arrays for inner core + ! + if (output_ic) then + + do ispec = 1, NSPEC_INNER_CORE + do k = 1,NGLLZ,1 + do j = 1,NGLLY,1 + do i = 1,NGLLX,1 + iglob = ibool_inner_core(i,j,k,ispec) + rval = rstore_inner_core(1,iglob) + thetaval = rstore_inner_core(2,iglob) + phival = rstore_inner_core(3,iglob) + + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + + store_val3D_x_ic(iglob) = xval + store_val3D_y_ic(iglob) = yval + store_val3D_z_ic(iglob) = zval + + ! dummy num_ibool_3dmovie for ic + num_ibool_3dmovie_ic(iglob) = iglob + ! all the mask_ibool_3dmovie_ic are true + mask_ibool_3dmovie_ic(iglob) = .true. + enddo + enddo + enddo + enddo + + endif ! create elm_conn for movie - call get_conn_for_movie(elm_conn, sum(offset_poin_vol(0:myrank-1)), iNIT, npoints_3dmovie, num_ibool_3dmovie, mask_ibool_3dmovie) + ! for crust and mantle (strain and vector output) + if (output_sv) call get_conn_for_movie(elm_conn, sum(offset_poin_vol(0:myrank-1)), iNIT, nelems_3dmovie, npoints_3dmovie, NSPEC_CRUST_MANTLE, num_ibool_3dmovie, mask_ibool_3dmovie, ibool_crust_mantle) + ! for crust and mantle (not strain or vector output) + if (output_cm) call get_conn_for_movie(elm_conn_cm, sum(offset_poin_vol_cm(0:myrank-1)), 1, nelems_3dmovie_cm, NGLOB_CRUST_MANTLE, NSPEC_CRUST_MANTLE, num_ibool_3dmovie_cm, mask_ibool_3dmovie_cm, ibool_crust_mantle) + ! for outer core + if (output_oc) call get_conn_for_movie(elm_conn_oc, sum(offset_poin_vol_oc(0:myrank-1)), 1, nelems_3dmovie_oc, NGLOB_OUTER_CORE, NSPEC_OUTER_CORE, num_ibool_3dmovie_oc, mask_ibool_3dmovie_oc, ibool_outer_core) + ! for inner core + if (output_ic) call get_conn_for_movie(elm_conn_ic, sum(offset_poin_vol_ic(0:myrank-1)), 1, nelems_3dmovie_ic, NGLOB_INNER_CORE, NSPEC_INNER_CORE, num_ibool_3dmovie_ic, mask_ibool_3dmovie_ic, ibool_inner_core) ! TODO ADD IOSERVER @@ -139,7 +431,7 @@ subroutine write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovi call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) ! file name and group name - file_name = 'movie_volume.h5' + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' group_name = 'mesh' ! create the file, group and dataset @@ -148,10 +440,34 @@ subroutine write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovi call h5_open_or_create_group(group_name) ! create the dataset - call h5_create_dataset_gen_in_group('elm_conn', (/9, npoints_vol_mov_all_proc/), 2, 1) - call h5_create_dataset_gen_in_group('x', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('y', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('z', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + ! for crust and mantle (strain and vector output) + if (output_sv) then + call h5_create_dataset_gen_in_group('elm_conn', (/9, nspec_vol_mov_all_proc/), 2, 1) + call h5_create_dataset_gen_in_group('x', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('y', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('z', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + endif + ! for crust and mantle (not strain or vector output) + if (output_cm) then + call h5_create_dataset_gen_in_group('elm_conn_cm', (/9, nspec_vol_mov_all_proc_cm_conn/), 2, 1) + call h5_create_dataset_gen_in_group('x_cm', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('y_cm', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('z_cm', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + endif + ! for outer core + if (output_oc) then + call h5_create_dataset_gen_in_group('elm_conn_oc', (/9, nspec_vol_mov_all_proc_oc_conn/), 2, 1) + call h5_create_dataset_gen_in_group('x_oc', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('y_oc', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('z_oc', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) + endif + ! for inner core + if (output_ic) then + call h5_create_dataset_gen_in_group('elm_conn_ic', (/9, nspec_vol_mov_all_proc_ic_conn/), 2, 1) + call h5_create_dataset_gen_in_group('x_ic', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('y_ic', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('z_ic', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + endif ! close the group call h5_close_group() @@ -160,23 +476,59 @@ subroutine write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovi endif + ! synchronize + call synchronize_all() + ! write the data call h5_open_file_p_collect(file_name) call h5_open_group(group_name) ! write the data - call h5_write_dataset_collect_hyperslab_in_group('elm_conn', elm_conn, (/0, sum(offset_poin_vol(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group('x', store_val3D_x, (/sum(offset_poin_vol(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group('y', store_val3D_y, (/sum(offset_poin_vol(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group('z', store_val3D_z, (/sum(offset_poin_vol(0:myrank-1))/), if_col) + ! for crust and mantle (strain and vector output) + if (output_sv) then + call h5_write_dataset_collect_hyperslab_in_group('elm_conn', elm_conn, (/0, sum(offset_nspec_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('x', store_val3D_x, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('y', store_val3D_y, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('z', store_val3D_z, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + endif + ! for crust and mantle (not strain or vector output) + if (output_cm) then + call h5_write_dataset_collect_hyperslab_in_group('elm_conn_cm', elm_conn_cm, (/0, sum(offset_nspec_vol_cm_conn(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('x_cm', store_val3D_x_cm, (/sum(offset_poin_vol_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('y_cm', store_val3D_y_cm, (/sum(offset_poin_vol_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('z_cm', store_val3D_z_cm, (/sum(offset_poin_vol_cm(0:myrank-1))/), H5_COL) + endif + ! for outer core + if (output_oc) then + call h5_write_dataset_collect_hyperslab_in_group('elm_conn_oc', elm_conn_oc, (/0, sum(offset_nspec_vol_oc_conn(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('x_oc', store_val3D_x_oc, (/sum(offset_poin_vol_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('y_oc', store_val3D_y_oc, (/sum(offset_poin_vol_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('z_oc', store_val3D_z_oc, (/sum(offset_poin_vol_oc(0:myrank-1))/), H5_COL) + endif + ! for inner core + if (output_ic) then + call h5_write_dataset_collect_hyperslab_in_group('elm_conn_ic', elm_conn_ic, (/0, sum(offset_nspec_vol_ic_conn(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('x_ic', store_val3D_x_ic, (/sum(offset_poin_vol_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('y_ic', store_val3D_y_ic, (/sum(offset_poin_vol_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('z_ic', store_val3D_z_ic, (/sum(offset_poin_vol_ic(0:myrank-1))/), H5_COL) + endif ! close the group call h5_close_group() ! close the file call h5_close_file_p() + ! deallocate + deallocate(elm_conn) + deallocate(elm_conn_cm) + deallocate(elm_conn_oc) + deallocate(elm_conn_ic) + ! write xdmf for all timesteps - call write_xdmf_vol_hdf5(.true., .false., .false., .false., .false., .false.) + call write_xdmf_vol_hdf5(npoints_vol_mov_all_proc, nspec_vol_mov_all_proc, & + npoints_vol_mov_all_proc_cm, nspec_vol_mov_all_proc_cm, & + npoints_vol_mov_all_proc_oc, nspec_vol_mov_all_proc_oc, & + npoints_vol_mov_all_proc_ic, nspec_vol_mov_all_proc_ic) #else @@ -196,7 +548,7 @@ subroutine write_movie_volume_strains_hdf5(vnspec_eps_cm, & epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle) use constants_solver - use shared_parameters, only: OUTPUT_FILES,MOVIE_VOLUME_TYPE,MOVIE_COARSE + use shared_parameters, only: OUTPUT_FILES,MOVIE_VOLUME_TYPE,MOVIE_COARSE,H5_COL use specfem_par, only: it use specfem_par_movie, only: npoints_3dmovie,muvstore_crust_mantle_3dmovie,mask_3dmovie,nu_3dmovie @@ -223,12 +575,6 @@ subroutine write_movie_volume_strains_hdf5(vnspec_eps_cm, & store_val3d_NE,store_val3d_NZ,store_val3d_EZ integer :: ipoints_3dmovie,i,j,k,ispec,iNIT,ier character(len=1) :: movie_prefix - integer, dimension(0:NPROCTOT_VAL-1) :: offset_poin_vol - integer :: npoints_vol_mov_all_proc - - ! prepare offset array - call gather_all_all_singlei(npoints_3dmovie, offset_poin_vol, NPROCTOT_VAL) - npoints_vol_mov_all_proc = sum(offset_poin_vol) ! check if (NDIM /= 3) call exit_MPI(myrank, 'write_movie_volume_strains() requires NDIM = 3') @@ -335,12 +681,12 @@ subroutine write_movie_volume_strains_hdf5(vnspec_eps_cm, & call h5_open_file_p_collect(file_name) call h5_open_group(group_name) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NN', store_val3d_NN, (/sum(offset_poin_vol(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EE', store_val3d_EE, (/sum(offset_poin_vol(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'ZZ', store_val3d_ZZ, (/sum(offset_poin_vol(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NE', store_val3d_NE, (/sum(offset_poin_vol(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NZ', store_val3d_NZ, (/sum(offset_poin_vol(0:myrank-1))/), if_col) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EZ', store_val3d_EZ, (/sum(offset_poin_vol(0:myrank-1))/), if_col) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NN', store_val3d_NN, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EE', store_val3d_EE, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'ZZ', store_val3d_ZZ, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NE', store_val3d_NE, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NZ', store_val3d_NZ, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EZ', store_val3d_EZ, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) call h5_close_group() call h5_close_file_p() @@ -374,8 +720,11 @@ subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_ ! outputs divergence and curl: MOVIE_VOLUME_TYPE == 4 use constants_solver - use shared_parameters, only: OUTPUT_FILES + use shared_parameters, only: OUTPUT_FILES,H5_COL use specfem_par, only: it + use specfem_par_crustmantle, only: ibool_crust_mantle + use specfem_par_innercore, only: ibool_inner_core + !use specfem_par_outercore, only: ibool_outer_core #ifdef USE_HDF5 use specfem_par_movie_hdf5 @@ -410,49 +759,7 @@ subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_ real(kind=CUSTOM_REAL) :: rhol,kappal real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: div_s_outer_core integer :: ispec,iglob,i,j,k,ier - real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: tmp_data - - ! output parameters - logical,parameter :: MOVIE_OUTPUT_DIV = .true. ! divergence - logical,parameter :: MOVIE_OUTPUT_CURL = .true. ! curl - logical,parameter :: MOVIE_OUTPUT_CURLNORM = .true. ! Frobenius norm of curl - - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_eps_cm - integer :: nspec_vol_mov_all_proc_eps_cm - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_eps_ic - integer :: nspec_vol_mov_all_proc_eps_ic - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_cm - integer :: nspec_vol_mov_all_proc_cm - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_oc - integer :: nspec_vol_mov_all_proc_oc - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_ic - integer :: nspec_vol_mov_all_proc_ic - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_cm_norm - integer :: nspec_vol_mov_all_proc_cm_norm - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_ic_norm - integer :: nspec_vol_mov_all_proc_ic_norm - - ! prepare offset array - call gather_all_all_singlei(vnspec_eps_cm, offset_nspec_vol_eps_cm, NPROCTOT_VAL) - nspec_vol_mov_all_proc_eps_cm = sum(offset_nspec_vol_cm) - call gather_all_all_singlei(vnspec_eps_ic, offset_nspec_vol_eps_ic, NPROCTOT_VAL) - nspec_vol_mov_all_proc_eps_ic = sum(offset_nspec_vol_ic) - call gather_all_all_singlei(vnspec_cm, offset_nspec_vol_cm, NPROCTOT_VAL) - nspec_vol_mov_all_proc_cm = sum(offset_nspec_vol_cm) - call gather_all_all_singlei(vnspec_ic, offset_nspec_vol_ic, NPROCTOT_VAL) - nspec_vol_mov_all_proc_ic = sum(offset_nspec_vol_ic) - if (NSPEC_OUTER_CORE_3DMOVIE > 1) then - call gather_all_all_singlei(NSPEC_OUTER_CORE_3DMOVIE, offset_nspec_vol_oc, NPROCTOT_VAL) - else - call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_vol_oc, NPROCTOT_VAL) - endif - nspec_vol_mov_all_proc_oc = sum(offset_nspec_vol_oc) - if (MOVIE_OUTPUT_CURLNORM) then - call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_vol_cm_norm, NPROCTOT_VAL) - nspec_vol_mov_all_proc_cm_norm = sum(offset_nspec_vol_cm_norm) - call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_vol_ic_norm, NPROCTOT_VAL) - nspec_vol_mov_all_proc_ic_norm = sum(offset_nspec_vol_ic_norm) - endif + real(kind=CUSTOM_REAL), dimension(:), allocatable :: tmp_data ! checks if (vnspec_cm /= NSPEC_CRUST_MANTLE) call exit_MPI(myrank,'Invalid vnspec_cm value for write_movie_volume_divcurl() routine') @@ -466,28 +773,28 @@ subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_ call h5_open_or_create_group(group_name) if (MOVIE_OUTPUT_DIV) then - call h5_create_dataset_gen_in_group('reg1_div_displ', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_eps_cm/), 4, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('reg2_div_displ', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_oc/), 4, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('reg3_div_displ', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_eps_ic/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg1_div_displ', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg2_div_displ', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg3_div_displ', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) endif if (MOVIE_OUTPUT_CURL) then - call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xx', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm/), 4, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_yy', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm/), 4, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xy', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm/), 4, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xz', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm/), 4, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_yz', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm/), 4, CUSTOM_REAL) - - call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xx', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic/), 4, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_yy', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic/), 4, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xy', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic/), 4, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xz', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic/), 4, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_yz', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xx', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_yy', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xy', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xz', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_yz', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xx', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_yy', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xy', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xz', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_yz', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) endif if (MOVIE_OUTPUT_CURLNORM) then - call h5_create_dataset_gen_in_group('reg1_epsdev_displ_norm', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_cm_norm/), 4, CUSTOM_REAL) - call h5_create_dataset_gen_in_group('reg3_epsdev_displ_norm', (/NGLLX,NGLLY,NGLLZ,nspec_vol_mov_all_proc_ic_norm/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg1_epsdev_displ_norm', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg3_epsdev_displ_norm', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) endif call h5_close_group() @@ -501,10 +808,10 @@ subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_ call h5_open_group(group_name) if (MOVIE_OUTPUT_DIV) then - call h5_write_dataset_collect_hyperslab_in_group('reg1_div_displ', eps_trace_over_3_crust_mantle, (/0,0,0,offset_nspec_vol_eps_cm(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg1_div_displ', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), eps_trace_over_3_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) if (NSPEC_OUTER_CORE_3DMOVIE > 1) then - call h5_write_dataset_collect_hyperslab_in_group('reg2_div_displ', ONE_THIRD*div_displ_outer_core, (/0,0,0,offset_nspec_vol_oc(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg2_div_displ', offset_nspec_vol_oc(myrank), offset_poin_vol_oc(myrank), div_displ_outer_core, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) else allocate(div_s_outer_core(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array div_s_outer_core') @@ -520,73 +827,71 @@ subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_ enddo enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg2_div_displ', div_s_outer_core, (/0,0,0,offset_nspec_vol_oc(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg2_div_displ', offset_nspec_vol_oc(myrank), offset_poin_vol_oc(myrank), div_s_outer_core, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) deallocate(div_s_outer_core) endif - call h5_write_dataset_collect_hyperslab_in_group('reg3_div_displ', eps_trace_over_3_inner_core, (/0,0,0,offset_nspec_vol_eps_ic(0:myrank-1)/), if_col) - + call write_array3dspec_as_1d_hdf5('reg3_div_displ', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), eps_trace_over_3_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) endif if (MOVIE_OUTPUT_CURL) then - call h5_write_dataset_collect_hyperslab_in_group('crust_mantle_epsdev_displ_xx', epsilondev_xx_crust_mantle, (/0,0,0,offset_nspec_vol_cm(0:myrank-1)/), if_col) - call h5_write_dataset_collect_hyperslab_in_group('crust_mantle_epsdev_displ_yy', epsilondev_yy_crust_mantle, (/0,0,0,offset_nspec_vol_cm(0:myrank-1)/), if_col) - call h5_write_dataset_collect_hyperslab_in_group('crust_mantle_epsdev_displ_xy', epsilondev_xy_crust_mantle, (/0,0,0,offset_nspec_vol_cm(0:myrank-1)/), if_col) - call h5_write_dataset_collect_hyperslab_in_group('crust_mantle_epsdev_displ_xz', epsilondev_xz_crust_mantle, (/0,0,0,offset_nspec_vol_cm(0:myrank-1)/), if_col) - call h5_write_dataset_collect_hyperslab_in_group('crust_mantle_epsdev_displ_yz', epsilondev_yz_crust_mantle, (/0,0,0,offset_nspec_vol_cm(0:myrank-1)/), if_col) - - call h5_write_dataset_collect_hyperslab_in_group('inner_core_epsdev_displ_xx', epsilondev_xx_inner_core, (/0,0,0,offset_nspec_vol_ic(0:myrank-1)/), if_col) - call h5_write_dataset_collect_hyperslab_in_group('inner_core_epsdev_displ_yy', epsilondev_yy_inner_core, (/0,0,0,offset_nspec_vol_ic(0:myrank-1)/), if_col) - call h5_write_dataset_collect_hyperslab_in_group('inner_core_epsdev_displ_xy', epsilondev_xy_inner_core, (/0,0,0,offset_nspec_vol_ic(0:myrank-1)/), if_col) - call h5_write_dataset_collect_hyperslab_in_group('inner_core_epsdev_displ_xz', epsilondev_xz_inner_core, (/0,0,0,offset_nspec_vol_ic(0:myrank-1)/), if_col) - call h5_write_dataset_collect_hyperslab_in_group('inner_core_epsdev_displ_yz', epsilondev_yz_inner_core, (/0,0,0,offset_nspec_vol_ic(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xx', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), epsilondev_xx_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_yy', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), epsilondev_yy_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xy', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), epsilondev_xy_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xz', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), epsilondev_xz_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_yz', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), epsilondev_yz_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xx', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), epsilondev_xx_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_yy', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), epsilondev_yy_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xy', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), epsilondev_xy_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xz', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), epsilondev_xz_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_yz', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), epsilondev_yz_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) endif if (MOVIE_OUTPUT_CURLNORM) then - allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE),stat=ier) + allocate(tmp_data(NGLOB_CRUST_MANTLE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') ! Frobenius norm do ispec = 1, NSPEC_CRUST_MANTLE do k = 1, NGLLZ do j = 1, NGLLY do i = 1, NGLLX - tmp_data(i,j,k,ispec) = sqrt( epsilondev_xx_crust_mantle(i,j,k,ispec)**2 & - + epsilondev_yy_crust_mantle(i,j,k,ispec)**2 & - + epsilondev_xy_crust_mantle(i,j,k,ispec)**2 & - + epsilondev_xz_crust_mantle(i,j,k,ispec)**2 & - + epsilondev_yz_crust_mantle(i,j,k,ispec)**2) + iglob = ibool_crust_mantle(i,j,k,ispec) + tmp_data(iglob) = sqrt( epsilondev_xx_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_yy_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_xy_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_xz_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_yz_crust_mantle(i,j,k,ispec)**2) enddo enddo enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg1_epsdev_displ_norm', tmp_data, (/0,0,0,offset_nspec_vol_cm_norm(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg1_epsdev_displ_norm', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) deallocate(tmp_data) ! Frobenius norm - allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE),stat=ier) + allocate(tmp_data(NGLOB_INNER_CORE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') do ispec = 1, NSPEC_INNER_CORE do k = 1, NGLLZ do j = 1, NGLLY do i = 1, NGLLX - tmp_data(i,j,k,ispec) = sqrt( epsilondev_xx_inner_core(i,j,k,ispec)**2 & - + epsilondev_yy_inner_core(i,j,k,ispec)**2 & - + epsilondev_xy_inner_core(i,j,k,ispec)**2 & - + epsilondev_xz_inner_core(i,j,k,ispec)**2 & - + epsilondev_yz_inner_core(i,j,k,ispec)**2) + tmp_data(iglob) = sqrt( epsilondev_xx_inner_core(i,j,k,ispec)**2 & + + epsilondev_yy_inner_core(i,j,k,ispec)**2 & + + epsilondev_xy_inner_core(i,j,k,ispec)**2 & + + epsilondev_xz_inner_core(i,j,k,ispec)**2 & + + epsilondev_yz_inner_core(i,j,k,ispec)**2) enddo enddo enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg3_epsdev_displ_norm', tmp_data, (/0,0,0,offset_nspec_vol_ic_norm(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg3_epsdev_displ_norm', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) deallocate(tmp_data) endif call h5_close_group() call h5_close_file() - !if (myrank == 0) call write_xdmf... - #else print*, 'Error: HDF5 is not enabled in this version of the code.' @@ -606,7 +911,7 @@ subroutine write_movie_volume_vector_hdf5(npoints_3dmovie, & ! outputs displacement/velocity: MOVIE_VOLUME_TYPE == 5 / 6 use constants_solver - use shared_parameters, only: OUTPUT_FILES,MOVIE_VOLUME_TYPE,MOVIE_COARSE + use shared_parameters, only: OUTPUT_FILES,MOVIE_VOLUME_TYPE,MOVIE_COARSE,H5_COL use specfem_par, only: it #ifdef USE_HDF5 @@ -635,17 +940,9 @@ subroutine write_movie_volume_vector_hdf5(npoints_3dmovie, & integer :: ipoints_3dmovie,i,j,k,ispec,iNIT,iglob,ier character(len=2) :: movie_prefix - integer, dimension(0:NPROCTOT_VAL-1) :: offset_npoints_3dmovie - integer :: npoints_3dmovie_all_proc - - ! check if (NDIM /= 3) call exit_MPI(myrank,'write_movie_volume requires NDIM = 3') - ! gather number of points - call gather_all_all_singlei(npoints_3dmovie, offset_npoints_3dmovie, NPROCTOT_VAL) - npoints_3dmovie_all_proc = sum(offset_npoints_3dmovie) - ! initialize h5 file for volume movie call world_get_comm(comm) call world_get_info_null(info) @@ -710,9 +1007,9 @@ subroutine write_movie_volume_vector_hdf5(npoints_3dmovie, & call h5_open_file(file_name) call h5_open_or_create_group(group_name) - call h5_create_dataset_gen_in_group(trim(movie_prefix)//'N', (/npoints_3dmovie_all_proc/), 1, CUSTOM_REAL) - call h5_create_dataset_gen_in_group(trim(movie_prefix)//'E', (/npoints_3dmovie_all_proc/), 1, CUSTOM_REAL) - call h5_create_dataset_gen_in_group(trim(movie_prefix)//'Z', (/npoints_3dmovie_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'N', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'E', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'Z', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) call h5_close_group() call h5_close_file() @@ -724,9 +1021,9 @@ subroutine write_movie_volume_vector_hdf5(npoints_3dmovie, & call h5_open_file_p_collect(file_name) call h5_open_group(group_name) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'N', store_val3d_N(1:npoints_3dmovie), (/offset_npoints_3dmovie(0:myrank-1)/), if_col) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'E', store_val3d_E(1:npoints_3dmovie), (/offset_npoints_3dmovie(0:myrank-1)/), if_col) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'Z', store_val3d_Z(1:npoints_3dmovie), (/offset_npoints_3dmovie(0:myrank-1)/), if_col) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'N', store_val3d_N(1:npoints_3dmovie), (/offset_poin_vol(0:myrank-1)/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'E', store_val3d_E(1:npoints_3dmovie), (/offset_poin_vol(0:myrank-1)/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'Z', store_val3d_Z(1:npoints_3dmovie), (/offset_poin_vol(0:myrank-1)/), H5_COL) call h5_close_group() call h5_close_file() @@ -751,7 +1048,7 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core ! outputs norm of displacement: MOVIE_VOLUME_TYPE == 7 use constants_solver - use shared_parameters, only: OUTPUT_FILES + use shared_parameters, only: OUTPUT_FILES,H5_COL use specfem_par, only: it, scale_displ #ifdef USE_HDF5 @@ -772,25 +1069,7 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core ! local parameters integer :: ispec,iglob,i,j,k,ier - real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: tmp_data - - logical,parameter :: OUTPUT_CRUST_MANTLE = .true. - logical,parameter :: OUTPUT_OUTER_CORE = .true. - logical,parameter :: OUTPUT_INNER_CORE = .true. - - ! offset arrays - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic - integer :: nspec_cm_all_proc,nspec_oc_all_proc,nspec_ic_all_proc - - ! gather number of points - call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_cm, NPROCTOT_VAL) - call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_oc, NPROCTOT_VAL) - call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_ic, NPROCTOT_VAL) - nspec_cm_all_proc = sum(offset_nspec_cm) - nspec_oc_all_proc = sum(offset_nspec_oc) - nspec_ic_all_proc = sum(offset_nspec_ic) + real(kind=CUSTOM_REAL), dimension(:), allocatable :: tmp_data ! initialize h5 file for volume movie call world_get_comm(comm) @@ -807,13 +1086,13 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core call h5_open_or_create_group(group_name) if (OUTPUT_CRUST_MANTLE) then - call h5_create_dataset_gen_in_group('reg1_displ', (/NGLLX,NGLLY,NGLLZ,nspec_cm_all_proc/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg1_displ', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) endif if (OUTPUT_OUTER_CORE) then - call h5_create_dataset_gen_in_group('reg2_displ', (/NGLLX,NGLLY,NGLLZ,nspec_oc_all_proc/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg2_displ', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) endif if (OUTPUT_INNER_CORE) then - call h5_create_dataset_gen_in_group('reg3_displ', (/NGLLX,NGLLY,NGLLZ,nspec_ic_all_proc/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg3_displ', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) endif call h5_close_group() @@ -830,7 +1109,7 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core if (OUTPUT_CRUST_MANTLE) then ! crust mantle ! these binary arrays can be converted into mesh format using the utility ./bin/xcombine_vol_data - allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE),stat=ier) + allocate(tmp_data(NGLOB_CRUST_MANTLE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') do ispec = 1, NSPEC_CRUST_MANTLE @@ -839,7 +1118,7 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core do i = 1, NGLLX iglob = ibool_crust_mantle(i,j,k,ispec) ! norm - tmp_data(i,j,k,ispec) = real(scale_displ,kind=CUSTOM_REAL) * sqrt( displ_crust_mantle(1,iglob)**2 & + tmp_data(iglob) = real(scale_displ,kind=CUSTOM_REAL) * sqrt( displ_crust_mantle(1,iglob)**2 & + displ_crust_mantle(2,iglob)**2 & + displ_crust_mantle(3,iglob)**2 ) enddo @@ -847,14 +1126,14 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg1_displ', tmp_data, (/0,0,0,offset_nspec_cm(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg1_displ', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) deallocate(tmp_data) endif if (OUTPUT_OUTER_CORE) then ! outer core - allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE),stat=ier) + allocate(tmp_data(NGLOB_OUTER_CORE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') do ispec = 1, NSPEC_OUTER_CORE @@ -865,20 +1144,20 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core ! norm ! note: disp_outer_core is potential, this just outputs the potential, ! not the actual displacement u = grad(rho * Chi) / rho - tmp_data(i,j,k,ispec) = abs(displ_outer_core(iglob)) + tmp_data(iglob) = abs(displ_outer_core(iglob)) enddo enddo enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg2_displ', tmp_data, (/0,0,0,offset_nspec_oc(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg2_displ', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) deallocate(tmp_data) endif if (OUTPUT_INNER_CORE) then ! inner core - allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE),stat=ier) + allocate(tmp_data(NGLOB_INNER_CORE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') do ispec = 1, NSPEC_INNER_CORE @@ -887,7 +1166,7 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core do i = 1, NGLLX iglob = ibool_inner_core(i,j,k,ispec) ! norm - tmp_data(i,j,k,ispec) = real(scale_displ,kind=CUSTOM_REAL) * sqrt( displ_inner_core(1,iglob)**2 & + tmp_data(iglob) = real(scale_displ,kind=CUSTOM_REAL) * sqrt( displ_inner_core(1,iglob)**2 & + displ_inner_core(2,iglob)**2 & + displ_inner_core(3,iglob)**2 ) enddo @@ -895,7 +1174,7 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg3_displ', tmp_data, (/0,0,0,offset_nspec_ic(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg3_displ', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) deallocate(tmp_data) endif @@ -921,7 +1200,7 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v ! outputs norm of velocity: MOVIE_VOLUME_TYPE == 8 use constants_solver - use shared_parameters, only: OUTPUT_FILES + use shared_parameters, only: OUTPUT_FILES,H5_COL use specfem_par, only: it, scale_veloc #ifdef USE_HDF5 @@ -942,25 +1221,7 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v ! local parameters integer :: ispec,iglob,i,j,k,ier - real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: tmp_data - - logical,parameter :: OUTPUT_CRUST_MANTLE = .true. - logical,parameter :: OUTPUT_OUTER_CORE = .true. - logical,parameter :: OUTPUT_INNER_CORE = .true. - - ! offset arrays - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic - integer :: nspec_cm_all_proc,nspec_oc_all_proc,nspec_ic_all_proc - - ! gather number of points - call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_cm, NPROCTOT_VAL) - call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_oc, NPROCTOT_VAL) - call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_ic, NPROCTOT_VAL) - nspec_cm_all_proc = sum(offset_nspec_cm) - nspec_oc_all_proc = sum(offset_nspec_oc) - nspec_ic_all_proc = sum(offset_nspec_ic) + real(kind=CUSTOM_REAL), dimension(:), allocatable :: tmp_data ! initialize h5 file for volume movie call world_get_comm(comm) @@ -977,13 +1238,13 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v call h5_open_or_create_group(group_name) if (OUTPUT_CRUST_MANTLE) then - call h5_create_dataset_gen_in_group('reg1_veloc', (/NGLLX,NGLLY,NGLLZ,nspec_cm_all_proc/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg1_veloc', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) endif if (OUTPUT_OUTER_CORE) then - call h5_create_dataset_gen_in_group('reg2_veloc', (/NGLLX,NGLLY,NGLLZ,nspec_oc_all_proc/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg2_veloc', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) endif if (OUTPUT_INNER_CORE) then - call h5_create_dataset_gen_in_group('reg3_veloc', (/NGLLX,NGLLY,NGLLZ,nspec_ic_all_proc/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg3_veloc', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) endif call h5_close_group() @@ -1000,7 +1261,7 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v if (OUTPUT_CRUST_MANTLE) then ! crust mantle ! these binary arrays can be converted into mesh format using the utility ./bin/xcombine_vol_data - allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE),stat=ier) + allocate(tmp_data(NGLOB_CRUST_MANTLE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') do ispec = 1, NSPEC_CRUST_MANTLE @@ -1009,7 +1270,7 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v do i = 1, NGLLX iglob = ibool_crust_mantle(i,j,k,ispec) ! norm of velocity - tmp_data(i,j,k,ispec) = real(scale_veloc,kind=CUSTOM_REAL) * sqrt( veloc_crust_mantle(1,iglob)**2 & + tmp_data(iglob) = real(scale_veloc,kind=CUSTOM_REAL) * sqrt( veloc_crust_mantle(1,iglob)**2 & + veloc_crust_mantle(2,iglob)**2 & + veloc_crust_mantle(3,iglob)**2 ) enddo @@ -1017,14 +1278,14 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg1_veloc', tmp_data, (/0,0,0,offset_nspec_cm(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg1_veloc', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) deallocate(tmp_data) endif if (OUTPUT_OUTER_CORE) then ! outer core - allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE),stat=ier) + allocate(tmp_data(NGLOB_OUTER_CORE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') do ispec = 1, NSPEC_OUTER_CORE @@ -1035,20 +1296,20 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v ! norm of velocity ! note: this outputs only the first time derivative of the potential, ! not the actual velocity v = grad(Chi_dot) - tmp_data(i,j,k,ispec) = abs(veloc_outer_core(iglob)) + tmp_data(iglob) = abs(veloc_outer_core(iglob)) enddo enddo enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg2_veloc', tmp_data, (/0,0,0,offset_nspec_oc(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg2_veloc', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) deallocate(tmp_data) endif if (OUTPUT_INNER_CORE) then ! inner core - allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE),stat=ier) + allocate(tmp_data(NGLOB_INNER_CORE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') do ispec = 1, NSPEC_INNER_CORE @@ -1057,15 +1318,15 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v do i = 1, NGLLX iglob = ibool_inner_core(i,j,k,ispec) ! norm of velocity - tmp_data(i,j,k,ispec) = real(scale_veloc,kind=CUSTOM_REAL) * sqrt( veloc_inner_core(1,iglob)**2 & - + veloc_inner_core(2,iglob)**2 & - + veloc_inner_core(3,iglob)**2 ) + tmp_data(iglob) = real(scale_veloc,kind=CUSTOM_REAL) * sqrt( veloc_inner_core(1,iglob)**2 & + + veloc_inner_core(2,iglob)**2 & + + veloc_inner_core(3,iglob)**2 ) enddo enddo enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg3_veloc', tmp_data, (/0,0,0,offset_nspec_ic(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg3_veloc', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) deallocate(tmp_data) endif @@ -1091,7 +1352,7 @@ subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core ! outputs norm of acceleration: MOVIE_VOLUME_TYPE == 1 use constants_solver - use shared_parameters, only: OUTPUT_FILES + use shared_parameters, only: OUTPUT_FILES,H5_COL use specfem_par, only: it, scale_t_inv,scale_veloc #ifdef USE_HDF5 @@ -1111,28 +1372,9 @@ subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core #ifdef USE_HDF5 ! local parameters integer :: ispec,iglob,i,j,k,ier - real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: tmp_data + real(kind=CUSTOM_REAL), dimension(:), allocatable :: tmp_data real(kind=CUSTOM_REAL) :: scale_accel - logical,parameter :: OUTPUT_CRUST_MANTLE = .true. - logical,parameter :: OUTPUT_OUTER_CORE = .true. - logical,parameter :: OUTPUT_INNER_CORE = .true. - - ! offset arrays - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc - integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic - - integer :: nspec_cm_all_proc,nspec_oc_all_proc,nspec_ic_all_proc - - ! gather number of points - call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_cm, NPROCTOT_VAL) - call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_oc, NPROCTOT_VAL) - call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_ic, NPROCTOT_VAL) - nspec_cm_all_proc = sum(offset_nspec_cm) - nspec_oc_all_proc = sum(offset_nspec_oc) - nspec_ic_all_proc = sum(offset_nspec_ic) - ! dimensionalized scaling scale_accel = real(scale_veloc * scale_t_inv,kind=CUSTOM_REAL) @@ -1151,13 +1393,13 @@ subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core call h5_open_or_create_group(group_name) if (OUTPUT_CRUST_MANTLE) then - call h5_create_dataset_gen_in_group('reg1_accel', (/NGLLX,NGLLY,NGLLZ,nspec_cm_all_proc/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg1_accel', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) endif if (OUTPUT_OUTER_CORE) then - call h5_create_dataset_gen_in_group('reg2_accel', (/NGLLX,NGLLY,NGLLZ,nspec_oc_all_proc/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg2_accel', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) endif if (OUTPUT_INNER_CORE) then - call h5_create_dataset_gen_in_group('reg3_accel', (/NGLLX,NGLLY,NGLLZ,nspec_ic_all_proc/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg3_accel', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) endif call h5_close_group() @@ -1174,7 +1416,7 @@ subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core if (OUTPUT_CRUST_MANTLE) then ! acceleration ! these binary arrays can be converted into mesh format using the utility ./bin/xcombine_vol_data - allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE),stat=ier) + allocate(tmp_data(NGLOB_CRUST_MANTLE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') do ispec = 1, NSPEC_CRUST_MANTLE @@ -1183,22 +1425,22 @@ subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core do i = 1, NGLLX iglob = ibool_crust_mantle(i,j,k,ispec) ! norm - tmp_data(i,j,k,ispec) = scale_accel * sqrt( accel_crust_mantle(1,iglob)**2 & - + accel_crust_mantle(2,iglob)**2 & - + accel_crust_mantle(3,iglob)**2 ) + tmp_data(iglob) = scale_accel * sqrt( accel_crust_mantle(1,iglob)**2 & + + accel_crust_mantle(2,iglob)**2 & + + accel_crust_mantle(3,iglob)**2 ) enddo enddo enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg1_accel', tmp_data, (/0,0,0,offset_nspec_cm(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg1_accel', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) deallocate(tmp_data) endif if (OUTPUT_OUTER_CORE) then ! outer core acceleration - allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE),stat=ier) + allocate(tmp_data(NGLOB_OUTER_CORE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') do ispec = 1, NSPEC_OUTER_CORE @@ -1209,20 +1451,20 @@ subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core ! norm ! note: this outputs only the second time derivative of the potential, ! not the actual acceleration or pressure p = - rho * Chi_dot_dot - tmp_data(i,j,k,ispec) = abs(accel_outer_core(iglob)) + tmp_data(iglob) = abs(accel_outer_core(iglob)) enddo enddo enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg2_accel', tmp_data, (/0,0,0,offset_nspec_oc(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg2_accel', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) deallocate(tmp_data) endif if (OUTPUT_INNER_CORE) then ! inner core - allocate(tmp_data(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE),stat=ier) + allocate(tmp_data(NGLOB_INNER_CORE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') do ispec = 1, NSPEC_INNER_CORE @@ -1231,15 +1473,15 @@ subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core do i = 1, NGLLX iglob = ibool_inner_core(i,j,k,ispec) ! norm of acceleration - tmp_data(i,j,k,ispec) = scale_accel * sqrt( accel_inner_core(1,iglob)**2 & - + accel_inner_core(2,iglob)**2 & - + accel_inner_core(3,iglob)**2 ) + tmp_data(iglob) = scale_accel * sqrt( accel_inner_core(1,iglob)**2 & + + accel_inner_core(2,iglob)**2 & + + accel_inner_core(3,iglob)**2 ) enddo enddo enddo enddo - call h5_write_dataset_collect_hyperslab_in_group('reg3_accel', tmp_data, (/0,0,0,offset_nspec_ic(0:myrank-1)/), if_col) + call write_array3dspec_as_1d_hdf5('reg3_accel', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) deallocate(tmp_data) endif @@ -1259,22 +1501,23 @@ end subroutine write_movie_volume_accelnorm_hdf5 ! #ifdef USE_HDF5 - subroutine get_conn_for_movie(elm_conn,offset,iNIT,npoints_3dmovie,num_ibool_3dmovie,mask_ibool_3dmovie) + subroutine get_conn_for_movie(elm_conn,offset,iNIT,nelems_3dmovie,npoints_3dmovie, nelems_in_this_region, & + num_ibool_3dmovie,mask_ibool_3dmovie,ibool_of_the_section) use specfem_par - use specfem_par_crustmantle, only: ibool_crust_mantle + !use specfem_par_crustmantle, only: ibool_crust_mantle use constants_solver implicit none - integer, dimension(9,npoints_3dmovie), intent(out) :: elm_conn + integer, dimension(9,nelems_3dmovie), intent(out) :: elm_conn integer, intent(in) :: offset ! node id offset (starting global element id of each proc) integer, intent(in) :: iNIT - integer, intent(in) :: npoints_3dmovie - integer, dimension(NGLOB_CRUST_MANTLE_3DMOVIE),intent(in) :: num_ibool_3dmovie - logical, dimension(NGLOB_CRUST_MANTLE_3DMOVIE),intent(in) :: mask_ibool_3dmovie - + integer, intent(in) :: nelems_3dmovie, npoints_3dmovie, nelems_in_this_region + integer, dimension(npoints_3dmovie),intent(in) :: num_ibool_3dmovie + logical, dimension(npoints_3dmovie),intent(in) :: mask_ibool_3dmovie + integer, dimension(NGLLX,NGLLY,NGLLZ,nelems_in_this_region),intent(in) :: ibool_of_the_section ! local parameters integer :: ispec,ii @@ -1285,29 +1528,29 @@ subroutine get_conn_for_movie(elm_conn,offset,iNIT,npoints_3dmovie,num_ibool_3dm integer :: ispecele, iglob_center ispecele = 0 - do ispec = 1,NSPEC_CRUST_MANTLE + do ispec = 1, nelems_in_this_region ! checks center of element for movie flag - iglob_center = ibool_crust_mantle((NGLLX+1)/2,(NGLLY+1)/2,(NGLLZ+1)/2,ispec) + iglob_center = ibool_of_the_section((NGLLX+1)/2,(NGLLY+1)/2,(NGLLZ+1)/2,ispec) ! checks if movie element if (mask_ibool_3dmovie(iglob_center)) then - ! this element is in the movie region - ispecele = ispecele+1 - do k = 1,NGLLZ-1,iNIT do j = 1,NGLLY-1,iNIT do i = 1,NGLLX-1,iNIT + ! this element is in the movie region + ispecele = ispecele+1 + ! defines corners of a vtk element - iglob1 = ibool_crust_mantle(i,j,k,ispec) - iglob2 = ibool_crust_mantle(i+iNIT,j,k,ispec) - iglob3 = ibool_crust_mantle(i+iNIT,j+iNIT,k,ispec) - iglob4 = ibool_crust_mantle(i,j+iNIT,k,ispec) - iglob5 = ibool_crust_mantle(i,j,k+iNIT,ispec) - iglob6 = ibool_crust_mantle(i+iNIT,j,k+iNIT,ispec) - iglob7 = ibool_crust_mantle(i+iNIT,j+iNIT,k+iNIT,ispec) - iglob8 = ibool_crust_mantle(i,j+iNIT,k+iNIT,ispec) + iglob1 = ibool_of_the_section(i,j,k,ispec) + iglob2 = ibool_of_the_section(i+iNIT,j,k,ispec) + iglob3 = ibool_of_the_section(i+iNIT,j+iNIT,k,ispec) + iglob4 = ibool_of_the_section(i,j+iNIT,k,ispec) + iglob5 = ibool_of_the_section(i,j,k+iNIT,ispec) + iglob6 = ibool_of_the_section(i+iNIT,j,k+iNIT,ispec) + iglob7 = ibool_of_the_section(i+iNIT,j+iNIT,k+iNIT,ispec) + iglob8 = ibool_of_the_section(i,j+iNIT,k+iNIT,ispec) ! vtk indexing starts at 0 -> adds minus 1 n1 = num_ibool_3dmovie(iglob1)-1 @@ -1342,207 +1585,476 @@ subroutine get_conn_for_movie(elm_conn,offset,iNIT,npoints_3dmovie,num_ibool_3dm enddo !ispec + ! check if ispecele is consistent with nelems_3dmovie + if (ispecele /= nelems_3dmovie) then + print *,'Error: number of movie elements is not consistent with nelems_3dmovie' + print *,'ispecele = ',ispecele,'nelems_3dmovie = ',nelems_3dmovie + call exit_mpi(myrank,'Error number of movie elements is not consistent with nelems_3dmovie') + endif + end subroutine get_conn_for_movie #endif +! +!------------------------------------------------------------------------------------------------- +! +#ifdef USE_HDF5 + + subroutine elm2node_base(array_3dspec, array_1dmovie, nelms, npoints, ibool_of_the_section) + + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nelms), intent(in) :: array_3dspec + real(kind=CUSTOM_REAL), dimension(npoints), intent(inout) :: array_1dmovie + integer, intent(in) :: nelms, npoints + integer, dimension(NGLLX,NGLLY,NGLLZ,nelms), intent(in) :: ibool_of_the_section + + ! local parameters + integer :: i,j,k,ispec,iglob + + ! convert 3d array to 1d array + do ispec = 1, nelms + + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_of_the_section(i,j,k,ispec) + array_1dmovie(iglob) = array_3dspec(i,j,k,ispec) + enddo + enddo + enddo + enddo + + end subroutine elm2node_base + +#endif + +#ifdef USE_HDF5 + + subroutine write_array3dspec_as_1d_hdf5(dset_name, nelms, npoints, array_3dspec, offset1d, ibool_of_the_section) + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + character(len=*), intent(in) :: dset_name + integer, intent(in) :: nelms, npoints + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nelms), intent(in) :: array_3dspec + integer, intent(in) :: offset1d + integer, dimension(NGLLX,NGLLY,NGLLZ,nelms), intent(in) :: ibool_of_the_section + + ! local parameters + integer :: i,j,k,ii + real(kind=CUSTOM_REAL), dimension(npoints) :: array_1dmovie + + ! convert 3d array to 1d array + call elm2node_base(array_3dspec, array_1dmovie, nelms, npoints, ibool_of_the_section) + + ! write 1d array to hdf5 + call h5_write_dataset_collect_hyperslab_in_group(dset_name, array_1dmovie, (/offset1d/), H5_COL) + + end subroutine write_array3dspec_as_1d_hdf5 + +#endif ! !------------------------------------------------------------------------------------------------- ! #ifdef USE_HDF5 - subroutine write_xdmf_vol_hdf5(pressure_io, divglob_io, div_io, veloc_io, curl_io, stress_io) + subroutine write_xdmf_vol_hdf5_one_data(fname_h5, attr_name, dset_name, len_data, target_unit, it_str, value_on_node) + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + character(len=*), intent(in) :: fname_h5 + character(len=*), intent(in) :: attr_name + character(len=*), intent(in) :: dset_name + integer, intent(in) :: len_data + integer, intent(in) :: target_unit + character(len=*), intent(in) :: it_str + logical, intent(in) :: value_on_node ! values are defined on nodes or elements + + character(len=20) :: center_str + + if (value_on_node) then + center_str = 'Node' ! len data should be the number of nodes + else + center_str = 'Cell' ! len data should be the number of elements + endif + + ! write a header for single data + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) ' '//trim(fname_h5)//':/it_'//trim(it_str)//'/'//trim(dset_name) + write(target_unit,*) '' + write(target_unit,*) '' + + end subroutine write_xdmf_vol_hdf5_one_data + + + subroutine write_xdmf_vol_hdf5_header(nelems, nglobs, fname_h5_data_vol_xdmf, target_unit, region_flag) + + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + integer, intent(in) :: nelems, nglobs + character(len=*), intent(in) :: fname_h5_data_vol_xdmf + integer, intent(in) :: target_unit + integer, intent(in) :: region_flag ! 1: crust mantle, 2: outer core, 3: inner core + + character(len=64) :: nelm_str, nglo_str, elemconn_str, x_str, y_str, z_str + + if (region_flag == 1) then + elemconn_str = 'elm_conn' + x_str = 'x' + y_str = 'y' + z_str = 'z' + else if (region_flag == 2) then + elemconn_str = 'elm_conn_cm' + x_str = 'x_cm' + y_str = 'y_cm' + z_str = 'z_cm' + else if (region_flag == 3) then + elemconn_str = 'elm_conn_oc' + x_str = 'x_oc' + y_str = 'y_oc' + z_str = 'z_oc' + else if (region_flag == 4) then + elemconn_str = 'elm_conn_ic' + x_str = 'x_ic' + y_str = 'y_ic' + z_str = 'z_ic' + else + print *,'Error: invalid region_flag in write_xdmf_vol_hdf5_header' + call exit_mpi(myrank,'Error invalid region_flag in write_xdmf_vol_hdf5_header') + endif + + ! convert integer to string + nelm_str = i2c(nelems) + nglo_str = i2c(nglobs) + + ! definition of topology and geometry + ! refer only control nodes (8 or 27) as a coarse output + ! data array need to be extracted from full data array on GLL points + write(target_unit,'(a)') '' + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) '' + + ! loop for writing information of mesh partitions + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/'//trim(elemconn_str) + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/'//trim(x_str) + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/'//trim(y_str) + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/'//trim(z_str) + write(target_unit,*) '' + write(target_unit,*) '' + + write(target_unit,*) '' + + end subroutine write_xdmf_vol_hdf5_header + + subroutine write_xdmf_vol_hdf5_footer(target_unit) + + implicit none + + integer, intent(in) :: target_unit + + ! file finish + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) '' + + end subroutine write_xdmf_vol_hdf5_footer + + + subroutine write_xdmf_vol_hdf5(npoints_3dmovie, nelems_3dmovie, & + npoints_3dmovie_cm, nelems_3dmovie_cm, & + npoints_3dmovie_oc, nelems_3dmovie_oc, & + npoints_3dmovie_ic, nelems_3dmovie_ic) use specfem_par use specfem_par_movie_hdf5 implicit none - logical, intent(in) :: pressure_io, divglob_io, div_io, veloc_io, curl_io, stress_io + integer, intent(in) :: npoints_3dmovie, nelems_3dmovie + integer, intent(in) :: npoints_3dmovie_cm, nelems_3dmovie_cm + integer, intent(in) :: npoints_3dmovie_oc, nelems_3dmovie_oc + integer, intent(in) :: npoints_3dmovie_ic, nelems_3dmovie_ic ! local parameters - integer :: i, ii - character(len=20) :: it_str,nelm_str,nglo_str - character(len=MAX_STRING_LEN) :: fname_xdmf_vol + integer :: i, ii + character(len=20) :: it_str, movie_prefix + character(len=20) :: nelm_str, nglo_str + character(len=20) :: nelm_str_cm, nglo_str_cm + character(len=20) :: nelm_str_oc, nglo_str_oc + character(len=20) :: nelm_str_ic, nglo_str_ic + character(len=MAX_STRING_LEN) :: fname_xdmf_vol, fname_xdmf_vol_oc, fname_xdmf_vol_ic character(len=MAX_STRING_LEN) :: fname_h5_data_vol_xdmf ! checks if anything do, only main process writes out xdmf file if (myrank /= 0) return - ! writeout xdmf file for volume movie - fname_xdmf_vol = trim(OUTPUT_FILES) // "/movie_volume.xmf" - fname_h5_data_vol_xdmf = "./movie_volume.h5" ! relative to movie_volume.xmf file - ! this seems to point to a wrong director: - ! fname_h5_data_vol_xdmf = trim(OUTPUT_FILES) // "/movie_volume.h5" - - open(unit=xdmf_vol, file=trim(fname_xdmf_vol), recl=256) - - ! definition of topology and geometry - ! refer only control nodes (8 or 27) as a coarse output - ! data array need to be extracted from full data array on GLL points - write(xdmf_vol,'(a)') '' - write(xdmf_vol,*) '' - write(xdmf_vol,*) '' - write(xdmf_vol,*) '' - - ! loop for writing information of mesh partitions -! nelm_str = i2c(sum(nelm_par_proc_nio(:))*(NGLLX-1)*(NGLLY-1)*(NGLLZ-1)) -! nglo_str = i2c(sum(nglob_par_proc_nio(:))) -! -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/elm_conn' -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/x' -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/y' -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/z' -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) '' -! -! do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) -! -! ii = i*NTSTEP_BETWEEN_FRAMES -! !write(it_str, "(i6.6)") ii -! it_str = i2c(ii) -! -! write(xdmf_vol,*) '' -! write(xdmf_vol,*) '' -! enddo + ! + ! write out the crust mantle xdmf file (for strain and vector) + ! + if (output_sv) then + + fname_xdmf_vol = trim(OUTPUT_FILES) // "/movie_volume.xmf" + fname_h5_data_vol_xdmf = "./movie_volume.h5" ! relative to movie_volume.xmf file + + ! open xdmf file + open(unit=xdmf_vol, file=trim(fname_xdmf_vol), recl=256) + + call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc, npoints_vol_mov_all_proc, fname_h5_data_vol_xdmf, xdmf_vol, 1) + + + do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) + + ii = i*NTSTEP_BETWEEN_FRAMES + !write(it_str, "(i6.6)") ii + it_str = i2c(ii) + + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + + enddo + + call write_xdmf_vol_hdf5_footer(xdmf_vol) + + ! close xdmf file + close(xdmf_vol) + + endif ! output_sv + + ! + ! write out the crust and mantle xdmf file (for strain and vector) + ! + if (output_cm) then + + fname_xdmf_vol = trim(OUTPUT_FILES) // "/movie_volume_cm.xmf" + fname_h5_data_vol_xdmf = "./movie_volume.h5" ! relative to movie_volume_cm.xmf file + + ! open xdmf file + open(unit=xdmf_vol, file=trim(fname_xdmf_vol), recl=256) + + call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_cm_conn, npoints_vol_mov_all_proc_cm, fname_h5_data_vol_xdmf, xdmf_vol, 2) + + do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) + + ii = i*NTSTEP_BETWEEN_FRAMES + !write(it_str, "(i6.6)") ii + it_str = i2c(ii) + + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + + enddo + + call write_xdmf_vol_hdf5_footer(xdmf_vol) + + ! close xdmf file + close(xdmf_vol) + + endif ! output_cm + + ! + ! write out the outer core xdmf file + ! + if (output_oc) then + + fname_xdmf_vol_oc = trim(OUTPUT_FILES) // "/movie_volume_oc.xmf" + fname_h5_data_vol_xdmf = "./movie_volume.h5" ! relative to movie_volume_oc.xmf file + + ! open xdmf file + open(unit=xdmf_vol, file=trim(fname_xdmf_vol_oc), recl=256) + + call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_oc_conn, npoints_vol_mov_all_proc_oc, fname_h5_data_vol_xdmf, xdmf_vol, 3) + + do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) + + ii = i*NTSTEP_BETWEEN_FRAMES + !write(it_str, "(i6.6)") ii + it_str = i2c(ii) + + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + + enddo + + call write_xdmf_vol_hdf5_footer(xdmf_vol) + + ! close xdmf file + close(xdmf_vol) + + endif ! output_oc + + ! + ! write out the inner core xdmf file + ! + if (output_ic) then + + fname_xdmf_vol_ic = trim(OUTPUT_FILES) // "/movie_volume_ic.xmf" + fname_h5_data_vol_xdmf = "./movie_volume.h5" ! relative to movie_volume_ic.xmf file + + ! open xdmf file + open(unit=xdmf_vol, file=trim(fname_xdmf_vol_ic), recl=256) + + call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_ic_conn, npoints_vol_mov_all_proc_ic, fname_h5_data_vol_xdmf, xdmf_vol, 4) + + do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) + + ii = i*NTSTEP_BETWEEN_FRAMES + !write(it_str, "(i6.6)") ii + it_str = i2c(ii) + + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + + enddo + + call write_xdmf_vol_hdf5_footer(xdmf_vol) - ! file finish - write(xdmf_vol,*) '' - write(xdmf_vol,*) '' - write(xdmf_vol,*) '' + ! close xdmf file + close(xdmf_vol) - close(xdmf_vol) + endif ! output_ic end subroutine write_xdmf_vol_hdf5 From c57989961b08e9a5cfe892ef740861cfbb02748a Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Thu, 31 Oct 2024 07:53:13 -0600 Subject: [PATCH 14/19] add seismo output with hdf5 --- DATA/Par_file | 1 + .../DATA/Par_file | 1 + .../SEMD/DATA/Par_file | 1 + .../SEMD/DATA/Par_file | 1 + .../SEMD/DATA/Par_file | 1 + .../DATA/Par_file | 1 + .../amplitude/DATA/Par_file | 1 + .../traveltime/DATA/Par_file | 1 + .../global_s362ani_shakemovie/DATA/Par_file | 1 + EXAMPLES/global_small/DATA/Par_file | 1 + EXAMPLES/mars_global/DATA/Par_file | 1 + EXAMPLES/mars_regional/DATA/Par_file | 1 + EXAMPLES/moon_global/DATA/Par_file | 1 + .../DATA/Par_file_NOISE_1_attenuation | 1 + .../DATA/Par_file_NOISE_1_noattenuation | 1 + .../DATA/Par_file_NOISE_2_attenuation | 1 + .../DATA/Par_file_NOISE_2_noattenuation | 1 + .../DATA/Par_file_NOISE_3_noattenuation | 1 + .../DATA/Par_file_NOISE_1_attenuation | 1 + .../DATA/Par_file_NOISE_1_noattenuation | 1 + .../DATA/Par_file_NOISE_2_attenuation | 1 + .../DATA/Par_file_NOISE_2_noattenuation | 1 + .../DATA/Par_file_NOISE_3_noattenuation | 1 + .../DATA/Par_file_NOISE_1_attenuation | 1 + .../DATA/Par_file_NOISE_1_noattenuation | 1 + .../DATA/Par_file_NOISE_2_attenuation | 1 + .../DATA/Par_file_NOISE_2_noattenuation | 1 + .../DATA/Par_file_NOISE_3_noattenuation | 1 + .../DATA/Par_file_NOISE_1_attenuation | 1 + .../DATA/Par_file_NOISE_1_noattenuation | 1 + .../DATA/Par_file_NOISE_2_attenuation | 1 + .../DATA/Par_file_NOISE_2_noattenuation | 1 + .../DATA/Par_file_NOISE_3_noattenuation | 1 + .../DATA/Par_file_NOISE_1_attenuation | 1 + .../DATA/Par_file_NOISE_1_noattenuation | 1 + .../DATA/Par_file_NOISE_2_attenuation | 1 + .../DATA/Par_file_NOISE_2_noattenuation | 1 + .../DATA/Par_file_NOISE_3_noattenuation | 1 + EXAMPLES/point_force/DATA/Par_file | 1 + EXAMPLES/regional_Berkeley/DATA/Par_file | 18 +- EXAMPLES/regional_EMC_model/DATA/Par_file | 1 + .../regional_EMC_model/DATA/Par_file.SCEC-CVM | 1 + .../regional_Greece_noise_small/DATA/Par_file | 1 + EXAMPLES/regional_Greece_small/DATA/Par_file | 1 + .../regional_Greece_small_LDDRK/DATA/Par_file | 1 + .../DATA/Par_file | 1 + .../DATA/Par_file_step1 | 1 + .../DATA/Par_file_step2 | 1 + EXAMPLES/regional_MiddleEast/DATA/Par_file | 1 + EXAMPLES/regional_s40rts/DATA/Par_file | 5 +- EXAMPLES/regional_sgloberani/DATA/Par_file | 1 + .../regional_simultaneous_runs/DATA/Par_file | 1 + EXAMPLES/regional_small_adjoint/DATA/Par_file | 7 +- EXAMPLES/regular_kernel/DATA/Par_file | 1 + .../DATA/Par_file | 1 + .../DATA/Par_file | 1 + src/shared/broadcast_computed_parameters.f90 | 5 +- src/shared/read_parameter_file.F90 | 4 +- src/shared/shared_par.f90 | 6 +- src/specfem3D/rules.mk | 1 + src/specfem3D/setup_sources_receivers.f90 | 2 + src/specfem3D/specfem3D_par.F90 | 3 + src/specfem3D/write_output_HDF5.F90 | 257 ++++++++++++++++++ src/specfem3D/write_seismograms.f90 | 50 +++- 64 files changed, 380 insertions(+), 31 deletions(-) create mode 100644 src/specfem3D/write_output_HDF5.F90 diff --git a/DATA/Par_file b/DATA/Par_file index 3208f64d2..56f205e5a 100644 --- a/DATA/Par_file +++ b/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file b/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file index 19bca021b..050be325a 100644 --- a/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file +++ b/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .true. diff --git a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file index 634cf6419..ee29c66cb 100644 --- a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file +++ b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file index 3b6dfc476..791019fc3 100644 --- a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file +++ b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file b/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file index d7a9f40ad..e3c98e1ae 100644 --- a/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file +++ b/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file b/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file index 914e4ab83..2ce1ca46f 100644 --- a/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file +++ b/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file b/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file index 4738537b8..1c9a7a560 100644 --- a/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file +++ b/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file b/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file index 4738537b8..1c9a7a560 100644 --- a/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file +++ b/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file b/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file index 2f231de7e..b3f9d5f49 100644 --- a/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file +++ b/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .true. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/global_small/DATA/Par_file b/EXAMPLES/global_small/DATA/Par_file index ee62ad37d..919273371 100644 --- a/EXAMPLES/global_small/DATA/Par_file +++ b/EXAMPLES/global_small/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .true. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/mars_global/DATA/Par_file b/EXAMPLES/mars_global/DATA/Par_file index 9c552bf75..b9a83e0f0 100644 --- a/EXAMPLES/mars_global/DATA/Par_file +++ b/EXAMPLES/mars_global/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/mars_regional/DATA/Par_file b/EXAMPLES/mars_regional/DATA/Par_file index a5a82cde4..c1eb814e7 100644 --- a/EXAMPLES/mars_regional/DATA/Par_file +++ b/EXAMPLES/mars_regional/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/moon_global/DATA/Par_file b/EXAMPLES/moon_global/DATA/Par_file index 33f8df16a..a7c4659b1 100644 --- a/EXAMPLES/moon_global/DATA/Par_file +++ b/EXAMPLES/moon_global/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation index 74e61d567..97fc77b6f 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation index 3a2978a72..760d1cc60 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation index 732914b35..cea8dad9c 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation index e04fe69a9..220b0f3b0 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation index 7496e4e49..6c6c874c9 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation index 103b08a64..d9676321b 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation index f19fd40cc..c99291b79 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation index 6e6513ddb..3c05d20a3 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation index 376c5bb99..f2068990f 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation index ef4c3eb9a..06905755a 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation index a30d5c936..ad51e4ce6 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation index dda5fddaf..34b396ebc 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation index 7fb9db9fa..5812fe342 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation index f3ae8f8d2..9805ffe4e 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation index b637a89f4..e1e621e2f 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation index f83bb8ee9..5893051ff 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation index 828770f97..dc1227362 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation index ef13ee0d8..80ab3873a 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation index 1ee922174..e8d094d7f 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation index 01d2aa7d2..007879c7a 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation index 614911048..df97e9387 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation index 43360d3ed..7c6622653 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation index 5e0f6e5a8..f512cf381 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation index c1c66e496..c7921649d 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation index fb398be66..4f58bd838 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/point_force/DATA/Par_file b/EXAMPLES/point_force/DATA/Par_file index 22b65e3ae..7e950ccd8 100644 --- a/EXAMPLES/point_force/DATA/Par_file +++ b/EXAMPLES/point_force/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .true. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/regional_Berkeley/DATA/Par_file b/EXAMPLES/regional_Berkeley/DATA/Par_file index 0e8ee669a..7e212475b 100644 --- a/EXAMPLES/regional_Berkeley/DATA/Par_file +++ b/EXAMPLES/regional_Berkeley/DATA/Par_file @@ -21,7 +21,7 @@ GAMMA_ROTATION_AZIMUTH = 0.d0 # number of elements at the surface along the two sides of the first chunk # (must be multiple of 16 and 8 * multiple of NPROC below) -NEX_XI = 48 +NEX_XI = 48 NEX_ETA = 48 # number of MPI processors along the two sides of the first chunk @@ -269,17 +269,6 @@ USE_MONOCHROMATIC_CMT_SOURCE = .false. # print source time function PRINT_SOURCE_TIME_FUNCTION = .true. -## Berkeley source time function -STF_IS_UCB_HEAVISIDE = .true. -# UCB source frequency content (i.e., heaviside function) -SOURCE_T1 = 500.d0 -SOURCE_T2 = 400.d0 -SOURCE_T3 = 50.d0 -SOURCE_T4 = 40.d0 -# UCB source time shift -TAU = 500.d0 - - #----------------------------------------------------------- # # Seismograms @@ -308,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -441,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_EMC_model/DATA/Par_file b/EXAMPLES/regional_EMC_model/DATA/Par_file index da297f043..1db17fb79 100644 --- a/EXAMPLES/regional_EMC_model/DATA/Par_file +++ b/EXAMPLES/regional_EMC_model/DATA/Par_file @@ -317,6 +317,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM b/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM index 885b00977..320ff92a7 100644 --- a/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM +++ b/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM @@ -317,6 +317,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/regional_Greece_noise_small/DATA/Par_file b/EXAMPLES/regional_Greece_noise_small/DATA/Par_file index 816a58a08..09d31fabd 100644 --- a/EXAMPLES/regional_Greece_noise_small/DATA/Par_file +++ b/EXAMPLES/regional_Greece_noise_small/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/regional_Greece_small/DATA/Par_file b/EXAMPLES/regional_Greece_small/DATA/Par_file index 73d25329f..d3039c250 100644 --- a/EXAMPLES/regional_Greece_small/DATA/Par_file +++ b/EXAMPLES/regional_Greece_small/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file b/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file index 154b719fc..26c812d7f 100644 --- a/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file +++ b/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file index b91c982ae..af09d75f7 100644 --- a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file +++ b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 index cec185420..e5bacdcb3 100644 --- a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 +++ b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 index b91c982ae..af09d75f7 100644 --- a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 +++ b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/regional_MiddleEast/DATA/Par_file b/EXAMPLES/regional_MiddleEast/DATA/Par_file index 06ecd9e45..46fcd43b6 100644 --- a/EXAMPLES/regional_MiddleEast/DATA/Par_file +++ b/EXAMPLES/regional_MiddleEast/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/regional_s40rts/DATA/Par_file b/EXAMPLES/regional_s40rts/DATA/Par_file index 7f981472c..b0602efc4 100644 --- a/EXAMPLES/regional_s40rts/DATA/Par_file +++ b/EXAMPLES/regional_s40rts/DATA/Par_file @@ -297,12 +297,13 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. # decide if main process writes all the seismograms or if all processes do it in parallel -WRITE_SEISMOGRAMS_BY_MAIN = .false. +WRITE_SEISMOGRAMS_BY_MAIN = .true. # save all seismograms in one large combined file instead of one file per seismogram # to avoid overloading shared non-local file systems such as LUSTRE or GPFS for instance @@ -432,5 +433,5 @@ ADIOS_FOR_UNDO_ATTENUATION = .true. # HDF5 Database I/O # (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) -HDF5_ENABLED = .false. +HDF5_ENABLED = .true. diff --git a/EXAMPLES/regional_sgloberani/DATA/Par_file b/EXAMPLES/regional_sgloberani/DATA/Par_file index 144f8441f..db2676361 100644 --- a/EXAMPLES/regional_sgloberani/DATA/Par_file +++ b/EXAMPLES/regional_sgloberani/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .true. diff --git a/EXAMPLES/regional_simultaneous_runs/DATA/Par_file b/EXAMPLES/regional_simultaneous_runs/DATA/Par_file index 02186559f..249386088 100644 --- a/EXAMPLES/regional_simultaneous_runs/DATA/Par_file +++ b/EXAMPLES/regional_simultaneous_runs/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/regional_small_adjoint/DATA/Par_file b/EXAMPLES/regional_small_adjoint/DATA/Par_file index 33cab9d6a..0e0ecb4bc 100644 --- a/EXAMPLES/regional_small_adjoint/DATA/Par_file +++ b/EXAMPLES/regional_small_adjoint/DATA/Par_file @@ -204,8 +204,8 @@ RATIO_BY_WHICH_TO_INCREASE_IT = 1.5d0 # save AVS or OpenDX movies #MOVIE_COARSE saves movie only at corners of elements (SURFACE OR VOLUME) #MOVIE_COARSE does not work with create_movie_AVS_DX -MOVIE_SURFACE = .false. -MOVIE_VOLUME = .false. +MOVIE_SURFACE = .true. +MOVIE_VOLUME = .true. MOVIE_COARSE = .false. NTSTEP_BETWEEN_FRAMES = 200 HDUR_MOVIE = 0.1d0 @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -432,5 +433,5 @@ ADIOS_FOR_UNDO_ATTENUATION = .true. # HDF5 Database I/O # (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) -HDF5_ENABLED = .false. +HDF5_ENABLED = .true. diff --git a/EXAMPLES/regular_kernel/DATA/Par_file b/EXAMPLES/regular_kernel/DATA/Par_file index 8d3c9a42f..22cde7c98 100644 --- a/EXAMPLES/regular_kernel/DATA/Par_file +++ b/EXAMPLES/regular_kernel/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .true. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file b/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file index f924108c7..c72472240 100644 --- a/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file +++ b/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file b/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file index 12b90cfa0..e9c28039f 100644 --- a/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file +++ b/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. diff --git a/src/shared/broadcast_computed_parameters.f90 b/src/shared/broadcast_computed_parameters.f90 index f85dd419b..eabfe82e4 100644 --- a/src/shared/broadcast_computed_parameters.f90 +++ b/src/shared/broadcast_computed_parameters.f90 @@ -37,7 +37,7 @@ subroutine broadcast_computed_parameters() integer, parameter :: nparam_i = 51 integer, dimension(nparam_i) :: bcast_integer - integer, parameter :: nparam_l = 76 + integer, parameter :: nparam_l = 77 logical, dimension(nparam_l) :: bcast_logical integer, parameter :: nparam_dp = 42 @@ -113,7 +113,7 @@ subroutine broadcast_computed_parameters() REGIONAL_MESH_CUTOFF,REGIONAL_MESH_ADD_2ND_DOUBLING, & EMC_MODEL, & FULL_GRAVITY, & - HDF5_ENABLED, HDF5_FOR_MOVIES /) + HDF5_ENABLED, HDF5_FOR_MOVIES, OUTPUT_SEISMOS_HDF5 /) bcast_double_precision = (/ & DT, & @@ -365,6 +365,7 @@ subroutine broadcast_computed_parameters() FULL_GRAVITY = bcast_logical(74) HDF5_ENABLED = bcast_logical(75) HDF5_FOR_MOVIES = bcast_logical(76) + OUTPUT_SEISMOS_HDF5 = bcast_logical(77) ! double precisions DT = bcast_double_precision(1) diff --git a/src/shared/read_parameter_file.F90 b/src/shared/read_parameter_file.F90 index 649675c0f..b30892b62 100644 --- a/src/shared/read_parameter_file.F90 +++ b/src/shared/read_parameter_file.F90 @@ -245,8 +245,10 @@ subroutine read_parameter_file() call read_value_logical(OUTPUT_SEISMOS_SAC_BINARY, 'OUTPUT_SEISMOS_SAC_BINARY', ier) if (ier /= 0) stop 'an error occurred while reading the parameter file: OUTPUT_SEISMOS_SAC_BINARY' call read_value_logical(OUTPUT_SEISMOS_ASDF, 'OUTPUT_SEISMOS_ASDF', ier) - if (ier /= 0) stop 'an error occurred while reading the parameter file: OUTPUT_ASDF' + if (ier /= 0) stop 'an error occurred while reading the parameter file: OUTPUT_HDF5' call read_value_logical(OUTPUT_SEISMOS_3D_ARRAY, 'OUTPUT_SEISMOS_3D_ARRAY', ier) + if (ier /= 0) stop 'an error occurred while reading the parameter file: OUTPUT_ASDF' + call read_value_logical(OUTPUT_SEISMOS_HDF5, 'OUTPUT_SEISMOS_HDF5', ier) if (ier /= 0) stop 'an error occurred while reading the parameter file: OUTPUT_3D_ARRAY' call read_value_logical(ROTATE_SEISMOGRAMS_RT, 'ROTATE_SEISMOGRAMS_RT', ier) if (ier /= 0) stop 'an error occurred while reading the parameter file: ROTATE_SEISMOGRAMS_RT' diff --git a/src/shared/shared_par.f90 b/src/shared/shared_par.f90 index 64e9182ad..a5081c66a 100644 --- a/src/shared/shared_par.f90 +++ b/src/shared/shared_par.f90 @@ -65,7 +65,7 @@ module shared_input_parameters logical :: RECEIVERS_CAN_BE_BURIED logical :: OUTPUT_SEISMOS_ASCII_TEXT,OUTPUT_SEISMOS_SAC_ALPHANUM,OUTPUT_SEISMOS_SAC_BINARY, & - OUTPUT_SEISMOS_ASDF,OUTPUT_SEISMOS_3D_ARRAY, & + OUTPUT_SEISMOS_ASDF,OUTPUT_SEISMOS_3D_ARRAY,OUTPUT_SEISMOS_HDF5, & ROTATE_SEISMOGRAMS_RT,WRITE_SEISMOGRAMS_BY_MAIN, & SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE,READ_ADJSRC_ASDF @@ -209,8 +209,8 @@ module shared_input_parameters double precision :: FILESYSTEM_IO_BANDWIDTH = 0.d0 ! HDF5 file i/o - logical :: HDF5_ENABLED = .false. ! for all databases i/o in hdf5 - logical :: HDF5_FOR_MOVIES = .false. ! for movies (shakemap, surface movies, volume movies) + logical :: HDF5_ENABLED = .false. ! for all databases i/o in hdf5 + logical :: HDF5_FOR_MOVIES = .false. ! for movies (shakemap, surface movies, volume movies) ! HDF5 IO server ! number of io dedicated nodes diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk index 87e3dab1b..efc9bdc2d 100644 --- a/src/specfem3D/rules.mk +++ b/src/specfem3D/rules.mk @@ -144,6 +144,7 @@ specfem3D_SOLVER_OBJECTS += \ $O/write_movie_surface_hdf5.solverstatic.o \ $O/write_output_ASCII.solverstatic.o \ $O/write_output_SAC.solverstatic.o \ + $O/write_output_HDF5.solverstatic.o \ $O/write_seismograms.solverstatic.o \ $(EMPTY_MACRO) diff --git a/src/specfem3D/setup_sources_receivers.f90 b/src/specfem3D/setup_sources_receivers.f90 index 9af62a78b..105939728 100644 --- a/src/specfem3D/setup_sources_receivers.f90 +++ b/src/specfem3D/setup_sources_receivers.f90 @@ -1889,6 +1889,8 @@ subroutine setup_receivers_precompute_intp() endif endif + ! TODO: add HDF5 if any + end subroutine setup_receivers_precompute_intp ! diff --git a/src/specfem3D/specfem3D_par.F90 b/src/specfem3D/specfem3D_par.F90 index ae600d68a..762bcb75d 100644 --- a/src/specfem3D/specfem3D_par.F90 +++ b/src/specfem3D/specfem3D_par.F90 @@ -403,6 +403,9 @@ module specfem_par ! process/partition name character(len=MAX_STRING_LEN) :: prname + ! hdf5 file name + character(len=MAX_STRING_LEN) :: hdf5_seismo_fname + !----------------------------------------------------------------- ! MPI partitions !----------------------------------------------------------------- diff --git a/src/specfem3D/write_output_HDF5.F90 b/src/specfem3D/write_output_HDF5.F90 new file mode 100644 index 000000000..7e93a20db --- /dev/null +++ b/src/specfem3D/write_output_HDF5.F90 @@ -0,0 +1,257 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + + subroutine split_string(input_string, delimiter, part1, part2) + implicit none + character(len=*), intent(in) :: input_string + character(len=*), intent(in) :: delimiter + character(len=*), intent(out) :: part1 + character(len=*), intent(out) :: part2 + integer :: delim_pos + + ! Find the position of the delimiter + delim_pos = index(input_string, delimiter) + + ! Split the string at the delimiter + if (delim_pos > 0) then + part1 = input_string(1:delim_pos-1) + part2 = input_string(delim_pos+1:) + else + part1 = input_string + part2 = '' + end if + end subroutine split_string + +#ifdef USE_HDF5 + + subroutine write_hdf5_seismogram_init(nlength_total_seismogram) + + use specfem_par, only: CUSTOM_REAL, MAX_LENGTH_NETWORK_NAME, MAX_LENGTH_STATION_NAME, & + MAX_STRING_LEN, t0, DT, IMAIN, IOUT, myrank, & + NSTEP, nrec, hdf5_seismo_fname, & + OUTPUT_FILES, SIMULATION_TYPE, WRITE_SEISMOGRAMS_BY_MAIN + + use shared_parameters, only: IO_compute_task, NTSTEP_BETWEEN_OUTPUT_SAMPLE + + use manager_hdf5 + + implicit none + + ! local parameters + integer :: i,irec,ier + integer, intent(in) :: nlength_total_seismogram + integer :: nrec_total + real(kind=CUSTOM_REAL), dimension(:), allocatable :: time_array + real(kind=CUSTOM_REAL), dimension(:), allocatable :: rec_dists ! store the distances + character(len=MAX_LENGTH_STATION_NAME), dimension(:), allocatable :: stations + character(len=MAX_LENGTH_NETWORK_NAME), dimension(:), allocatable :: networks + integer, parameter :: n_header_lines = 3 + character(len=MAX_STRING_LEN) :: tmpstr, dummy, line + + ! only main process writes out + if (myrank /= 0) return + + ! create the file and all the datasets for the seismograms + ! set the name of the file + hdf5_seismo_fname = trim(OUTPUT_FILES)//'/seismograms.h5' + + ! user output + if (myrank == 0 .and. IO_compute_task) then + write(IMAIN,*) 'Creating seismograms in HDF5 file format' + if (WRITE_SEISMOGRAMS_BY_MAIN) then + write(IMAIN,*) ' writing waveforms by main...' + else + write(IMAIN,*) ' writing waveforms in parallel...' + endif + write(IMAIN,*) ' seismogram file: ',trim(hdf5_seismo_fname) + call flush_IMAIN() + endif + + ! initialize the HDF5 manager + call h5_initialize() + + ! create the file + call h5_create_file(hdf5_seismo_fname) + + ! create time dataset it = 1 ~ NTSTEP + allocate(time_array(nlength_total_seismogram),stat=ier) + if (ier /= 0) stop 'Error: write_hdf5_seismogram_init: time_array allocation failed' + time_array(:) = 0.0_CUSTOM_REAL + + do i = 1,nlength_total_seismogram + if (SIMULATION_TYPE == 1) then + ! forward simulation + time_array(i) = real( dble((i-1)*NTSTEP_BETWEEN_OUTPUT_SAMPLE) * DT - t0, kind=CUSTOM_REAL) + else + ! adjoint simulation: backward/reconstructed wavefields + time_array(i) = real( dble((NSTEP-i)*NTSTEP_BETWEEN_OUTPUT_SAMPLE) * DT - t0, kind=CUSTOM_REAL) + end if + end do + + ! write the time dataset + call h5_write_dataset_no_group('time',time_array) + ! free the memory + deallocate(time_array) + + ! read output_list_stations.txt generated at locate_receivers.f90:613 here to write in the h5 file. + allocate(stations(nrec), & + networks(nrec), & + rec_dists(nrec),stat=ier) + if (ier /= 0) stop 'Error: write_hdf5_seismogram_init: stations allocation failed' + rec_dists(:) = 0.0_CUSTOM_REAL + + open(unit=IOUT,file=trim(OUTPUT_FILES)//'/output_list_stations.txt', & + status='unknown',action='read',iostat=ier) + if (ier /= 0) then + stop 'Error: write_hdf5_seismogram_init: error opening output_list_stations.txt' + end if + + ! skip the header (3 lines) + do i = 1,n_header_lines + read(IOUT,*) + end do + + ! read the station information + ! each line includes: network.station dummy dummy dist dummy + do irec = 1,nrec + ! read network.station and separate them by '.' + !read(IOUT) tmpstr, dummy, dummy, rec_dists(irec), dummy + !read(IOUT, '(A, 1X, A, 1X, A, 3X, F9.7, 6X, A)') tmpstr, dummy, dummy, rec_dists(irec), dummy + read(IOUT, '(A)', iostat=ier) line + if (ier /= 0) stop 'Error: write_hdf5_seismogram_init: error reading output_list_stations.txt' + read(line, *) tmpstr, dummy, dummy, rec_dists(irec), dummy + call split_string(trim(tmpstr), '.', networks(irec), stations(irec)) + end do + + close(IOUT) + + ! write the station information + call h5_write_dataset_no_group('stations',stations) + call h5_write_dataset_no_group('networks',networks) + call h5_write_dataset_no_group('dists',rec_dists) + + ! free the memory + deallocate(stations,networks,rec_dists) + + ! close the file + call h5_close_file() + + end subroutine write_hdf5_seismogram_init + + +#endif + + subroutine write_output_hdf5(seismogram_tmp_in, irec_local, irec, chn, iorientation) + + use specfem_par, only: & + seismo_current, nrec ,& + nlength_seismogram, & + ROTATE_SEISMOGRAMS_RT, NTSTEP_BETWEEN_OUTPUT_SAMPLE, & + CUSTOM_REAL, hdf5_seismo_fname, & + WRITE_SEISMOGRAMS_BY_MAIN, ROTATE_SEISMOGRAMS_RT, myrank + + use shared_parameters, only: & + NSTEP, OUTPUT_SEISMOS_HDF5 + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + ! input/output variables + character(len=4),intent(in) :: chn + integer,intent(in) :: irec_local, irec, iorientation + real(kind=CUSTOM_REAL),dimension(5,nlength_seismogram),intent(in) :: seismogram_tmp_in + + real(kind=CUSTOM_REAL), dimension(nlength_seismogram,1) :: seismogram_tmp + logical, save :: is_initialized = .false. + integer :: i, ier, nlength_total_seismogram + logical :: if_dataset_exists + +#ifdef USE_HDF5 + + ! check if anything to do + if (.not. OUTPUT_SEISMOS_HDF5) return + + ! safety check + if (.not. WRITE_SEISMOGRAMS_BY_MAIN) & + stop 'Error: WRITE_SEIMSMOGRAMS_BY_MAIN must be true to use HDF5 seismogram output' + + ! total length of the seismogram + nlength_total_seismogram = NSTEP / NTSTEP_BETWEEN_OUTPUT_SAMPLE + print*, 'nlength_total_seismogram = ', nlength_total_seismogram + print*, 'NSTEP = ', NSTEP + print*, 'NTSTEP_BETWEEN_OUTPUT_SAMPLE = ', NTSTEP_BETWEEN_OUTPUT_SAMPLE + print*, 'nrec = ', nrec + print*, 'irec = ', irec + print*, 'seismo_current = ', seismo_current + print*, 'shape(seismogram_tmp) = ', shape(seismogram_tmp) + print*, 'shape(seismogram_tmp(iorientation,1:seismo_current)) = ', shape(seismogram_tmp(iorientation,1:seismo_current)) + ! convert array with shape (seimo_current) to (nlength_total_seismogram, 1) + do i = 1, seismo_current + seismogram_tmp(i,1) = seismogram_tmp_in(iorientation,i) + end do + + + + ! initialize + if (.not. is_initialized) then + call write_hdf5_seismogram_init(nlength_total_seismogram) + is_initialized = .true. + endif + + ! only main process writes out + if (myrank /= 0) return + + ! write the seismograms + call h5_open_file(hdf5_seismo_fname) + + ! check if the target dataset components are already created + call h5_check_dataset_exists(trim(chn),if_dataset_exists) + + ! if the dataset does not exist, create it + if (.not. if_dataset_exists) then + call h5_create_dataset_gen(trim(chn),(/nlength_total_seismogram, nrec/), 2, CUSTOM_REAL) + endif + + ! write the seismogram + call h5_write_dataset_collect_hyperslab(trim(chn), seismogram_tmp(1:seismo_current, :), (/0,irec-1/), .false.) + + ! close the file + call h5_close_file() + +#else + + write(*,*) 'Error: HDF5 support not enabled in this version of Specfem3D_Globe' + write(*,*) 'Please recompile with the --with-hdf5 option' + stop + +#endif + + end subroutine write_output_hdf5 \ No newline at end of file diff --git a/src/specfem3D/write_seismograms.f90 b/src/specfem3D/write_seismograms.f90 index 80855333d..de8618b5d 100644 --- a/src/specfem3D/write_seismograms.f90 +++ b/src/specfem3D/write_seismograms.f90 @@ -39,7 +39,7 @@ subroutine write_seismograms() nlength_seismogram, & NTSTEP_BETWEEN_OUTPUT_SEISMOS,NTSTEP_BETWEEN_OUTPUT_SAMPLE, & do_save_seismograms, & - WRITE_SEISMOGRAMS_BY_MAIN,OUTPUT_SEISMOS_ASDF, & + WRITE_SEISMOGRAMS_BY_MAIN,OUTPUT_SEISMOS_ASDF, OUTPUT_SEISMOS_HDF5, & SAVE_SEISMOGRAMS_STRAIN, & moment_der,sloc_der,shdur_der,stshift_der, & scale_displ @@ -252,6 +252,7 @@ subroutine write_seismograms_to_file(istore) OUTPUT_SEISMOS_SAC_ALPHANUM,OUTPUT_SEISMOS_SAC_BINARY, & OUTPUT_SEISMOS_ASDF, & OUTPUT_SEISMOS_3D_ARRAY, & + OUTPUT_SEISMOS_HDF5, & SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE, & OUTPUT_FILES, & WRITE_SEISMOGRAMS_BY_MAIN, & @@ -321,7 +322,7 @@ subroutine write_seismograms_to_file(istore) ! write this seismogram ! note: ASDF data structure is given in module ! stores all traces into ASDF container in case - call write_one_seismogram(one_seismogram,irec,irec_local,.true.,component,istore) + call write_one_seismogram(one_seismogram,irec,irec_local,1,component,istore) enddo ! writes ASDF file output @@ -390,12 +391,18 @@ subroutine write_seismograms_to_file(istore) endif endif - ! ASCII / SAC format - if (OUTPUT_SEISMOS_ASCII_TEXT .or. OUTPUT_SEISMOS_SAC_ALPHANUM .or. OUTPUT_SEISMOS_SAC_BINARY) then + ! ASCII / SAC / HDF5 format + if ( OUTPUT_SEISMOS_ASCII_TEXT & + .or. OUTPUT_SEISMOS_SAC_ALPHANUM & + .or. OUTPUT_SEISMOS_SAC_BINARY & + .or. OUTPUT_SEISMOS_HDF5) then ! write out seismograms: all processes write their local seismograms themselves if (.not. WRITE_SEISMOGRAMS_BY_MAIN) then + ! HDF5 is not supported + if (OUTPUT_SEISMOS_HDF5) call exit_MPI(myrank,'WRITE_SEISMOGRAMS_BY_MAIN must be true for HDF5 format') + ! all the processes write their local seismograms themselves if (SAVE_ALL_SEISMOS_IN_ONE_FILE .and. OUTPUT_SEISMOS_ASCII_TEXT) then write(sisname,'(A,I5.5)') '/all_seismograms_'//trim(component)//'_node_',myrank @@ -428,7 +435,7 @@ subroutine write_seismograms_to_file(istore) ! write this seismogram ! note: ASDF data structure is given in module ! stores all traces into ASDF container in case - call write_one_seismogram(one_seismogram,irec,irec_local,.false.,component,istore) + call write_one_seismogram(one_seismogram,irec,irec_local,0,component,istore) enddo ! create one large file instead of one small file per station to avoid file system overload @@ -506,7 +513,13 @@ subroutine write_seismograms_to_file(istore) endif ! write this seismogram - call write_one_seismogram(one_seismogram,irec,irec_local,.false.,component,istore) + if (.not. OUTPUT_SEISMOS_HDF5) then + ! ASCII or SAC format + call write_one_seismogram(one_seismogram,irec,irec_local,0,component,istore) + else + ! HDF5 format + call write_one_seismogram(one_seismogram,irec,irec_local,2,component,istore) + endif ! counts seismos written total_seismos = total_seismos + 1 @@ -632,7 +645,7 @@ end subroutine write_seismograms_to_file !------------------------------------------------------------------------------------------------- ! - subroutine write_one_seismogram(one_seismogram,irec,irec_local,is_for_asdf,component,istore) + subroutine write_one_seismogram(one_seismogram,irec,irec_local,ftype_flag,component,istore) use constants_solver, only: MAX_STRING_LEN,CUSTOM_REAL,NDIM,DEGREES_TO_RADIANS, & MAX_LENGTH_STATION_NAME,MAX_LENGTH_NETWORK_NAME @@ -652,7 +665,8 @@ subroutine write_one_seismogram(one_seismogram,irec,irec_local,is_for_asdf,compo implicit none integer,intent(in) :: irec,irec_local - logical,intent(in) :: is_for_asdf + integer,intent(in) :: ftype_flag + logical :: is_for_asdf,is_for_hdf5 real(kind=CUSTOM_REAL), dimension(NDIM,nlength_seismogram),intent(in) :: one_seismogram integer,intent(in) :: istore @@ -677,6 +691,23 @@ subroutine write_one_seismogram(one_seismogram,irec,irec_local,is_for_asdf,compo ! initializes seismogram_tmp(:,:) = 0.0_CUSTOM_REAL + ! check file type + if (ftype_flag == 0) then + ! ascii + is_for_asdf = .false. + is_for_hdf5 = .false. + else if (ftype_flag == 1) then + ! asdf + is_for_asdf = .true. + is_for_hdf5 = .false. + else if (ftype_flag == 2) then + ! hdf5 + is_for_asdf = .false. + is_for_hdf5 = .true. + else + call exit_MPI(myrank,'wrong file type flag') + endif + ! get band code call band_instrument_code(DT,bic) @@ -804,6 +835,9 @@ subroutine write_one_seismogram(one_seismogram,irec,irec_local,is_for_asdf,compo if (OUTPUT_SEISMOS_ASDF) then call store_asdf_data(seismogram_tmp,irec_local,irec,chn,iorientation) endif + else if (is_for_hdf5) then + ! TODO: add HDF5 format + call write_output_hdf5(seismogram_tmp,irec_local,irec,chn,iorientation) else ! SAC output format if (OUTPUT_SEISMOS_SAC_ALPHANUM .or. OUTPUT_SEISMOS_SAC_BINARY ) then From 73626e1686ca63b822fd04110fca9dff6af2f25b Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Tue, 5 Nov 2024 10:10:40 -0700 Subject: [PATCH 15/19] erase comment outted lines --- DATA/Par_file | 2 +- src/meshfem3D/rules.mk | 7 ------- src/specfem3D/rules.mk | 11 +---------- 3 files changed, 2 insertions(+), 18 deletions(-) diff --git a/DATA/Par_file b/DATA/Par_file index 56f205e5a..13766788a 100644 --- a/DATA/Par_file +++ b/DATA/Par_file @@ -227,7 +227,7 @@ MOVIE_START = 0 MOVIE_STOP = 40000 # save mesh files to check the mesh -SAVE_MESH_FILES = .true. +SAVE_MESH_FILES = .false. # restart files (number of runs can be 1 or higher, choose 1 for no restart files) NUMBER_OF_RUNS = 1 diff --git a/src/meshfem3D/rules.mk b/src/meshfem3D/rules.mk index 594b60260..a95a7f193 100644 --- a/src/meshfem3D/rules.mk +++ b/src/meshfem3D/rules.mk @@ -409,10 +409,3 @@ $O/%.check_adios.o: $S/%.F90 $O/shared_par.shared_module.o $O/meshfem3D_par.chec $O/%.checknetcdf.o: $S/%.f90 $O/shared_par.shared_module.o $O/meshfem3D_par.check_module.o ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(NETCDF_INCLUDE) -c -o $@ $< -## HDF5 file i/o - -#$O/%.mesh_hdf5.o: $S/%.F90 $O/shared_par.shared_module.o $O/meshfem3D_par.check_module.o $O/hdf5_manager.shared_hdf5_module.o -# ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< -# -#$O/%.mesh_hdf5.o: $S/%.f90 $O/shared_par.shared_module.o $O/meshfem3D_par.check_module.o $O/hdf5_manager.shared_hdf5_module.o -# ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk index efc9bdc2d..40461bd97 100644 --- a/src/specfem3D/rules.mk +++ b/src/specfem3D/rules.mk @@ -511,13 +511,4 @@ $O/%.visualc.o: $S/%.c ${SETUP}/config.h ### CEM ### $O/%.checknetcdf.o: $S/%.f90 $O/shared_par.shared_module.o $O/specfem3D_par.solverstatic_module.o - ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(NETCDF_INCLUDE) -c -o $@ $< - -### -### HDF5 -### -#$O/%.spec_hdf5_module.o: $S/%.F90 $O/shared_par.shared_module.o $O/specfem3D_par.solverstatic_module.o $O/hdf5_manager.shared_hdf5_module.o -# ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(HDF5_INCLUDE) -c -o $@ $< -# -#$O/%.spec_hdf5_module.o: $S/%.f90 $O/shared_par.shared_module.o $O/specfem3D_par.solverstatic_module.o $O/hdf5_manager.shared_hdf5_module.o -# ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(HDF5_INCLUDE) -c -o $@ $< \ No newline at end of file + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(NETCDF_INCLUDE) -c -o $@ $< \ No newline at end of file From 09c2acae086a79d6c3a52af51fc5ee012c5a1897 Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Tue, 5 Nov 2024 10:22:50 -0700 Subject: [PATCH 16/19] lin truncated error --- src/specfem3D/write_movie_output.f90 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/specfem3D/write_movie_output.f90 b/src/specfem3D/write_movie_output.f90 index bd931141d..e712dca9a 100644 --- a/src/specfem3D/write_movie_output.f90 +++ b/src/specfem3D/write_movie_output.f90 @@ -161,16 +161,16 @@ subroutine write_movie_output() ! output the Time Integral of Strain, or \mu*TIS if (HDF5_ENABLED) then call write_movie_volume_strains_hdf5(NSPEC_CRUST_MANTLE_3DMOVIE, & - Ieps_trace_over_3_crust_mantle, & - NSPEC_CRUST_MANTLE_3DMOVIE, & - Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle,Iepsilondev_xy_crust_mantle, & - Iepsilondev_xz_crust_mantle,Iepsilondev_yz_crust_mantle) + Ieps_trace_over_3_crust_mantle, & + NSPEC_CRUST_MANTLE_3DMOVIE, & + Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle,Iepsilondev_xy_crust_mantle, & + Iepsilondev_xz_crust_mantle,Iepsilondev_yz_crust_mantle) else call write_movie_volume_strains(NSPEC_CRUST_MANTLE_3DMOVIE, & - Ieps_trace_over_3_crust_mantle, & - NSPEC_CRUST_MANTLE_3DMOVIE, & - Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle,Iepsilondev_xy_crust_mantle, & - Iepsilondev_xz_crust_mantle,Iepsilondev_yz_crust_mantle) + Ieps_trace_over_3_crust_mantle, & + NSPEC_CRUST_MANTLE_3DMOVIE, & + Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle,Iepsilondev_xy_crust_mantle, & + Iepsilondev_xz_crust_mantle,Iepsilondev_yz_crust_mantle) endif case (4) From 9a5bc9408da495d105ec07736780e904a6f08f37 Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Tue, 5 Nov 2024 10:32:17 -0700 Subject: [PATCH 17/19] replace a use parameter within USE_HDF5 scope --- src/meshfem3D/save_arrays_solver_hdf5.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/meshfem3D/save_arrays_solver_hdf5.F90 b/src/meshfem3D/save_arrays_solver_hdf5.F90 index 154d6e54b..03506a848 100644 --- a/src/meshfem3D/save_arrays_solver_hdf5.F90 +++ b/src/meshfem3D/save_arrays_solver_hdf5.F90 @@ -783,9 +783,9 @@ subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & num_colors_outer,num_colors_inner, & num_elem_colors) use constants - use shared_parameters, only: H5_COL #ifdef USE_HDF5 + use shared_parameters, only: H5_COL use meshfem_par, only: & NPROCTOT use manager_hdf5 From 02860b5db1c7a12e8985405af09b36bfb6248e20 Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Tue, 5 Nov 2024 10:52:26 -0700 Subject: [PATCH 18/19] erase compiler errors --- src/shared/hdf5_manager.F90 | 90 ++++++++-------- src/specfem3D/write_movie_volume_hdf5.F90 | 27 +++-- src/specfem3D/write_output_HDF5.F90 | 123 +++++++++++----------- src/specfem3D/write_seismograms.f90 | 2 +- 4 files changed, 120 insertions(+), 122 deletions(-) diff --git a/src/shared/hdf5_manager.F90 b/src/shared/hdf5_manager.F90 index d53259f45..fc1a64495 100644 --- a/src/shared/hdf5_manager.F90 +++ b/src/shared/hdf5_manager.F90 @@ -4804,51 +4804,51 @@ end subroutine h5_write_dataset_1d_r_collect_hyperslab_in_group ! - subroutine h5_write_dataset_1d_d_collect_hyperslab(dataset_name, data, offset_in, if_collective) - implicit none - character(len=*), intent(in) :: dataset_name - double precision, dimension(:), intent(in), target :: data - integer, dimension(:), intent(in) :: offset_in - logical, intent(in) :: if_collective - ! local parameters - integer :: rank = 1 - integer(HSIZE_T), dimension(1) :: dim - integer(HSIZE_T), dimension(1) :: count ! size of hyperslab - integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted - - dim = shape(data) - offset = offset_in ! convert data type - - ! open dataset - call h5_open_dataset2(trim(dataset_name)) - - ! select a place where data is inserted. - count(1) = dim(1) - - ! select hyperslab in the file - call h5screate_simple_f(rank,count, mem_dspace_id, error) - call check_error() - call h5dget_space_f(dataset_id, file_dspace_id, error) - call check_error() - call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) - call check_error() - call h5_create_dataset_prop_list(if_collective) - - ! write array using Fortran pointer - !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & - ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) - call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1)), error, & - file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) - - if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name - call check_error() - call h5_close_prop_list(dataset_name) - call h5sclose_f(mem_dspace_id, error) - call check_error() - call h5sclose_f(file_dspace_id, error) - call check_error() - call h5_close_dataset() - end subroutine h5_write_dataset_1d_d_collect_hyperslab +! subroutine h5_write_dataset_1d_d_collect_hyperslab(dataset_name, data, offset_in, if_collective) +! implicit none +! character(len=*), intent(in) :: dataset_name +! double precision, dimension(:), intent(in), target :: data +! integer, dimension(:), intent(in) :: offset_in +! logical, intent(in) :: if_collective +! ! local parameters +! integer :: rank = 1 +! integer(HSIZE_T), dimension(1) :: dim +! integer(HSIZE_T), dimension(1) :: count ! size of hyperslab +! integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted +! +! dim = shape(data) +! offset = offset_in ! convert data type +! +! ! open dataset +! call h5_open_dataset2(trim(dataset_name)) +! +! ! select a place where data is inserted. +! count(1) = dim(1) +! +! ! select hyperslab in the file +! call h5screate_simple_f(rank,count, mem_dspace_id, error) +! call check_error() +! call h5dget_space_f(dataset_id, file_dspace_id, error) +! call check_error() +! call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) +! call check_error() +! call h5_create_dataset_prop_list(if_collective) +! +! ! write array using Fortran pointer +! !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & +! ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) +! call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1)), error, & +! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) +! +! if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name +! call check_error() +! call h5_close_prop_list(dataset_name) +! call h5sclose_f(mem_dspace_id, error) +! call check_error() +! call h5sclose_f(file_dspace_id, error) +! call check_error() +! call h5_close_dataset() +! end subroutine h5_write_dataset_1d_d_collect_hyperslab ! diff --git a/src/specfem3D/write_movie_volume_hdf5.F90 b/src/specfem3D/write_movie_volume_hdf5.F90 index 6920d0350..7bade7083 100644 --- a/src/specfem3D/write_movie_volume_hdf5.F90 +++ b/src/specfem3D/write_movie_volume_hdf5.F90 @@ -134,11 +134,11 @@ subroutine write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovi muvstore_crust_mantle_3dmovie,npoints_3dmovie,nelems_3dmovie_in) use specfem_par + +#ifdef USE_HDF5 use specfem_par_crustmantle, only: ibool_crust_mantle,rstore_crust_mantle use specfem_par_outercore, only: ibool_outer_core,rstore_outer_core use specfem_par_innercore, only: ibool_inner_core,rstore_inner_core - -#ifdef USE_HDF5 use specfem_par_movie_hdf5 #endif @@ -548,11 +548,10 @@ subroutine write_movie_volume_strains_hdf5(vnspec_eps_cm, & epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle) use constants_solver +#ifdef USE_HDF5 use shared_parameters, only: OUTPUT_FILES,MOVIE_VOLUME_TYPE,MOVIE_COARSE,H5_COL use specfem_par, only: it use specfem_par_movie, only: npoints_3dmovie,muvstore_crust_mantle_3dmovie,mask_3dmovie,nu_3dmovie - -#ifdef USE_HDF5 use specfem_par_movie_hdf5 #endif @@ -720,13 +719,12 @@ subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_ ! outputs divergence and curl: MOVIE_VOLUME_TYPE == 4 use constants_solver + +#ifdef USE_HDF5 use shared_parameters, only: OUTPUT_FILES,H5_COL use specfem_par, only: it use specfem_par_crustmantle, only: ibool_crust_mantle use specfem_par_innercore, only: ibool_inner_core - !use specfem_par_outercore, only: ibool_outer_core - -#ifdef USE_HDF5 use specfem_par_movie_hdf5 #endif @@ -911,10 +909,9 @@ subroutine write_movie_volume_vector_hdf5(npoints_3dmovie, & ! outputs displacement/velocity: MOVIE_VOLUME_TYPE == 5 / 6 use constants_solver +#ifdef USE_HDF5 use shared_parameters, only: OUTPUT_FILES,MOVIE_VOLUME_TYPE,MOVIE_COARSE,H5_COL use specfem_par, only: it - -#ifdef USE_HDF5 use specfem_par_movie_hdf5 #endif @@ -1048,10 +1045,10 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core ! outputs norm of displacement: MOVIE_VOLUME_TYPE == 7 use constants_solver - use shared_parameters, only: OUTPUT_FILES,H5_COL - use specfem_par, only: it, scale_displ #ifdef USE_HDF5 + use shared_parameters, only: OUTPUT_FILES,H5_COL + use specfem_par, only: it, scale_displ use specfem_par_movie_hdf5 #endif @@ -1200,10 +1197,10 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v ! outputs norm of velocity: MOVIE_VOLUME_TYPE == 8 use constants_solver - use shared_parameters, only: OUTPUT_FILES,H5_COL - use specfem_par, only: it, scale_veloc #ifdef USE_HDF5 + use shared_parameters, only: OUTPUT_FILES,H5_COL + use specfem_par, only: it, scale_veloc use specfem_par_movie_hdf5 #endif @@ -1352,10 +1349,10 @@ subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core ! outputs norm of acceleration: MOVIE_VOLUME_TYPE == 1 use constants_solver - use shared_parameters, only: OUTPUT_FILES,H5_COL - use specfem_par, only: it, scale_t_inv,scale_veloc #ifdef USE_HDF5 + use shared_parameters, only: OUTPUT_FILES,H5_COL + use specfem_par, only: it, scale_t_inv,scale_veloc use specfem_par_movie_hdf5 #endif diff --git a/src/specfem3D/write_output_HDF5.F90 b/src/specfem3D/write_output_HDF5.F90 index 7e93a20db..28a13d978 100644 --- a/src/specfem3D/write_output_HDF5.F90 +++ b/src/specfem3D/write_output_HDF5.F90 @@ -169,88 +169,89 @@ end subroutine write_hdf5_seismogram_init subroutine write_output_hdf5(seismogram_tmp_in, irec_local, irec, chn, iorientation) use specfem_par, only: & - seismo_current, nrec ,& nlength_seismogram, & - ROTATE_SEISMOGRAMS_RT, NTSTEP_BETWEEN_OUTPUT_SAMPLE, & - CUSTOM_REAL, hdf5_seismo_fname, & - WRITE_SEISMOGRAMS_BY_MAIN, ROTATE_SEISMOGRAMS_RT, myrank + CUSTOM_REAL +#ifdef USE_HDF5 + use specfem_par, only: & + myrank, seismo_current, nrec, & + ROTATE_SEISMOGRAMS_RT, NTSTEP_BETWEEN_OUTPUT_SAMPLE, & + WRITE_SEISMOGRAMS_BY_MAIN, hdf5_seismo_fname, & use shared_parameters, only: & NSTEP, OUTPUT_SEISMOS_HDF5 - -#ifdef USE_HDF5 use manager_hdf5 #endif - implicit none - - ! input/output variables - character(len=4),intent(in) :: chn - integer,intent(in) :: irec_local, irec, iorientation - real(kind=CUSTOM_REAL),dimension(5,nlength_seismogram),intent(in) :: seismogram_tmp_in + implicit none - real(kind=CUSTOM_REAL), dimension(nlength_seismogram,1) :: seismogram_tmp - logical, save :: is_initialized = .false. - integer :: i, ier, nlength_total_seismogram - logical :: if_dataset_exists + ! input/output variables + character(len=4),intent(in) :: chn + integer,intent(in) :: irec_local, irec, iorientation + real(kind=CUSTOM_REAL),dimension(5,nlength_seismogram),intent(in) :: seismogram_tmp_in #ifdef USE_HDF5 + real(kind=CUSTOM_REAL), dimension(nlength_seismogram,1) :: seismogram_tmp + logical, save :: is_initialized = .false. + integer :: i, ier, nlength_total_seismogram + logical :: if_dataset_exists + + ! check if anything to do - if (.not. OUTPUT_SEISMOS_HDF5) return - - ! safety check - if (.not. WRITE_SEISMOGRAMS_BY_MAIN) & - stop 'Error: WRITE_SEIMSMOGRAMS_BY_MAIN must be true to use HDF5 seismogram output' - - ! total length of the seismogram - nlength_total_seismogram = NSTEP / NTSTEP_BETWEEN_OUTPUT_SAMPLE - print*, 'nlength_total_seismogram = ', nlength_total_seismogram - print*, 'NSTEP = ', NSTEP - print*, 'NTSTEP_BETWEEN_OUTPUT_SAMPLE = ', NTSTEP_BETWEEN_OUTPUT_SAMPLE - print*, 'nrec = ', nrec - print*, 'irec = ', irec - print*, 'seismo_current = ', seismo_current - print*, 'shape(seismogram_tmp) = ', shape(seismogram_tmp) - print*, 'shape(seismogram_tmp(iorientation,1:seismo_current)) = ', shape(seismogram_tmp(iorientation,1:seismo_current)) - ! convert array with shape (seimo_current) to (nlength_total_seismogram, 1) - do i = 1, seismo_current - seismogram_tmp(i,1) = seismogram_tmp_in(iorientation,i) - end do - - - - ! initialize - if (.not. is_initialized) then - call write_hdf5_seismogram_init(nlength_total_seismogram) - is_initialized = .true. - endif + if (.not. OUTPUT_SEISMOS_HDF5) return + + ! safety check + if (.not. WRITE_SEISMOGRAMS_BY_MAIN) & + stop 'Error: WRITE_SEIMSMOGRAMS_BY_MAIN must be true to use HDF5 seismogram output' + + ! total length of the seismogram + nlength_total_seismogram = NSTEP / NTSTEP_BETWEEN_OUTPUT_SAMPLE + print*, 'nlength_total_seismogram = ', nlength_total_seismogram + print*, 'NSTEP = ', NSTEP + print*, 'NTSTEP_BETWEEN_OUTPUT_SAMPLE = ', NTSTEP_BETWEEN_OUTPUT_SAMPLE + print*, 'nrec = ', nrec + print*, 'irec = ', irec + print*, 'seismo_current = ', seismo_current + print*, 'shape(seismogram_tmp) = ', shape(seismogram_tmp) + print*, 'shape(seismogram_tmp(iorientation,1:seismo_current)) = ', shape(seismogram_tmp(iorientation,1:seismo_current)) + ! convert array with shape (seimo_current) to (nlength_total_seismogram, 1) + do i = 1, seismo_current + seismogram_tmp(i,1) = seismogram_tmp_in(iorientation,i) + end do - ! only main process writes out - if (myrank /= 0) return - ! write the seismograms - call h5_open_file(hdf5_seismo_fname) - ! check if the target dataset components are already created - call h5_check_dataset_exists(trim(chn),if_dataset_exists) + ! initialize + if (.not. is_initialized) then + call write_hdf5_seismogram_init(nlength_total_seismogram) + is_initialized = .true. + endif - ! if the dataset does not exist, create it - if (.not. if_dataset_exists) then - call h5_create_dataset_gen(trim(chn),(/nlength_total_seismogram, nrec/), 2, CUSTOM_REAL) - endif + ! only main process writes out + if (myrank /= 0) return + + ! write the seismograms + call h5_open_file(hdf5_seismo_fname) - ! write the seismogram - call h5_write_dataset_collect_hyperslab(trim(chn), seismogram_tmp(1:seismo_current, :), (/0,irec-1/), .false.) + ! check if the target dataset components are already created + call h5_check_dataset_exists(trim(chn),if_dataset_exists) - ! close the file - call h5_close_file() + ! if the dataset does not exist, create it + if (.not. if_dataset_exists) then + call h5_create_dataset_gen(trim(chn),(/nlength_total_seismogram, nrec/), 2, CUSTOM_REAL) + endif + + ! write the seismogram + call h5_write_dataset_collect_hyperslab(trim(chn), seismogram_tmp(1:seismo_current, :), (/0,irec-1/), .false.) + + ! close the file + call h5_close_file() #else - write(*,*) 'Error: HDF5 support not enabled in this version of Specfem3D_Globe' - write(*,*) 'Please recompile with the --with-hdf5 option' - stop + write(*,*) 'Error: HDF5 support not enabled in this version of Specfem3D_Globe' + write(*,*) 'Please recompile with the --with-hdf5 option' + stop #endif diff --git a/src/specfem3D/write_seismograms.f90 b/src/specfem3D/write_seismograms.f90 index de8618b5d..bd2542eb0 100644 --- a/src/specfem3D/write_seismograms.f90 +++ b/src/specfem3D/write_seismograms.f90 @@ -39,7 +39,7 @@ subroutine write_seismograms() nlength_seismogram, & NTSTEP_BETWEEN_OUTPUT_SEISMOS,NTSTEP_BETWEEN_OUTPUT_SAMPLE, & do_save_seismograms, & - WRITE_SEISMOGRAMS_BY_MAIN,OUTPUT_SEISMOS_ASDF, OUTPUT_SEISMOS_HDF5, & + WRITE_SEISMOGRAMS_BY_MAIN,OUTPUT_SEISMOS_ASDF, & SAVE_SEISMOGRAMS_STRAIN, & moment_der,sloc_der,shdur_der,stshift_der, & scale_displ From 794973ba990cf476461b02cdc64f3c36a9ab345b Mon Sep 17 00:00:00 2001 From: Masaru Nagaso Date: Tue, 5 Nov 2024 11:49:56 -0700 Subject: [PATCH 19/19] eliminate compiler warnings on hdf5 side --- src/meshfem3D/save_arrays_solver_hdf5.F90 | 138 ++++++---- src/specfem3D/read_arrays_solver_hdf5.F90 | 231 ++++++++++------ src/specfem3D/read_forward_arrays_hdf5.F90 | 198 +++++++++----- src/specfem3D/save_forward_arrays_hdf5.F90 | 294 ++++++++++++++------- src/specfem3D/save_kernels_hdf5.F90 | 152 +++++++---- src/specfem3D/write_movie_volume_hdf5.F90 | 236 +++++++++++------ src/specfem3D/write_output_HDF5.F90 | 2 +- 7 files changed, 829 insertions(+), 422 deletions(-) diff --git a/src/meshfem3D/save_arrays_solver_hdf5.F90 b/src/meshfem3D/save_arrays_solver_hdf5.F90 index 03506a848..d7a6d9faf 100644 --- a/src/meshfem3D/save_arrays_solver_hdf5.F90 +++ b/src/meshfem3D/save_arrays_solver_hdf5.F90 @@ -447,9 +447,11 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & call h5_write_dataset_collect_hyperslab_in_group('c66store', c66store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) else if (TRANSVERSE_ISOTROPY) then - call h5_write_dataset_collect_hyperslab_in_group('kappahstore', kappahstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('kappahstore', kappahstore, & + (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('muhstore', muhstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('eta_anisostore', eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('eta_anisostore', eta_anisostore, & + (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif endif ! for azimutahl aniso kernels @@ -496,7 +498,8 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & ! ocean load mass matrix if (OCEANS .and. iregion_code == IREGION_CRUST_MANTLE) then - call h5_write_dataset_collect_hyperslab_in_group('rmass_ocean_load', rmass_ocean_load, (/sum(offset_nnodes_oceans(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('rmass_ocean_load', rmass_ocean_load, & + (/sum(offset_nnodes_oceans(0:myrank-1))/), H5_COL) endif ! close group for iregion_code @@ -528,34 +531,52 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_top', (/arr_nspec2d_top(myrank)/), (/myrank/), H5_COL) if (arr_nspec2d_xmin(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmin', ibelm_xmin, (/sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('normal_xmin', normal_xmin, (/0,0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmin', jacobian2D_xmin, (/0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmin', ibelm_xmin, & + (/sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_xmin', normal_xmin, & + (/0,0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmin', jacobian2D_xmin, & + (/0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) endif if (arr_nspec2d_xmax(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmax', ibelm_xmax, (/sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('normal_xmax', normal_xmax, (/0,0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmax', jacobian2D_xmax, (/0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmax', ibelm_xmax, & + (/sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_xmax', normal_xmax, & + (/0,0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmax', jacobian2D_xmax, & + (/0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) endif if (arr_nspec2d_ymin(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymin', ibelm_ymin, (/sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('normal_ymin', normal_ymin, (/0,0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymin', jacobian2D_ymin, (/0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymin', ibelm_ymin, & + (/sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_ymin', normal_ymin, & + (/0,0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymin', jacobian2D_ymin, & + (/0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) endif if (arr_nspec2d_ymax(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymax', ibelm_ymax, (/sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('normal_ymax', normal_ymax, (/0,0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymax', jacobian2D_ymax, (/0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymax', ibelm_ymax, & + (/sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_ymax', normal_ymax, & + (/0,0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymax', jacobian2D_ymax, & + (/0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) endif if (arr_nspec2d_bottom(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_bottom', ibelm_bottom, (/sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('normal_bottom', normal_bottom, (/0,0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_bottom', jacobian2D_bottom, (/0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_bottom', ibelm_bottom, & + (/sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_bottom', normal_bottom, & + (/0,0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_bottom', jacobian2D_bottom, & + (/0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) endif if (arr_nspec2d_top(myrank) /= 0) then - call h5_write_dataset_collect_hyperslab_in_group('ibelm_top', ibelm_top, (/sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('normal_top', normal_top, (/0,0,0,sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_top', jacobian2D_top, (/0,0,sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_top', ibelm_top, & + (/sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_top', normal_top, & + (/0,0,0,sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_top', jacobian2D_top, & + (/0,0,sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) endif ! close group for iregion_code @@ -574,8 +595,10 @@ subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & ! create dataset for attenuation call h5_write_dataset_collect_hyperslab_in_group('tau_s_store', tau_s_store, (/myrank*N_SLS/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('tau_e_store', tau_e_store, (/0,0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('Qmu_store', Qmu_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('tau_e_store', tau_e_store, & + (/0,0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('Qmu_store', Qmu_store, & + (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('att_f_c_source', (/ATT_F_C_SOURCE/), (/myrank/), H5_COL) call h5_close_group() @@ -742,13 +765,16 @@ subroutine save_arrays_boundary_hdf5() call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_670_top', (/act_arr_nspec2d_670_top(myrank)/), (/myrank/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_670_bot', (/act_arr_nspec2d_670_bot(myrank)/), (/myrank/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_top', ibelm_moho_top, (/sum(arr_nspec2d_moho_top(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_bot', ibelm_moho_bot, (/sum(arr_nspec2d_moho_bot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_top', ibelm_moho_top, & + (/sum(arr_nspec2d_moho_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_bot', ibelm_moho_bot, & + (/sum(arr_nspec2d_moho_bot(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('ibelm_400_top', ibelm_400_top, (/sum(arr_nspec2d_400_top(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('ibelm_400_bot', ibelm_400_bot, (/sum(arr_nspec2d_400_bot(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('ibelm_670_top', ibelm_670_top, (/sum(arr_nspec2d_670_top(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('ibelm_670_bot', ibelm_670_bot, (/sum(arr_nspec2d_670_bot(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('normal_moho', normal_moho, (/0,0,0,sum(arr_nspec2d_moho_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_moho', normal_moho, & + (/0,0,0,sum(arr_nspec2d_moho_top(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('normal_400', normal_400, (/0,0,0,sum(arr_nspec2d_400_top(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('normal_670', normal_670, (/0,0,0,sum(arr_nspec2d_670_top(0:myrank-1))/), H5_COL) @@ -868,7 +894,8 @@ subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & call h5_create_dataset_gen_in_group("max_nibool_interfaces", (/NPROCTOT/), 1, 1) call h5_create_dataset_gen_in_group("my_neighbors", (/sum(offset_num_interfaces(:))/), 1, 1) call h5_create_dataset_gen_in_group("nibool_interfaces", (/sum(offset_num_interfaces(:))/), 1, 1) - call h5_create_dataset_gen_in_group("ibool_interfaces", (/maxval(offset_max_nibool_interfaces),sum(offset_num_interfaces(:))/), 2, 1) + call h5_create_dataset_gen_in_group("ibool_interfaces", & + (/maxval(offset_max_nibool_interfaces),sum(offset_num_interfaces(:))/), 2, 1) else ! dummy call h5_create_dataset_gen_in_group("max_nibool_interfaces", (/NPROCTOT/), 1, 1) @@ -887,7 +914,8 @@ subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & ! mesh coloring if (sum(offset_num_colors_outer) + sum(offset_num_colors_inner) > 0) then - call h5_create_dataset_gen_in_group("num_elem_colors", (/sum(offset_num_colors_outer(:)) + sum(offset_num_colors_inner(:))/), 1, 1) + call h5_create_dataset_gen_in_group("num_elem_colors", & + (/sum(offset_num_colors_outer(:)) + sum(offset_num_colors_inner(:))/), 1, 1) else ! dummy call h5_create_dataset_gen_in_group("num_elem_colors", (/NPROCTOT/), 1, 1) @@ -910,18 +938,25 @@ subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & ! write datasets call h5_write_dataset_collect_hyperslab_in_group('offset_num_interfaces', (/offset_num_interfaces(myrank)/), (/myrank/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('offset_max_nibool_interfaces', (/offset_max_nibool_interfaces(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_max_nibool_interfaces', & + (/offset_max_nibool_interfaces(myrank)/), (/myrank/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('offset_nspec_inner', (/offset_nspec_inner(myrank)/), (/myrank/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('offset_nspec_outer', (/offset_nspec_outer(myrank)/), (/myrank/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('offset_num_phase_ispec', (/offset_num_phase_ispec(myrank)/), (/myrank/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_outer', (/offset_num_colors_outer(myrank)/), (/myrank/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_inner', (/offset_num_colors_inner(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_phase_ispec', & + (/offset_num_phase_ispec(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_outer', & + (/offset_num_colors_outer(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_inner', & + (/offset_num_colors_inner(myrank)/), (/myrank/), H5_COL) if (sum(offset_num_interfaces) > 0) then call h5_write_dataset_collect_hyperslab_in_group("max_nibool_interfaces", (/max_nibool_interfaces/), (/myrank/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group("my_neighbors", my_neighbors, (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group("nibool_interfaces", nibool_interfaces, (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group("ibool_interfaces", ibool_interfaces, (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("my_neighbors", my_neighbors, & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("nibool_interfaces", nibool_interfaces, & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("ibool_interfaces", ibool_interfaces, & + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) else ! dummy call h5_write_dataset_collect_hyperslab_in_group("max_nibool_interfaces", (/max_nibool_interfaces/), (/myrank/), H5_COL) @@ -932,7 +967,8 @@ subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & ! num_phase_ispec if (sum(offset_num_phase_ispec) > 0) then - call h5_write_dataset_collect_hyperslab_in_group("phase_ispec_inner", phase_ispec_inner, (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("phase_ispec_inner", phase_ispec_inner, & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) else ! dummy call h5_write_dataset_collect_hyperslab_in_group("phase_ispec_inner", phase_ispec_inner, (/myrank,0/), H5_COL) @@ -940,7 +976,8 @@ subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & ! mesh coloring if (sum(offset_num_colors_outer) + sum(offset_num_colors_inner) > 0) then - call h5_write_dataset_collect_hyperslab_in_group("num_elem_colors", num_elem_colors, (/sum(offset_num_colors_outer(0:myrank-1)) + sum(offset_num_colors_inner(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("num_elem_colors", num_elem_colors, & + (/sum(offset_num_colors_outer(0:myrank-1)) + sum(offset_num_colors_inner(0:myrank-1))/), H5_COL) else ! dummy call h5_write_dataset_collect_hyperslab_in_group("num_elem_colors", num_elem_colors, (/myrank/), H5_COL) @@ -1037,11 +1074,14 @@ subroutine get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & if (sum(offset_num_abs_boundary_faces) > 0) then call h5_create_dataset_gen_in_group("abs_boundary_ispec", (/sum(offset_num_abs_boundary_faces(:))/), 1, 1) call h5_create_dataset_gen_in_group("abs_boundary_npoin", (/sum(offset_num_abs_boundary_faces(:))/), 1, 1) - call h5_create_dataset_gen_in_group("abs_boundary_ijk", (/NDIM,NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 3, CUSTOM_REAL) - call h5_create_dataset_gen_in_group("abs_boundary_jacobian2Dw", (/NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 2, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("abs_boundary_ijk", & + (/NDIM,NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("abs_boundary_jacobian2Dw", & + (/NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 2, CUSTOM_REAL) if (iregion == IREGION_CRUST_MANTLE) then - call h5_create_dataset_gen_in_group("abs_boundary_normal", (/NDIM,NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("abs_boundary_normal", & + (/NDIM,NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 3, CUSTOM_REAL) endif else ! dummy @@ -1066,15 +1106,21 @@ subroutine get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & call h5_open_group(gname_region) ! write datasets - call h5_write_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", (/offset_num_abs_boundary_faces(myrank)/), (/myrank/), .true.) + call h5_write_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", & + (/offset_num_abs_boundary_faces(myrank)/), (/myrank/), .true.) if (sum(offset_num_abs_boundary_faces) > 0) then - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk, & + (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw, & + (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) if (iregion == IREGION_CRUST_MANTLE) then - call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal, & + (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) endif else ! dummy diff --git a/src/specfem3D/read_arrays_solver_hdf5.F90 b/src/specfem3D/read_arrays_solver_hdf5.F90 index 1ea524a38..e33ac481e 100644 --- a/src/specfem3D/read_arrays_solver_hdf5.F90 +++ b/src/specfem3D/read_arrays_solver_hdf5.F90 @@ -221,11 +221,13 @@ subroutine read_arrays_solver_hdf5(iregion_code, & else if (TRANSVERSE_ISOTROPY_VAL) then ! kappahstore - call h5_read_dataset_collect_hyperslab_in_group("kappahstore", kappahstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("kappahstore", kappahstore, & + (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! muhstore call h5_read_dataset_collect_hyperslab_in_group("muhstore", muhstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) ! eta_anisostore - call h5_read_dataset_collect_hyperslab_in_group("eta_anisostore", eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("eta_anisostore", eta_anisostore, & + (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) endif endif @@ -285,7 +287,8 @@ subroutine read_arrays_solver_hdf5(iregion_code, & ! read additional ocean load mass matrix if (OCEANS_VAL .and. iregion_code == IREGION_CRUST_MANTLE) then ! rmass_ocean_load - call h5_read_dataset_collect_hyperslab_in_group("rmass_ocean_load", rmass_ocean_load, (/sum(offset_nnodes_oceans(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("rmass_ocean_load", rmass_ocean_load, & + (/sum(offset_nnodes_oceans(0:myrank-1))/), H5_COL) endif ! close group @@ -737,43 +740,61 @@ subroutine read_mesh_databases_coupling_hdf5() call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) ! ibelm_xmin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_crust_mantle, (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_crust_mantle, & + (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! ibelm_xmax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_crust_mantle, (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_crust_mantle, & + (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! ibelm_ymin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_crust_mantle, (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_crust_mantle, & + (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! ibelm_ymax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_crust_mantle, (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_crust_mantle, & + (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! ibelm_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_crust_mantle, (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_crust_mantle, & + (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! ibelm_bottom - call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_crust_mantle, (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_crust_mantle, & + (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! normal_xmin - call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_crust_mantle, (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! normal_xmax - call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_crust_mantle, (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! normal_ymin - call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_crust_mantle, (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! normal_ymax - call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_crust_mantle, (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! normal_top - call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_crust_mantle, (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! normal_bottom - call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_crust_mantle, (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! jacobian_xmin - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_crust_mantle, (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_crust_mantle, & + (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! jacobian_xmax - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_crust_mantle, (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_crust_mantle,& + (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! jacobian_ymin - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_crust_mantle, (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_crust_mantle, & + (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! jacobian_ymax - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_crust_mantle, (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_crust_mantle, & + (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! jacobian_top - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_crust_mantle, (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_crust_mantle, & + (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! jacobian_bottom - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_crust_mantle, (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_crust_mantle, & + (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -803,43 +824,61 @@ subroutine read_mesh_databases_coupling_hdf5() call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) ! ibelm_xmin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_outer_core, (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_outer_core, & + (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! ibelm_xmax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_outer_core, (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_outer_core, & + (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! ibelm_ymin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_outer_core, (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_outer_core, & + (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! ibelm_ymax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_outer_core, (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_outer_core, & + (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! ibelm_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_outer_core, (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_outer_core, & + (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! ibelm_bottom - call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_outer_core, (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_outer_core, & + (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! normal_xmin - call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_outer_core, (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_outer_core, & + (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! normal_xmax - call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_outer_core, (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_outer_core, & + (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! normal_ymin - call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_outer_core, (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_outer_core, & + (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! normal_ymax - call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_outer_core, (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_outer_core, & + (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! normal_top - call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_outer_core, (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_outer_core, & + (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! normal_bot - call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_outer_core, (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_outer_core, & + (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! jacobian_xmin - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_outer_core, (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_outer_core, & + (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! jacobian_xmax - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_outer_core, (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_outer_core, & + (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! jacobian_ymin - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_outer_core, (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_outer_core, & + (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! jacobian_ymax - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_outer_core, (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_outer_core, & + (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! jacobian_top - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_outer_core, (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_outer_core, & + (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! jacobian_bottom - call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_outer_core, (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_outer_core, & + (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -870,17 +909,23 @@ subroutine read_mesh_databases_coupling_hdf5() call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) ! ibelm_xmin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_inner_core, (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_inner_core, & + (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! ibelm_xmax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_inner_core, (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_inner_core, & + (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! ibelm_ymin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_inner_core, (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_inner_core, & + (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! ibelm_ymax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_inner_core, (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_inner_core, & + (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! ibelm_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_inner_core, (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_inner_core, & + (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! ibelm_bottom - call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_inner_core, (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_inner_core, & + (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -912,17 +957,23 @@ subroutine read_mesh_databases_coupling_hdf5() call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) ! ibelm_xmin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_trinfinite, (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_trinfinite, & + (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! ibelm_xmax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_trinfinite, (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_trinfinite, & + (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! ibelm_ymin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_trinfinite, (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_trinfinite, & + (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! ibelm_ymax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_trinfinite, (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_trinfinite, & + (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! ibelm_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_trinfinite, (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_trinfinite, & + (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! ibelm_bottom - call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_trinfinite, (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_trinfinite, & + (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -953,17 +1004,23 @@ subroutine read_mesh_databases_coupling_hdf5() call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) ! ibelm_xmin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_infinite, (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_infinite, & + (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) ! ibelm_xmax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_infinite, (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_infinite, & + (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) ! ibelm_ymin - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_infinite, (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_infinite, & + (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) ! ibelm_ymax - call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_infinite, (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_infinite, & + (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) ! ibelm_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_infinite, (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_infinite, & + (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) ! ibelm_bottom - call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_infinite, (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_infinite, & + (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -992,7 +1049,8 @@ subroutine read_mesh_databases_coupling_hdf5() ! checks setup if (tmp_nspec2d_moho /= NSPEC2D_MOHO .or. tmp_nspec2d_400 /= NSPEC2D_400 .or. tmp_nspec2d_670 /= NSPEC2D_670) then - print *,'Error: invalid NSPEC2D values read in for solver: ',tmp_nspec2d_moho,tmp_nspec2d_400,tmp_nspec2d_670,'(boundary_disc.h5)' + print *,'Error: invalid NSPEC2D values read in for solver: ',& + tmp_nspec2d_moho,tmp_nspec2d_400,tmp_nspec2d_670,'(boundary_disc.h5)' print *,' should be MOHO/400/670 : ',NSPEC2D_MOHO,NSPEC2D_400,NSPEC2D_670,'(mesh_parameters.h5)' call exit_mpi(myrank, 'Error reading boundary_disc.h5 file') endif @@ -1012,23 +1070,32 @@ subroutine read_mesh_databases_coupling_hdf5() call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_670_bot", offset_nspec2D_670_bottom, (/0/), H5_COL) ! ibelm_moho_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_top", ibelm_moho_top, (/sum(offset_nspec2D_moho_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_top", ibelm_moho_top, & + (/sum(offset_nspec2D_moho_top(0:myrank-1))/), H5_COL) ! ibelm_moho_bot - call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_bot", ibelm_moho_bot, (/sum(offset_nspec2D_moho_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_bot", ibelm_moho_bot, & + (/sum(offset_nspec2D_moho_bottom(0:myrank-1))/), H5_COL) ! ibelm_400_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_top", ibelm_400_top, (/sum(offset_nspec2D_400_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_top", ibelm_400_top, & + (/sum(offset_nspec2D_400_top(0:myrank-1))/), H5_COL) ! ibelm_400_bot - call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_bot", ibelm_400_bot, (/sum(offset_nspec2D_400_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_bot", ibelm_400_bot, & + (/sum(offset_nspec2D_400_bottom(0:myrank-1))/), H5_COL) ! ibelm_670_top - call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_top", ibelm_670_top, (/sum(offset_nspec2D_670_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_top", ibelm_670_top, & + (/sum(offset_nspec2D_670_top(0:myrank-1))/), H5_COL) ! ibelm_670_bot - call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_bot", ibelm_670_bot, (/sum(offset_nspec2D_670_bottom(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_bot", ibelm_670_bot, & + (/sum(offset_nspec2D_670_bottom(0:myrank-1))/), H5_COL) ! normal_moho - call h5_read_dataset_collect_hyperslab_in_group("normal_moho", normal_moho, (/0,0,0,sum(offset_nspec2D_moho_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_moho", normal_moho, & + (/0,0,0,sum(offset_nspec2D_moho_top(0:myrank-1))/), H5_COL) ! normal_400 - call h5_read_dataset_collect_hyperslab_in_group("normal_400", normal_400, (/0,0,0,sum(offset_nspec2D_400_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_400", normal_400, & + (/0,0,0,sum(offset_nspec2D_400_top(0:myrank-1))/), H5_COL) ! normal_670 - call h5_read_dataset_collect_hyperslab_in_group("normal_670", normal_670, (/0,0,0,sum(offset_nspec2D_670_top(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("normal_670", normal_670, & + (/0,0,0,sum(offset_nspec2D_670_top(0:myrank-1))/), H5_COL) ! close group call h5_close_group() @@ -1111,15 +1178,20 @@ subroutine read_mesh_databases_stacey_hdf5 if (ier /= 0) stop 'Error allocating array abs_boundary_ispec etc.' ! abs_boundary_ispec - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_crust_mantle, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_crust_mantle, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_npoin - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_crust_mantle, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_crust_mantle, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_ijk - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_crust_mantle, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_crust_mantle, & + (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_jacobian2Dw - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_crust_mantle, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_crust_mantle, & + (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_normal - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_crust_mantle, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_crust_mantle, & + (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) else ! dummy ! dummy arrays @@ -1168,15 +1240,20 @@ subroutine read_mesh_databases_stacey_hdf5 if (ier /= 0) stop 'Error allocating array abs_boundary_ispec etc.' ! abs_boundary_ispec - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_outer_core, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_outer_core, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_npoin - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_outer_core, (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_outer_core, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_ijk - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_outer_core, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_outer_core, & + (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_jacobian2Dw - call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_outer_core, (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_outer_core, & + (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) ! abs_boundary_normal - !call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_outer_core, (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + !call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_outer_core, & + ! (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) else ! dummy ! dummy arrays diff --git a/src/specfem3D/read_forward_arrays_hdf5.F90 b/src/specfem3D/read_forward_arrays_hdf5.F90 index e23c4c5de..1023a50c4 100644 --- a/src/specfem3D/read_forward_arrays_hdf5.F90 +++ b/src/specfem3D/read_forward_arrays_hdf5.F90 @@ -101,33 +101,55 @@ subroutine read_intermediate_forward_arrays_hdf5() call h5_read_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) call h5_read_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), H5_COL) call h5_read_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_read_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('A_array_rotation', B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', B_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif if (FULL_GRAVITY_VAL) then @@ -225,33 +247,55 @@ subroutine read_forward_arrays_hdf5() call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif if (FULL_GRAVITY_VAL) then @@ -364,33 +408,55 @@ subroutine read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif if (FULL_GRAVITY_VAL) then diff --git a/src/specfem3D/save_forward_arrays_hdf5.F90 b/src/specfem3D/save_forward_arrays_hdf5.F90 index 08b149946..8962127fc 100644 --- a/src/specfem3D/save_forward_arrays_hdf5.F90 +++ b/src/specfem3D/save_forward_arrays_hdf5.F90 @@ -101,11 +101,16 @@ subroutine save_intermediate_forward_arrays_hdf5() call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) call h5_create_dataset_gen('epsilondev_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) call h5_create_dataset_gen('epsilondev_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) call h5_create_dataset_gen('epsilondev_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) @@ -156,33 +161,55 @@ subroutine save_intermediate_forward_arrays_hdf5() call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif ! ATTENUATION_VAL if (FULL_GRAVITY_VAL) then @@ -280,16 +307,26 @@ subroutine save_forward_arrays_hdf5() call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_inner_core', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_inner_core', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_inner_core', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_inner_core', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_inner_core', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) if (ROTATION_VAL) then call h5_create_dataset_gen('A_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) @@ -336,31 +373,53 @@ subroutine save_forward_arrays_hdf5() call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif ! ATTENUATION_VAL if (FULL_GRAVITY_VAL) then @@ -461,11 +520,16 @@ subroutine save_forward_arrays_undoatt_hdf5() call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) - call h5_create_dataset_gen('epsilondev_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) call h5_create_dataset_gen('epsilondev_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) call h5_create_dataset_gen('epsilondev_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) call h5_create_dataset_gen('epsilondev_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) @@ -516,33 +580,55 @@ subroutine save_forward_arrays_undoatt_hdf5() call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) if (ROTATION_VAL) then - call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) endif if (ATTENUATION_VAL) then - call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) endif ! ATTENUATION_VAL if (FULL_GRAVITY_VAL) then @@ -751,27 +837,33 @@ subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_cru temp_store(:,:,:,:) = sqrt((kappavstore_crust_mantle(:,:,:,:) & + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & * scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vpv_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('vpv_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! vph temp_store(:,:,:,:) = sqrt((kappahstore_crust_mantle(:,:,:,:) & + FOUR_THIRDS * muh_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & * scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vph_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('vph_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! vsv temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vsv_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('vsv_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! vsh temp_store(:,:,:,:) = sqrt( muh_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vsh_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('vsh_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) else ! isotropic ! vp temp_store(:,:,:,:) = sqrt((kappavstore_crust_mantle(:,:,:,:) & + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & * scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! vs temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) endif @@ -825,10 +917,12 @@ subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_cru temp_store(:,:,:,:) = sqrt((kappavstore_inner_core(:,:,:,:) & + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_inner_core(:,:,:,:)) & * scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), H5_COL) ! vs temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_inner_core(:,:,:,:) )*scaleval1 - call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), H5_COL) endif deallocate(temp_store,muv_shifted) @@ -857,13 +951,17 @@ subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_cru call h5_open_group(group_name) ! muv_relaxed - call h5_write_dataset_collect_hyperslab_in_group('muv_relaxed', muvstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('muv_relaxed', muvstore_crust_mantle*scaleval1, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! muh_relaxed - call h5_write_dataset_collect_hyperslab_in_group('muh_relaxed', muhstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('muh_relaxed', muhstore_crust_mantle*scaleval1, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! kappav_relaxed - call h5_write_dataset_collect_hyperslab_in_group('kappav_relaxed', kappavstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('kappav_relaxed', kappavstore_crust_mantle*scaleval1, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! kappah_relaxed - call h5_write_dataset_collect_hyperslab_in_group('kappah_relaxed', kappahstore_crust_mantle*scaleval1, (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('kappah_relaxed', kappahstore_crust_mantle*scaleval1, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) ! close group call h5_close_group() diff --git a/src/specfem3D/save_kernels_hdf5.F90 b/src/specfem3D/save_kernels_hdf5.F90 index 17dea71f9..8248ba1ac 100644 --- a/src/specfem3D/save_kernels_hdf5.F90 +++ b/src/specfem3D/save_kernels_hdf5.F90 @@ -71,7 +71,8 @@ subroutine write_kernels_strength_noise_hdf5() call h5_open_file_p_collect(file_name) ! write data - call h5_write_dataset_collect_hyperslab('sigma_kernel', sigma_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('sigma_kernel', sigma_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) ! close hdf5 call h5_close_file_p() @@ -220,42 +221,68 @@ subroutine write_kernels_cm_ani_hdf5(alphav_kl_crust_mantle,alphah_kl_crust_mant ! write data if (SAVE_TRANSVERSE_KL_ONLY) then - call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - - call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - - call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) else if (SAVE_AZIMUTHAL_ANISO_KL_ONLY) then - call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - - call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - - call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - - call h5_write_dataset_collect_hyperslab('Gc_prime_kernel', Gc_prime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('Gs_prime_kernel', Gs_prime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('Gc_prime_kernel', Gc_prime_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('Gs_prime_kernel', Gs_prime_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) ! check isotropic kernel if (.false.) then - call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) endif ! check anisotropic kernels @@ -279,8 +306,10 @@ subroutine write_kernels_cm_ani_hdf5(alphav_kl_crust_mantle,alphah_kl_crust_mant else ! fully anisotropic kernels - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('cijkl_kernel', cijkl_kl_crust_mantle, (/0,0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('cijkl_kernel', cijkl_kl_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) endif @@ -362,16 +391,24 @@ subroutine write_kernels_cm_iso_hdf5(mu_kl_crust_mantle, kappa_kl_crust_mantle, call h5_open_file_p_collect(file_name) ! write data - call h5_write_dataset_collect_hyperslab('rhonotprime_kernel', rhonotprime_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('kappa_kernel', kappa_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('mu_kernel', mu_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - - call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('rhonotprime_kernel', rhonotprime_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('kappa_kernel', kappa_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('mu_kernel', mu_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) ! close hdf5 call h5_close_file_p() @@ -433,8 +470,10 @@ subroutine write_kernels_oc_hdf5() call h5_open_file_p_collect(file_name) ! write data - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_outer_core, (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_outer_core, (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_outer_core, & + (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_outer_core, & + (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), H5_COL) ! close hdf5 call h5_close_file_p() @@ -497,9 +536,12 @@ subroutine write_kernels_ic_hdf5() call h5_open_file_p_collect(file_name) ! write data - call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_inner_core, (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_inner_core, & + (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_inner_core, & + (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_inner_core, & + (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) ! close hdf5 call h5_close_file_p() @@ -649,10 +691,14 @@ subroutine write_kernels_Hessian_hdf5() call h5_open_file_p_collect(file_name) ! write data - call h5_write_dataset_collect_hyperslab('hess_kernel', hess_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('hess_rho_kernel', hess_rho_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('hess_kappa_kernel', hess_kappa_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab('hess_mu_kernel', hess_mu_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('hess_kernel', hess_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('hess_rho_kernel', hess_rho_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('hess_kappa_kernel', hess_kappa_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('hess_mu_kernel', hess_mu_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) ! close hdf5 call h5_close_file_p() diff --git a/src/specfem3D/write_movie_volume_hdf5.F90 b/src/specfem3D/write_movie_volume_hdf5.F90 index 7bade7083..e60f0242e 100644 --- a/src/specfem3D/write_movie_volume_hdf5.F90 +++ b/src/specfem3D/write_movie_volume_hdf5.F90 @@ -414,13 +414,17 @@ subroutine write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovi ! create elm_conn for movie ! for crust and mantle (strain and vector output) - if (output_sv) call get_conn_for_movie(elm_conn, sum(offset_poin_vol(0:myrank-1)), iNIT, nelems_3dmovie, npoints_3dmovie, NSPEC_CRUST_MANTLE, num_ibool_3dmovie, mask_ibool_3dmovie, ibool_crust_mantle) + if (output_sv) call get_conn_for_movie(elm_conn, sum(offset_poin_vol(0:myrank-1)), iNIT, nelems_3dmovie, & + npoints_3dmovie, NSPEC_CRUST_MANTLE, num_ibool_3dmovie, mask_ibool_3dmovie, ibool_crust_mantle) ! for crust and mantle (not strain or vector output) - if (output_cm) call get_conn_for_movie(elm_conn_cm, sum(offset_poin_vol_cm(0:myrank-1)), 1, nelems_3dmovie_cm, NGLOB_CRUST_MANTLE, NSPEC_CRUST_MANTLE, num_ibool_3dmovie_cm, mask_ibool_3dmovie_cm, ibool_crust_mantle) + if (output_cm) call get_conn_for_movie(elm_conn_cm, sum(offset_poin_vol_cm(0:myrank-1)), 1, nelems_3dmovie_cm, & + NGLOB_CRUST_MANTLE, NSPEC_CRUST_MANTLE, num_ibool_3dmovie_cm, mask_ibool_3dmovie_cm, ibool_crust_mantle) ! for outer core - if (output_oc) call get_conn_for_movie(elm_conn_oc, sum(offset_poin_vol_oc(0:myrank-1)), 1, nelems_3dmovie_oc, NGLOB_OUTER_CORE, NSPEC_OUTER_CORE, num_ibool_3dmovie_oc, mask_ibool_3dmovie_oc, ibool_outer_core) + if (output_oc) call get_conn_for_movie(elm_conn_oc, sum(offset_poin_vol_oc(0:myrank-1)), 1, nelems_3dmovie_oc, & + NGLOB_OUTER_CORE, NSPEC_OUTER_CORE, num_ibool_3dmovie_oc, mask_ibool_3dmovie_oc, ibool_outer_core) ! for inner core - if (output_ic) call get_conn_for_movie(elm_conn_ic, sum(offset_poin_vol_ic(0:myrank-1)), 1, nelems_3dmovie_ic, NGLOB_INNER_CORE, NSPEC_INNER_CORE, num_ibool_3dmovie_ic, mask_ibool_3dmovie_ic, ibool_inner_core) + if (output_ic) call get_conn_for_movie(elm_conn_ic, sum(offset_poin_vol_ic(0:myrank-1)), 1, nelems_3dmovie_ic, & + NGLOB_INNER_CORE, NSPEC_INNER_CORE, num_ibool_3dmovie_ic, mask_ibool_3dmovie_ic, ibool_inner_core) ! TODO ADD IOSERVER @@ -493,21 +497,24 @@ subroutine write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovi endif ! for crust and mantle (not strain or vector output) if (output_cm) then - call h5_write_dataset_collect_hyperslab_in_group('elm_conn_cm', elm_conn_cm, (/0, sum(offset_nspec_vol_cm_conn(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('elm_conn_cm', elm_conn_cm, & + (/0, sum(offset_nspec_vol_cm_conn(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('x_cm', store_val3D_x_cm, (/sum(offset_poin_vol_cm(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('y_cm', store_val3D_y_cm, (/sum(offset_poin_vol_cm(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('z_cm', store_val3D_z_cm, (/sum(offset_poin_vol_cm(0:myrank-1))/), H5_COL) endif ! for outer core if (output_oc) then - call h5_write_dataset_collect_hyperslab_in_group('elm_conn_oc', elm_conn_oc, (/0, sum(offset_nspec_vol_oc_conn(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('elm_conn_oc', elm_conn_oc, & + (/0, sum(offset_nspec_vol_oc_conn(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('x_oc', store_val3D_x_oc, (/sum(offset_poin_vol_oc(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('y_oc', store_val3D_y_oc, (/sum(offset_poin_vol_oc(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('z_oc', store_val3D_z_oc, (/sum(offset_poin_vol_oc(0:myrank-1))/), H5_COL) endif ! for inner core if (output_ic) then - call h5_write_dataset_collect_hyperslab_in_group('elm_conn_ic', elm_conn_ic, (/0, sum(offset_nspec_vol_ic_conn(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('elm_conn_ic', elm_conn_ic, & + (/0, sum(offset_nspec_vol_ic_conn(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('x_ic', store_val3D_x_ic, (/sum(offset_poin_vol_ic(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('y_ic', store_val3D_y_ic, (/sum(offset_poin_vol_ic(0:myrank-1))/), H5_COL) call h5_write_dataset_collect_hyperslab_in_group('z_ic', store_val3D_z_ic, (/sum(offset_poin_vol_ic(0:myrank-1))/), H5_COL) @@ -680,12 +687,18 @@ subroutine write_movie_volume_strains_hdf5(vnspec_eps_cm, & call h5_open_file_p_collect(file_name) call h5_open_group(group_name) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NN', store_val3d_NN, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EE', store_val3d_EE, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'ZZ', store_val3d_ZZ, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NE', store_val3d_NE, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NZ', store_val3d_NZ, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EZ', store_val3d_EZ, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NN', store_val3d_NN, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EE', store_val3d_EE, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'ZZ', store_val3d_ZZ, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NE', store_val3d_NE, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NZ', store_val3d_NZ, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EZ', store_val3d_EZ, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) call h5_close_group() call h5_close_file_p() @@ -806,10 +819,12 @@ subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_ call h5_open_group(group_name) if (MOVIE_OUTPUT_DIV) then - call write_array3dspec_as_1d_hdf5('reg1_div_displ', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), eps_trace_over_3_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('reg1_div_displ', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + eps_trace_over_3_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) if (NSPEC_OUTER_CORE_3DMOVIE > 1) then - call write_array3dspec_as_1d_hdf5('reg2_div_displ', offset_nspec_vol_oc(myrank), offset_poin_vol_oc(myrank), div_displ_outer_core, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) + call write_array3dspec_as_1d_hdf5('reg2_div_displ', offset_nspec_vol_oc(myrank), offset_poin_vol_oc(myrank), & + div_displ_outer_core, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) else allocate(div_s_outer_core(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array div_s_outer_core') @@ -825,25 +840,37 @@ subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_ enddo enddo enddo - call write_array3dspec_as_1d_hdf5('reg2_div_displ', offset_nspec_vol_oc(myrank), offset_poin_vol_oc(myrank), div_s_outer_core, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) + call write_array3dspec_as_1d_hdf5('reg2_div_displ', offset_nspec_vol_oc(myrank), offset_poin_vol_oc(myrank), & + div_s_outer_core, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) deallocate(div_s_outer_core) endif - call write_array3dspec_as_1d_hdf5('reg3_div_displ', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), eps_trace_over_3_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('reg3_div_displ', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + eps_trace_over_3_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) endif if (MOVIE_OUTPUT_CURL) then - call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xx', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), epsilondev_xx_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) - call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_yy', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), epsilondev_yy_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) - call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xy', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), epsilondev_xy_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) - call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xz', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), epsilondev_xz_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) - call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_yz', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), epsilondev_yz_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) - - call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xx', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), epsilondev_xx_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) - call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_yy', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), epsilondev_yy_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) - call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xy', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), epsilondev_xy_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) - call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xz', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), epsilondev_xz_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) - call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_yz', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), epsilondev_yz_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xx', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + epsilondev_xx_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_yy', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + epsilondev_yy_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xy', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + epsilondev_xy_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xz', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + epsilondev_xz_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_yz', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + epsilondev_yz_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xx', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + epsilondev_xx_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_yy', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + epsilondev_yy_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xy', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + epsilondev_xy_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xz', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + epsilondev_xz_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_yz', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + epsilondev_yz_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) endif if (MOVIE_OUTPUT_CURLNORM) then @@ -864,7 +891,8 @@ subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_ enddo enddo enddo - call write_array3dspec_as_1d_hdf5('reg1_epsdev_displ_norm', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('reg1_epsdev_displ_norm', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) deallocate(tmp_data) ! Frobenius norm @@ -883,7 +911,8 @@ subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_ enddo enddo enddo - call write_array3dspec_as_1d_hdf5('reg3_epsdev_displ_norm', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('reg3_epsdev_displ_norm', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) deallocate(tmp_data) endif @@ -1018,9 +1047,12 @@ subroutine write_movie_volume_vector_hdf5(npoints_3dmovie, & call h5_open_file_p_collect(file_name) call h5_open_group(group_name) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'N', store_val3d_N(1:npoints_3dmovie), (/offset_poin_vol(0:myrank-1)/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'E', store_val3d_E(1:npoints_3dmovie), (/offset_poin_vol(0:myrank-1)/), H5_COL) - call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'Z', store_val3d_Z(1:npoints_3dmovie), (/offset_poin_vol(0:myrank-1)/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'N', store_val3d_N(1:npoints_3dmovie), & + (/offset_poin_vol(0:myrank-1)/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'E', store_val3d_E(1:npoints_3dmovie), & + (/offset_poin_vol(0:myrank-1)/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'Z', store_val3d_Z(1:npoints_3dmovie), & + (/offset_poin_vol(0:myrank-1)/), H5_COL) call h5_close_group() call h5_close_file() @@ -1123,7 +1155,8 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core enddo enddo - call write_array3dspec_as_1d_hdf5('reg1_displ', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('reg1_displ', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), & + tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) deallocate(tmp_data) endif @@ -1147,7 +1180,8 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core enddo enddo - call write_array3dspec_as_1d_hdf5('reg2_displ', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) + call write_array3dspec_as_1d_hdf5('reg2_displ', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), & + tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) deallocate(tmp_data) endif @@ -1171,7 +1205,8 @@ subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core enddo enddo - call write_array3dspec_as_1d_hdf5('reg3_displ', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('reg3_displ', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), & + tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) deallocate(tmp_data) endif @@ -1275,7 +1310,8 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v enddo enddo - call write_array3dspec_as_1d_hdf5('reg1_veloc', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('reg1_veloc', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), & + tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) deallocate(tmp_data) endif @@ -1299,7 +1335,8 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v enddo enddo - call write_array3dspec_as_1d_hdf5('reg2_veloc', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) + call write_array3dspec_as_1d_hdf5('reg2_veloc', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), & + tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) deallocate(tmp_data) endif @@ -1323,7 +1360,8 @@ subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,v enddo enddo - call write_array3dspec_as_1d_hdf5('reg3_veloc', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('reg3_veloc', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), & + tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) deallocate(tmp_data) endif @@ -1430,7 +1468,8 @@ subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core enddo enddo - call write_array3dspec_as_1d_hdf5('reg1_accel', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('reg1_accel', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), & + tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) deallocate(tmp_data) endif @@ -1454,7 +1493,8 @@ subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core enddo enddo - call write_array3dspec_as_1d_hdf5('reg2_accel', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) + call write_array3dspec_as_1d_hdf5('reg2_accel', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), & + tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) deallocate(tmp_data) endif @@ -1478,7 +1518,8 @@ subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core enddo enddo - call write_array3dspec_as_1d_hdf5('reg3_accel', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('reg3_accel', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), & + tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) deallocate(tmp_data) endif @@ -1498,8 +1539,9 @@ end subroutine write_movie_volume_accelnorm_hdf5 ! #ifdef USE_HDF5 - subroutine get_conn_for_movie(elm_conn,offset,iNIT,nelems_3dmovie,npoints_3dmovie, nelems_in_this_region, & - num_ibool_3dmovie,mask_ibool_3dmovie,ibool_of_the_section) + subroutine get_conn_for_movie(elm_conn,offset,iNIT,nelems_3dmovie,npoints_3dmovie, & + nelems_in_this_region, & + num_ibool_3dmovie,mask_ibool_3dmovie,ibool_of_the_section) use specfem_par !use specfem_par_crustmantle, only: ibool_crust_mantle @@ -1508,14 +1550,13 @@ subroutine get_conn_for_movie(elm_conn,offset,iNIT,nelems_3dmovie,npoints_3dmovi implicit none - integer, dimension(9,nelems_3dmovie), intent(out) :: elm_conn integer, intent(in) :: offset ! node id offset (starting global element id of each proc) integer, intent(in) :: iNIT integer, intent(in) :: nelems_3dmovie, npoints_3dmovie, nelems_in_this_region integer, dimension(npoints_3dmovie),intent(in) :: num_ibool_3dmovie logical, dimension(npoints_3dmovie),intent(in) :: mask_ibool_3dmovie integer, dimension(NGLLX,NGLLY,NGLLZ,nelems_in_this_region),intent(in) :: ibool_of_the_section - + integer, dimension(9,nelems_3dmovie), intent(out) :: elm_conn ! local parameters integer :: ispec,ii integer,parameter :: cell_type = 9 @@ -1605,10 +1646,10 @@ subroutine elm2node_base(array_3dspec, array_1dmovie, nelms, npoints, ibool_of_t implicit none - real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nelms), intent(in) :: array_3dspec - real(kind=CUSTOM_REAL), dimension(npoints), intent(inout) :: array_1dmovie integer, intent(in) :: nelms, npoints integer, dimension(NGLLX,NGLLY,NGLLZ,nelms), intent(in) :: ibool_of_the_section + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nelms), intent(in) :: array_3dspec + real(kind=CUSTOM_REAL), dimension(npoints), intent(inout) :: array_1dmovie ! local parameters integer :: i,j,k,ispec,iglob @@ -1854,12 +1895,18 @@ subroutine write_xdmf_vol_hdf5(npoints_3dmovie, nelems_3dmovie, & endif ! movie_prefix/NN,EE,ZZ,NE,NZ,EZ - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_NN', trim(movie_prefix)//'NN', npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_EE', trim(movie_prefix)//'EE', npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_ZZ', trim(movie_prefix)//'ZZ', npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_NE', trim(movie_prefix)//'NE', npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_NZ', trim(movie_prefix)//'NZ', npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_EZ', trim(movie_prefix)//'EZ', npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_NN', trim(movie_prefix)//'NN', & + npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_EE', trim(movie_prefix)//'EE', & + npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_ZZ', trim(movie_prefix)//'ZZ', & + npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_NE', trim(movie_prefix)//'NE', & + npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_NZ', trim(movie_prefix)//'NZ', & + npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_EZ', trim(movie_prefix)//'EZ', & + npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) ! volume vector else if (MOVIE_VOLUME_TYPE == 5 .or. MOVIE_VOLUME_TYPE == 6) then @@ -1870,9 +1917,12 @@ subroutine write_xdmf_vol_hdf5(npoints_3dmovie, nelems_3dmovie, & endif ! movie_prefix/N,E,Z - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_N', trim(movie_prefix)//'N', npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_E', trim(movie_prefix)//'E', npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_Z', trim(movie_prefix)//'Z', npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_N', trim(movie_prefix)//'N', & + npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_E', trim(movie_prefix)//'E', & + npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, trim(movie_prefix)//'_Z', trim(movie_prefix)//'Z', & + npoints_vol_mov_all_proc, xdmf_vol, it_str, .true.) endif @@ -1898,7 +1948,8 @@ subroutine write_xdmf_vol_hdf5(npoints_3dmovie, nelems_3dmovie, & ! open xdmf file open(unit=xdmf_vol, file=trim(fname_xdmf_vol), recl=256) - call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_cm_conn, npoints_vol_mov_all_proc_cm, fname_h5_data_vol_xdmf, xdmf_vol, 2) + call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_cm_conn, npoints_vol_mov_all_proc_cm, & + fname_h5_data_vol_xdmf, xdmf_vol, 2) do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) @@ -1915,17 +1966,24 @@ subroutine write_xdmf_vol_hdf5(npoints_3dmovie, nelems_3dmovie, & ! volume divcurl (div) if (MOVIE_VOLUME_TYPE == 4 .and. MOVIE_OUTPUT_DIV) then ! reg1_div_displ - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg1_div_displ', 'reg1_div_displ', npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg1_div_displ', 'reg1_div_displ', & + npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element else if (MOVIE_VOLUME_TYPE == 4 .and. MOVIE_OUTPUT_CURL) then ! curst_mantle_epsdev_disple_xx,yy,xy,xz,yz - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'crust_mantle_epsdev_displ_xx', 'crust_mantle_epsdev_displ_xx', npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'crust_mantle_epsdev_displ_yy', 'crust_mantle_epsdev_displ_yy', npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'crust_mantle_epsdev_displ_xy', 'crust_mantle_epsdev_displ_xy', npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'crust_mantle_epsdev_displ_xz', 'crust_mantle_epsdev_displ_xz', npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'crust_mantle_epsdev_displ_yz', 'crust_mantle_epsdev_displ_yz', npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'crust_mantle_epsdev_displ_xx', 'crust_mantle_epsdev_displ_xx', & + npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'crust_mantle_epsdev_displ_yy', 'crust_mantle_epsdev_displ_yy', & + npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'crust_mantle_epsdev_displ_xy', 'crust_mantle_epsdev_displ_xy', & + npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'crust_mantle_epsdev_displ_xz', 'crust_mantle_epsdev_displ_xz', & + npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'crust_mantle_epsdev_displ_yz', 'crust_mantle_epsdev_displ_yz', & + npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element else if (MOVIE_VOLUME_TYPE == 4 .and. MOVIE_OUTPUT_CURLNORM) then ! reg1_epsdev_displ_norm - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg1_epsdev_displ_norm', 'reg1_epsdev_displ_norm', npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg1_epsdev_displ_norm', 'reg1_epsdev_displ_norm', & + npoints_vol_mov_all_proc_cm, xdmf_vol, it_str, .true.) ! value on element endif write(xdmf_vol,*) '' @@ -1950,7 +2008,8 @@ subroutine write_xdmf_vol_hdf5(npoints_3dmovie, nelems_3dmovie, & ! open xdmf file open(unit=xdmf_vol, file=trim(fname_xdmf_vol_oc), recl=256) - call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_oc_conn, npoints_vol_mov_all_proc_oc, fname_h5_data_vol_xdmf, xdmf_vol, 3) + call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_oc_conn, npoints_vol_mov_all_proc_oc, & + fname_h5_data_vol_xdmf, xdmf_vol, 3) do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) @@ -1966,20 +2025,24 @@ subroutine write_xdmf_vol_hdf5(npoints_3dmovie, nelems_3dmovie, & ! write headers for each dataset if (MOVIE_VOLUME_TYPE == 4 .and. MOVIE_OUTPUT_DIV) then ! reg2_div_displ - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg2_div_displ', 'reg2_div_displ', npoints_vol_mov_all_proc_oc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg2_div_displ', 'reg2_div_displ', & + npoints_vol_mov_all_proc_oc, xdmf_vol, it_str, .true.) else if (MOVIE_VOLUME_TYPE == 4 .and. MOVIE_OUTPUT_CURL) then ! no output else if (MOVIE_VOLUME_TYPE == 4 .and. MOVIE_OUTPUT_CURLNORM) then ! no output else if (MOVIE_VOLUME_TYPE == 7 .and. OUTPUT_OUTER_CORE) then ! reg2_displ - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg2_displ', 'reg2_displ', npoints_vol_mov_all_proc_oc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg2_displ', 'reg2_displ', & + npoints_vol_mov_all_proc_oc, xdmf_vol, it_str, .true.) else if (MOVIE_VOLUME_TYPE == 8 .and. OUTPUT_OUTER_CORE) then ! reg2_veloc - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg2_veloc', 'reg2_veloc', npoints_vol_mov_all_proc_oc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg2_veloc', 'reg2_veloc', & + npoints_vol_mov_all_proc_oc, xdmf_vol, it_str, .true.) else if (MOVIE_VOLUME_TYPE == 9 .and. OUTPUT_OUTER_CORE) then ! reg2_accel - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg2_accel', 'reg2_accel', npoints_vol_mov_all_proc_oc, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg2_accel', 'reg2_accel', & + npoints_vol_mov_all_proc_oc, xdmf_vol, it_str, .true.) endif write(xdmf_vol,*) '' @@ -2004,7 +2067,8 @@ subroutine write_xdmf_vol_hdf5(npoints_3dmovie, nelems_3dmovie, & ! open xdmf file open(unit=xdmf_vol, file=trim(fname_xdmf_vol_ic), recl=256) - call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_ic_conn, npoints_vol_mov_all_proc_ic, fname_h5_data_vol_xdmf, xdmf_vol, 4) + call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_ic_conn, npoints_vol_mov_all_proc_ic, & + fname_h5_data_vol_xdmf, xdmf_vol, 4) do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) @@ -2020,26 +2084,36 @@ subroutine write_xdmf_vol_hdf5(npoints_3dmovie, nelems_3dmovie, & ! write headers for each dataset if (MOVIE_VOLUME_TYPE == 4 .and. MOVIE_OUTPUT_DIV) then ! reg3_div_displ - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg3_div_displ', 'reg3_div_displ', npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg3_div_displ', 'reg3_div_displ', & + npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) else if (MOVIE_VOLUME_TYPE == 4 .and. MOVIE_OUTPUT_CURL) then ! inner_core_epsdev_disple_xx,yy,xy,xz,yz - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'inner_core_epsdev_displ_xx', 'inner_core_epsdev_displ_xx', npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'inner_core_epsdev_displ_yy', 'inner_core_epsdev_displ_yy', npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'inner_core_epsdev_displ_xy', 'inner_core_epsdev_displ_xy', npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'inner_core_epsdev_displ_xz', 'inner_core_epsdev_displ_xz', npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'inner_core_epsdev_displ_yz', 'inner_core_epsdev_displ_yz', npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'inner_core_epsdev_displ_xx', 'inner_core_epsdev_displ_xx', & + npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'inner_core_epsdev_displ_yy', 'inner_core_epsdev_displ_yy', & + npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'inner_core_epsdev_displ_xy', 'inner_core_epsdev_displ_xy', & + npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'inner_core_epsdev_displ_xz', 'inner_core_epsdev_displ_xz', & + npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'inner_core_epsdev_displ_yz', 'inner_core_epsdev_displ_yz', & + npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) else if (MOVIE_VOLUME_TYPE == 4 .and. MOVIE_OUTPUT_CURLNORM) then ! reg3_epsdev_displ_norm - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg3_epsdev_displ_norm', 'reg3_epsdev_displ_norm', npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg3_epsdev_displ_norm', 'reg3_epsdev_displ_norm', & + npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) else if (MOVIE_VOLUME_TYPE == 7 .and. OUTPUT_INNER_CORE) then ! reg3_displ - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg3_displ', 'reg3_displ', npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg3_displ', 'reg3_displ', & + npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) else if (MOVIE_VOLUME_TYPE == 8 .and. OUTPUT_INNER_CORE) then ! reg3_veloc - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg3_veloc', 'reg3_veloc', npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg3_veloc', 'reg3_veloc', & + npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) else if (MOVIE_VOLUME_TYPE == 9 .and. OUTPUT_INNER_CORE) then ! reg3_accel - call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg3_accel', 'reg3_accel', npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) + call write_xdmf_vol_hdf5_one_data(fname_h5_data_vol_xdmf, 'reg3_accel', 'reg3_accel', & + npoints_vol_mov_all_proc_ic, xdmf_vol, it_str, .true.) endif write(xdmf_vol,*) '' diff --git a/src/specfem3D/write_output_HDF5.F90 b/src/specfem3D/write_output_HDF5.F90 index 28a13d978..c7ac9311c 100644 --- a/src/specfem3D/write_output_HDF5.F90 +++ b/src/specfem3D/write_output_HDF5.F90 @@ -176,7 +176,7 @@ subroutine write_output_hdf5(seismogram_tmp_in, irec_local, irec, chn, iorientat use specfem_par, only: & myrank, seismo_current, nrec, & ROTATE_SEISMOGRAMS_RT, NTSTEP_BETWEEN_OUTPUT_SAMPLE, & - WRITE_SEISMOGRAMS_BY_MAIN, hdf5_seismo_fname, & + WRITE_SEISMOGRAMS_BY_MAIN, hdf5_seismo_fname use shared_parameters, only: & NSTEP, OUTPUT_SEISMOS_HDF5 use manager_hdf5