diff --git a/DATA/Par_file b/DATA/Par_file index a027f0d99..13766788a 100644 --- a/DATA/Par_file +++ b/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file b/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file index 6c33f13a1..050be325a 100644 --- a/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file +++ b/EXAMPLES/benchmarks/attenuation_benchmark_GJI_2002_versus_normal_modes/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .true. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file index 6159fbb94..ee29c66cb 100644 --- a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file +++ b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Bolivia_9s/SEMD/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file index 8a54d8510..791019fc3 100644 --- a/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file +++ b/EXAMPLES/benchmarks/prem_ani_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file b/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file index 79e12f62b..e3c98e1ae 100644 --- a/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file +++ b/EXAMPLES/benchmarks/prem_iso_noocean_Q_Cowling_Vanuatu/SEMD/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file b/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file index 5c096c033..2ce1ca46f 100644 --- a/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file +++ b/EXAMPLES/benchmarks/undo_attenuation_benchmark_GJI_2016/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file b/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file index 9a525ceb9..1c9a7a560 100644 --- a/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file +++ b/EXAMPLES/global_PREM_kernels/amplitude/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file b/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file index 9a525ceb9..1c9a7a560 100644 --- a/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file +++ b/EXAMPLES/global_PREM_kernels/traveltime/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file b/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file index 17e375429..b3f9d5f49 100644 --- a/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file +++ b/EXAMPLES/global_s362ani_shakemovie/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .true. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/global_small/DATA/Par_file b/EXAMPLES/global_small/DATA/Par_file index eb37a3af8..919273371 100644 --- a/EXAMPLES/global_small/DATA/Par_file +++ b/EXAMPLES/global_small/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .true. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/mars_global/DATA/Par_file b/EXAMPLES/mars_global/DATA/Par_file index 79e0e182e..b9a83e0f0 100644 --- a/EXAMPLES/mars_global/DATA/Par_file +++ b/EXAMPLES/mars_global/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/mars_regional/DATA/Par_file b/EXAMPLES/mars_regional/DATA/Par_file index 267ca2b5a..c1eb814e7 100644 --- a/EXAMPLES/mars_regional/DATA/Par_file +++ b/EXAMPLES/mars_regional/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/moon_global/DATA/Par_file b/EXAMPLES/moon_global/DATA/Par_file index 88d8b3b03..a7c4659b1 100644 --- a/EXAMPLES/moon_global/DATA/Par_file +++ b/EXAMPLES/moon_global/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation index 46f994bee..97fc77b6f 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation index beda34cab..760d1cc60 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_1_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation index 10a55fa7f..cea8dad9c 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation index a6dee9868..220b0f3b0 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_2_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation index 5619dd35d..6c6c874c9 100644 --- a/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/global_long/DATA/Par_file_NOISE_3_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation index c07198bb2..d9676321b 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation index b7e303498..c99291b79 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_1_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation index 6278a1544..3c05d20a3 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation index 9165ae7f0..f2068990f 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_2_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation index 6b2418ba0..06905755a 100644 --- a/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/global_short/DATA/Par_file_NOISE_3_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation index ad5e47c29..ad51e4ce6 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation index 58664b314..34b396ebc 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_1_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation index 0b883f0d9..5812fe342 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation index 1df56738b..9805ffe4e 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_2_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation index bd75b9a9b..e1e621e2f 100644 --- a/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/regional/DATA/Par_file_NOISE_3_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation index 77f765ca7..5893051ff 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation index 30d7359da..dc1227362 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_1_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation index 72a63808e..80ab3873a 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation index ad3b136de..e8d094d7f 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_2_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation index 8491244e6..007879c7a 100644 --- a/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/test_global/DATA/Par_file_NOISE_3_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation index f03efb743..df97e9387 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation index 8404b4041..7c6622653 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_1_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation index a21255839..f512cf381 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_attenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation index 3e1d8d887..c7921649d 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_2_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation index 8cabf7fcb..4f58bd838 100644 --- a/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation +++ b/EXAMPLES/noise_examples/test_regional/DATA/Par_file_NOISE_3_noattenuation @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/point_force/DATA/Par_file b/EXAMPLES/point_force/DATA/Par_file index ad8da35bd..7e950ccd8 100644 --- a/EXAMPLES/point_force/DATA/Par_file +++ b/EXAMPLES/point_force/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .true. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Berkeley/DATA/Par_file b/EXAMPLES/regional_Berkeley/DATA/Par_file index 0e8ee669a..7e212475b 100644 --- a/EXAMPLES/regional_Berkeley/DATA/Par_file +++ b/EXAMPLES/regional_Berkeley/DATA/Par_file @@ -21,7 +21,7 @@ GAMMA_ROTATION_AZIMUTH = 0.d0 # number of elements at the surface along the two sides of the first chunk # (must be multiple of 16 and 8 * multiple of NPROC below) -NEX_XI = 48 +NEX_XI = 48 NEX_ETA = 48 # number of MPI processors along the two sides of the first chunk @@ -269,17 +269,6 @@ USE_MONOCHROMATIC_CMT_SOURCE = .false. # print source time function PRINT_SOURCE_TIME_FUNCTION = .true. -## Berkeley source time function -STF_IS_UCB_HEAVISIDE = .true. -# UCB source frequency content (i.e., heaviside function) -SOURCE_T1 = 500.d0 -SOURCE_T2 = 400.d0 -SOURCE_T3 = 50.d0 -SOURCE_T4 = 40.d0 -# UCB source time shift -TAU = 500.d0 - - #----------------------------------------------------------- # # Seismograms @@ -308,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -441,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_EMC_model/DATA/Par_file b/EXAMPLES/regional_EMC_model/DATA/Par_file index 3655ac2de..1db17fb79 100644 --- a/EXAMPLES/regional_EMC_model/DATA/Par_file +++ b/EXAMPLES/regional_EMC_model/DATA/Par_file @@ -317,6 +317,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -450,3 +451,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM b/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM index bbf05aeeb..320ff92a7 100644 --- a/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM +++ b/EXAMPLES/regional_EMC_model/DATA/Par_file.SCEC-CVM @@ -317,6 +317,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -450,3 +451,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Greece_noise_small/DATA/Par_file b/EXAMPLES/regional_Greece_noise_small/DATA/Par_file index 40f5b1803..09d31fabd 100644 --- a/EXAMPLES/regional_Greece_noise_small/DATA/Par_file +++ b/EXAMPLES/regional_Greece_noise_small/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Greece_small/DATA/Par_file b/EXAMPLES/regional_Greece_small/DATA/Par_file index 01b2852b3..d3039c250 100644 --- a/EXAMPLES/regional_Greece_small/DATA/Par_file +++ b/EXAMPLES/regional_Greece_small/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file b/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file index 04a9b4e6f..26c812d7f 100644 --- a/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file +++ b/EXAMPLES/regional_Greece_small_LDDRK/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file index 0b49f1757..af09d75f7 100644 --- a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file +++ b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 index 8cc029108..e5bacdcb3 100644 --- a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 +++ b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step1 @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 index 0b49f1757..af09d75f7 100644 --- a/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 +++ b/EXAMPLES/regional_Mexico_noise_non_uniform/DATA/Par_file_step2 @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_MiddleEast/DATA/Par_file b/EXAMPLES/regional_MiddleEast/DATA/Par_file index defd61a91..46fcd43b6 100644 --- a/EXAMPLES/regional_MiddleEast/DATA/Par_file +++ b/EXAMPLES/regional_MiddleEast/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_s40rts/DATA/Par_file b/EXAMPLES/regional_s40rts/DATA/Par_file index b1d1f6851..b0602efc4 100644 --- a/EXAMPLES/regional_s40rts/DATA/Par_file +++ b/EXAMPLES/regional_s40rts/DATA/Par_file @@ -297,12 +297,13 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. # decide if main process writes all the seismograms or if all processes do it in parallel -WRITE_SEISMOGRAMS_BY_MAIN = .false. +WRITE_SEISMOGRAMS_BY_MAIN = .true. # save all seismograms in one large combined file instead of one file per seismogram # to avoid overloading shared non-local file systems such as LUSTRE or GPFS for instance @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .true. + diff --git a/EXAMPLES/regional_sgloberani/DATA/Par_file b/EXAMPLES/regional_sgloberani/DATA/Par_file index 3e392f0c1..db2676361 100644 --- a/EXAMPLES/regional_sgloberani/DATA/Par_file +++ b/EXAMPLES/regional_sgloberani/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .true. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_simultaneous_runs/DATA/Par_file b/EXAMPLES/regional_simultaneous_runs/DATA/Par_file index ea90c3dfb..249386088 100644 --- a/EXAMPLES/regional_simultaneous_runs/DATA/Par_file +++ b/EXAMPLES/regional_simultaneous_runs/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/regional_small_adjoint/DATA/Par_file b/EXAMPLES/regional_small_adjoint/DATA/Par_file index bd40baea2..0e0ecb4bc 100644 --- a/EXAMPLES/regional_small_adjoint/DATA/Par_file +++ b/EXAMPLES/regional_small_adjoint/DATA/Par_file @@ -204,8 +204,8 @@ RATIO_BY_WHICH_TO_INCREASE_IT = 1.5d0 # save AVS or OpenDX movies #MOVIE_COARSE saves movie only at corners of elements (SURFACE OR VOLUME) #MOVIE_COARSE does not work with create_movie_AVS_DX -MOVIE_SURFACE = .false. -MOVIE_VOLUME = .false. +MOVIE_SURFACE = .true. +MOVIE_VOLUME = .true. MOVIE_COARSE = .false. NTSTEP_BETWEEN_FRAMES = 200 HDUR_MOVIE = 0.1d0 @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .true. + diff --git a/EXAMPLES/regular_kernel/DATA/Par_file b/EXAMPLES/regular_kernel/DATA/Par_file index db42b20bf..22cde7c98 100644 --- a/EXAMPLES/regular_kernel/DATA/Par_file +++ b/EXAMPLES/regular_kernel/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .true. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file b/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file index c4eee50e8..c72472240 100644 --- a/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file +++ b/EXAMPLES/small_benchmark_run_to_test_more_complex_Earth/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file b/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file index c74f29023..e9c28039f 100644 --- a/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file +++ b/EXAMPLES/small_benchmark_run_to_test_very_simple_Earth/DATA/Par_file @@ -297,6 +297,7 @@ OUTPUT_SEISMOS_SAC_ALPHANUM = .false. OUTPUT_SEISMOS_SAC_BINARY = .false. OUTPUT_SEISMOS_ASDF = .false. OUTPUT_SEISMOS_3D_ARRAY = .false. +OUTPUT_SEISMOS_HDF5 = .false. # rotate seismograms to Radial-Transverse-Z or use default North-East-Z reference frame ROTATE_SEISMOGRAMS_RT = .false. @@ -430,3 +431,7 @@ ADIOS_FOR_KERNELS = .true. ADIOS_FOR_MODELS = .true. ADIOS_FOR_UNDO_ATTENUATION = .true. +# HDF5 Database I/O +# (note the flags for HDF5 and ADIOS are mutually exclusive, only one can be used) +HDF5_ENABLED = .false. + diff --git a/Makefile.in b/Makefile.in index 40e3a3cd2..cf1b3e7ee 100644 --- a/Makefile.in +++ b/Makefile.in @@ -496,6 +496,20 @@ endif @COND_PETSC_TRUE@LDFLAGS += @PETSC_LDFLAGS@ @COND_PETSC_TRUE@MPILIBS += @HIP_LDFLAGS@ @PETSC_LIBS@ +####################################### +#### +#### HDF5 (parallel) +#### with configure: ./configure --with-hdf5 HDF5_LIBS=.. HDF5_FCFLAGS=.. HDF5_INC=.. +#### +####################################### + +@COND_HDF5_TRUE@HDF5 = yes +@COND_HDF5_FALSE@HDF5 = no + +# adds compiler flag +@COND_HDF5_TRUE@FCFLAGS += @HDF5_FCFLAGS@ @HDF5_INCLUDES@ $(FC_DEFINE)USE_HDF5 +@COND_HDF5_TRUE@LDFLAGS += @HDF5_LIBS@ -lhdf5_fortran -lhdf5hl_fortran # -lhdf5_hl -lhdf5 -lstdc++ + ####################################### ## static compilation ####################################### diff --git a/configure b/configure index c8c926281..ec9134a1c 100755 --- a/configure +++ b/configure @@ -645,6 +645,10 @@ LIBOBJS GIT_DATE_VERSION GIT_COMMIT_VERSION GIT_PACKAGE_VERSION +HDF5_INCLUDES +HDF5_LIBS +HDF5_FCFLAGS +HDF5_INC PETSC_LIBS PETSC_LDFLAGS PETSC_LIB @@ -740,6 +744,8 @@ ac_ct_FC LDFLAGS FCFLAGS FC +COND_HDF5_FALSE +COND_HDF5_TRUE COND_PETSC_FALSE COND_PETSC_TRUE COND_NETCDF_FALSE @@ -899,6 +905,7 @@ with_emc with_netcdf enable_netcdf with_petsc +with_hdf5 with_vtk with_vtk_suffix with_vtk_version @@ -954,7 +961,10 @@ EMC_FCFLAGS EMC_LIBS PETSC_INC PETSC_FCFLAGS -PETSC_LIB' +PETSC_LIB +HDF5_INC +HDF5_FCFLAGS +HDF5_LIBS' # Initialize some variables set by options. @@ -1613,6 +1623,7 @@ Optional Packages: --with-netcdf build NetCDF enabled version (IRIS EMC & CEM models) [default=no] --with-petsc build PETSc enabled version [default=no] + --with-hdf5 build HDF5 enabled version [default=no] --with-vtk The prefix where VTK is installed [default=/usr] --with-vtk-suffix Suffix to append to VTK's include directory, e.g., for vtk-5.2/ the suffix is "-5.2" @@ -1681,6 +1692,10 @@ Some influential environment variables: PETSC_FCFLAGS Fortran compiler flags (PETSC specific flags) PETSC_LIB extra library directory for linking programs with PETSC + HDF5_INC HDF5 include directory + HDF5_FCFLAGS + Fortran compiler flags + HDF5_LIBS extra libraries for linking programs Use these variables to override the choices made by 'configure' or to help it to find libraries and programs with nonstandard names/locations. @@ -4092,6 +4107,28 @@ else fi +### +### HDF5 +### + + +# Check whether --with-hdf5 was given. +if test ${with_hdf5+y} +then : + withval=$with_hdf5; want_hdf5="$withval" +else case e in #( + e) want_hdf5=no ;; +esac +fi + + if test x$"$want_hdf5" != xno; then + COND_HDF5_TRUE= + COND_HDF5_FALSE='#' +else + COND_HDF5_TRUE='#' + COND_HDF5_FALSE= +fi + ############################################################ @@ -11881,6 +11918,71 @@ ac_compiler_gnu=$ac_cv_fc_compiler_gnu +fi + +## +## HDF5 +## + +if test x"$want_hdf5" != xno +then : + + printf "%s\n" "## ---- ## +## HDF5 ## +## ---- ##" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: HDF5 is enabled" >&5 +printf "%s\n" "$as_me: HDF5 is enabled" >&6;} + + + + + + # HDF5 checking for Fortran + ac_ext=${ac_fc_srcext-f} +ac_compile='$FC -c $FCFLAGS $ac_fcflags_srcext conftest.$ac_ext >&5' +ac_link='$FC -o conftest$ac_exeext $FCFLAGS $LDFLAGS $ac_fcflags_srcext conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_fc_compiler_gnu + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HDF5 modules" >&5 +printf %s "checking for HDF5 modules... " >&6; } + FCFLAGS="$HDF5_FCFLAGS $FCFLAGS" + # adds include directory + if test "x$HDF5_INC" != "x"; then + HDF5_INCLUDES="-I$HDF5_INC" + FCFLAGS+=" $HDF5_INCLUDES" + fi + # compilation test + cat > conftest.$ac_ext <<_ACEOF + + program main + + use hdf5 + + end + +_ACEOF +if ac_fn_fc_try_compile "$LINENO" +then : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +else case e in #( + e) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + as_fn_error $? "HDF5 module not found; is HDF5 built with Fortran support for this compiler?" "$LINENO" 5 + ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ac_ext=${ac_fc_srcext-f} +ac_compile='$FC -c $FCFLAGS $ac_fcflags_srcext conftest.$ac_ext >&5' +ac_link='$FC -o conftest$ac_exeext $FCFLAGS $LDFLAGS $ac_fcflags_srcext conftest.$ac_ext $LIBS >&5' +ac_compiler_gnu=$ac_cv_fc_compiler_gnu + + + fi ## @@ -12275,6 +12377,10 @@ if test -z "${COND_PETSC_TRUE}" && test -z "${COND_PETSC_FALSE}"; then as_fn_error $? "conditional \"COND_PETSC\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi +if test -z "${COND_HDF5_TRUE}" && test -z "${COND_HDF5_FALSE}"; then + as_fn_error $? "conditional \"COND_HDF5\" was never defined. +Usually this means the macro was only invoked conditionally." "$LINENO" 5 +fi case $FC_MODINC in #( *\ ) FC_MODINC=$FC_MODINC'${ac_empty}' ;; esac diff --git a/configure.ac b/configure.ac index 49cc402d8..392bd4362 100644 --- a/configure.ac +++ b/configure.ac @@ -8,8 +8,8 @@ dnl You must have recent versions of Autoconf and Automake installed. ############################################################ -AC_PREREQ(2.61) -AC_INIT([Specfem3D_Globe], m4_normalize(m4_include([VERSION])), [see the wiki], [Specfem3DGlobe]) +AC_PREREQ([2.72]) +AC_INIT([Specfem3D_Globe],[m4_normalize(m4_include(VERSION))],[see the wiki],[Specfem3DGlobe]) AC_CONFIG_SRCDIR([src/specfem3D/specfem3D.F90]) AC_CONFIG_HEADERS([setup/config.h]) @@ -326,6 +326,16 @@ AC_ARG_WITH([petsc], [want_petsc=no]) AM_CONDITIONAL([COND_PETSC], [test x$"$want_petsc" != xno]) +### +### HDF5 +### + +AC_ARG_WITH([hdf5], + [AS_HELP_STRING([--with-hdf5], + [build HDF5 enabled version @<:@default=no@:>@])], + [want_hdf5="$withval"], + [want_hdf5=no]) +AM_CONDITIONAL([COND_HDF5], [test x$"$want_hdf5" != xno]) ############################################################ @@ -944,6 +954,42 @@ AS_IF([test x"$want_petsc" != xno], [ AC_SUBST([PETSC_LIBS]) ]) +## +## HDF5 +## + +AS_IF([test x"$want_hdf5" != xno], [ + AS_BOX([HDF5]) + AC_MSG_NOTICE([HDF5 is enabled]) + + AC_ARG_VAR(HDF5_INC, [HDF5 include directory]) + AC_ARG_VAR(HDF5_FCFLAGS, [Fortran compiler flags]) + AC_ARG_VAR(HDF5_LIBS, [extra libraries for linking programs]) + + # HDF5 checking for Fortran + AC_LANG_PUSH(Fortran) + AC_MSG_CHECKING([for HDF5 modules]) + FCFLAGS="$HDF5_FCFLAGS $FCFLAGS" + # adds include directory + if test "x$HDF5_INC" != "x"; then + HDF5_INCLUDES="-I$HDF5_INC" + FCFLAGS+=" $HDF5_INCLUDES" + fi + # compilation test + AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM([], [[ + use hdf5 + ]]) + ], [ + AC_MSG_RESULT(yes) + ], [ + AC_MSG_RESULT(no) + AC_MSG_ERROR([HDF5 module not found; is HDF5 built with Fortran support for this compiler?]) + ]) + AC_LANG_POP([Fortran]) + AC_SUBST([HDF5_INCLUDES]) +]) + ## ## Git version info ## diff --git a/src/gindex3D/rules.mk b/src/gindex3D/rules.mk index 778ae6f35..a5607dfd9 100644 --- a/src/gindex3D/rules.mk +++ b/src/gindex3D/rules.mk @@ -48,6 +48,7 @@ gindex3D_SHARED_OBJECTS = \ $O/bcast_mesh_databases.solverstatic.o \ $O/locate_regular_points.solverstatic.o \ $O/read_arrays_solver.solverstatic.o \ + $O/read_arrays_solver_hdf5.solverstatic.o \ $O/read_mesh_parameters.solverstatic.o \ $O/read_mesh_databases.solverstatic.o \ $O/specfem3D_par.solverstatic_module.o \ @@ -68,6 +69,7 @@ gindex3D_SHARED_OBJECTS += \ $O/get_model_parameters.shared.o \ $O/get_timestep_and_layers.shared.o \ $O/hex_nodes.shared.o \ + $O/hdf5_manager.shared_hdf5_module.o \ $O/memory_eval.shared.o \ $O/parallel.sharedmpi.o \ $O/param_reader.cc.o \ diff --git a/src/meshfem3D/create_regions_mesh.F90 b/src/meshfem3D/create_regions_mesh.F90 index 39388220e..ac94b8bff 100644 --- a/src/meshfem3D/create_regions_mesh.F90 +++ b/src/meshfem3D/create_regions_mesh.F90 @@ -51,7 +51,8 @@ subroutine create_regions_mesh(npointot, & SAVE_BOUNDARY_MESH,SAVE_MESHFILES_AVS_DX_FORMAT use shared_parameters, only: & - R_CENTRAL_CUBE,RICB,RCMB,RINF + R_CENTRAL_CUBE,RICB,RCMB,RINF, & + HDF5_ENABLED use meshfem_par, only: & myrank,nspec,nglob,iregion_code, & @@ -460,6 +461,9 @@ subroutine create_regions_mesh(npointot, & call save_arrays_solver_adios(idoubling,ibool,xstore,ystore,zstore, & NSPEC2DMAX_XMIN_XMAX, NSPEC2DMAX_YMIN_YMAX, & NSPEC2D_TOP,NSPEC2D_BOTTOM) + else if (HDF5_ENABLED) then + call save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & + NSPEC2D_TOP,NSPEC2D_BOTTOM) else call save_arrays_solver(idoubling,ibool,xstore,ystore,zstore, & NSPEC2D_TOP,NSPEC2D_BOTTOM) @@ -480,6 +484,8 @@ subroutine create_regions_mesh(npointot, & ! saves boundary file if (ADIOS_FOR_ARRAYS_SOLVER) then call save_arrays_boundary_adios() + else if (HDF5_ENABLED) then + call save_arrays_boundary_hdf5() else call save_arrays_boundary() endif @@ -501,6 +507,8 @@ subroutine create_regions_mesh(npointot, & if (ADIOS_FOR_SOLVER_MESHFILES) then ! adios file output call save_model_meshfiles_adios() + else if (HDF5_ENABLED) then + call save_model_meshfiles_hdf5() else ! outputs model files in binary format call save_model_meshfiles() diff --git a/src/meshfem3D/get_absorb.f90 b/src/meshfem3D/get_absorb.f90 index 28d6e3672..177f60c3f 100644 --- a/src/meshfem3D/get_absorb.f90 +++ b/src/meshfem3D/get_absorb.f90 @@ -34,6 +34,8 @@ subroutine get_absorb(prname,iregion,iboun, & use constants use meshfem_par, only: ADIOS_FOR_ARRAYS_SOLVER,nspec + use shared_parameters, only: HDF5_ENABLED + implicit none integer,intent(in) :: iregion @@ -146,6 +148,9 @@ subroutine get_absorb(prname,iregion,iboun, & call get_absorb_adios(iregion, & nimin, nimax, njmin, njmax, nkmin_xi, nkmin_eta, & NSPEC2DMAX_XMIN_XMAX,NSPEC2DMAX_YMIN_YMAX) + else if (HDF5_ENABLED) then + ! no HDF5 support yet + call exit_MPI(myrank,'HDF5 not supported yet for old format Stacey boundary condition output') else open(unit=IOUT,file=prname(1:len_trim(prname))//'stacey.old_format.bin', & status='unknown',form='unformatted',action='write',iostat=ier) @@ -196,8 +201,8 @@ subroutine get_absorb_create_Stacey_boundary_arrays(iregion,NSPEC2D_BOTTOM) abs_boundary_ijk,abs_boundary_normal,abs_boundary_jacobian2Dw ! input parameters - use shared_parameters, only: REGIONAL_MESH_CUTOFF,ADIOS_FOR_ARRAYS_SOLVER - + use shared_parameters, only: REGIONAL_MESH_CUTOFF,ADIOS_FOR_ARRAYS_SOLVER, & + HDF5_ENABLED ! debug use meshfem_par, only: nspec,nglob,ibool,xstore_glob,ystore_glob,zstore_glob @@ -511,6 +516,10 @@ subroutine get_absorb_create_Stacey_boundary_arrays(iregion,NSPEC2D_BOTTOM) call get_absorb_stacey_boundary_adios(iregion, num_abs_boundary_faces, & abs_boundary_ispec,abs_boundary_npoin, & abs_boundary_ijk,abs_boundary_normal,abs_boundary_jacobian2Dw) + else if (HDF5_ENABLED) then + call get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & + abs_boundary_ispec,abs_boundary_npoin, & + abs_boundary_ijk,abs_boundary_normal,abs_boundary_jacobian2Dw) else ! binary format open(unit=IOUT,file=prname(1:len_trim(prname))//'stacey.bin', & diff --git a/src/meshfem3D/initialize_mesher.f90 b/src/meshfem3D/initialize_mesher.f90 index 264528242..c84f775a4 100644 --- a/src/meshfem3D/initialize_mesher.f90 +++ b/src/meshfem3D/initialize_mesher.f90 @@ -119,6 +119,7 @@ subroutine im_initialize_system() use meshfem_models_par use manager_adios + use manager_hdf5 implicit none @@ -135,6 +136,11 @@ subroutine im_initialize_system() call initialize_adios() endif + ! HDF5 + if (HDF5_ENABLED) then + call h5_initialize() + endif + ! gravity integrals if (GRAVITY_INTEGRALS) then call gravity_initialize_integrals() diff --git a/src/meshfem3D/rules.mk b/src/meshfem3D/rules.mk index 230a267bf..a95a7f193 100644 --- a/src/meshfem3D/rules.mk +++ b/src/meshfem3D/rules.mk @@ -122,7 +122,9 @@ meshfem3D_MESHER_OBJECTS = \ $O/model_spiral.check.o \ $O/moho_stretching.check.o \ $O/save_arrays_solver.check.o \ + $O/save_arrays_solver_hdf5.check.o \ $O/save_model_meshfiles.check.o \ + $O/save_model_meshfiles_hdf5.check.o \ $O/setup_color_perm.check.o \ $O/setup_counters.check.o \ $O/setup_inner_outer.check.o \ @@ -215,6 +217,7 @@ meshfem3D_SHARED_OBJECTS = \ $O/get_model_parameters.shared.o \ $O/get_timestep_and_layers.shared.o \ $O/gll_library.shared.o \ + $O/hdf5_manager.shared_hdf5_module.o \ $O/heap_sort.shared.o \ $O/hex_nodes.shared.o \ $O/init_openmp.shared.o \ @@ -405,3 +408,4 @@ $O/%.check_adios.o: $S/%.F90 $O/shared_par.shared_module.o $O/meshfem3D_par.chec $O/%.checknetcdf.o: $S/%.f90 $O/shared_par.shared_module.o $O/meshfem3D_par.check_module.o ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(NETCDF_INCLUDE) -c -o $@ $< + diff --git a/src/meshfem3D/save_arrays_solver.f90 b/src/meshfem3D/save_arrays_solver.f90 index c790dcc45..a311db5f5 100644 --- a/src/meshfem3D/save_arrays_solver.f90 +++ b/src/meshfem3D/save_arrays_solver.f90 @@ -334,6 +334,8 @@ subroutine save_arrays_solver_MPI() IREGION_TRINFINITE,IREGION_INFINITE, & ADIOS_FOR_MPI_ARRAYS + use shared_parameters, only: HDF5_ENABLED + ! use MPI_interfaces_par use MPI_crust_mantle_par @@ -357,6 +359,15 @@ subroutine save_arrays_solver_MPI() num_phase_ispec_crust_mantle,phase_ispec_inner_crust_mantle, & num_colors_outer_crust_mantle,num_colors_inner_crust_mantle, & num_elem_colors_crust_mantle) + else if (HDF5_ENABLED) then + call save_MPI_arrays_hdf5(IREGION_CRUST_MANTLE,LOCAL_PATH, & + num_interfaces_crust_mantle,max_nibool_interfaces_cm, & + my_neighbors_crust_mantle,nibool_interfaces_crust_mantle, & + ibool_interfaces_crust_mantle, & + nspec_inner_crust_mantle,nspec_outer_crust_mantle, & + num_phase_ispec_crust_mantle,phase_ispec_inner_crust_mantle, & + num_colors_outer_crust_mantle,num_colors_inner_crust_mantle, & + num_elem_colors_crust_mantle) else call save_MPI_arrays(IREGION_CRUST_MANTLE,LOCAL_PATH, & num_interfaces_crust_mantle,max_nibool_interfaces_cm, & @@ -379,6 +390,15 @@ subroutine save_arrays_solver_MPI() num_phase_ispec_outer_core,phase_ispec_inner_outer_core, & num_colors_outer_outer_core,num_colors_inner_outer_core, & num_elem_colors_outer_core) + else if (HDF5_ENABLED) then + call save_MPI_arrays_hdf5(IREGION_OUTER_CORE,LOCAL_PATH, & + num_interfaces_outer_core,max_nibool_interfaces_oc, & + my_neighbors_outer_core,nibool_interfaces_outer_core, & + ibool_interfaces_outer_core, & + nspec_inner_outer_core,nspec_outer_outer_core, & + num_phase_ispec_outer_core,phase_ispec_inner_outer_core, & + num_colors_outer_outer_core,num_colors_inner_outer_core, & + num_elem_colors_outer_core) else call save_MPI_arrays(IREGION_OUTER_CORE,LOCAL_PATH, & num_interfaces_outer_core,max_nibool_interfaces_oc, & @@ -401,6 +421,16 @@ subroutine save_arrays_solver_MPI() num_phase_ispec_inner_core,phase_ispec_inner_inner_core, & num_colors_outer_inner_core,num_colors_inner_inner_core, & num_elem_colors_inner_core) + else if (HDF5_ENABLED) then + call save_MPI_arrays_hdf5(IREGION_INNER_CORE,LOCAL_PATH, & + num_interfaces_inner_core,max_nibool_interfaces_ic, & + my_neighbors_inner_core,nibool_interfaces_inner_core, & + ibool_interfaces_inner_core, & + nspec_inner_inner_core,nspec_outer_inner_core, & + num_phase_ispec_inner_core,phase_ispec_inner_inner_core, & + num_colors_outer_inner_core,num_colors_inner_inner_core, & + num_elem_colors_inner_core) + else call save_MPI_arrays(IREGION_INNER_CORE,LOCAL_PATH, & num_interfaces_inner_core,max_nibool_interfaces_ic, & @@ -423,6 +453,15 @@ subroutine save_arrays_solver_MPI() num_phase_ispec_trinfinite,phase_ispec_inner_trinfinite, & num_colors_outer_trinfinite,num_colors_inner_trinfinite, & num_elem_colors_trinfinite) + else if (HDF5_ENABLED) then + call save_MPI_arrays_hdf5(IREGION_TRINFINITE,LOCAL_PATH, & + num_interfaces_trinfinite,max_nibool_interfaces_trinfinite, & + my_neighbors_trinfinite,nibool_interfaces_trinfinite, & + ibool_interfaces_trinfinite, & + nspec_inner_trinfinite,nspec_outer_trinfinite, & + num_phase_ispec_trinfinite,phase_ispec_inner_trinfinite, & + num_colors_outer_trinfinite,num_colors_inner_trinfinite, & + num_elem_colors_trinfinite) else call save_MPI_arrays(IREGION_TRINFINITE,LOCAL_PATH, & num_interfaces_trinfinite,max_nibool_interfaces_trinfinite, & @@ -445,6 +484,15 @@ subroutine save_arrays_solver_MPI() num_phase_ispec_infinite,phase_ispec_inner_infinite, & num_colors_outer_infinite,num_colors_inner_infinite, & num_elem_colors_infinite) + else if (HDF5_ENABLED) then + call save_MPI_arrays_hdf5(IREGION_INFINITE,LOCAL_PATH, & + num_interfaces_infinite,max_nibool_interfaces_infinite, & + my_neighbors_infinite,nibool_interfaces_infinite, & + ibool_interfaces_infinite, & + nspec_inner_infinite,nspec_outer_infinite, & + num_phase_ispec_infinite,phase_ispec_inner_infinite, & + num_colors_outer_infinite,num_colors_inner_infinite, & + num_elem_colors_infinite) else call save_MPI_arrays(IREGION_INFINITE,LOCAL_PATH, & num_interfaces_infinite,max_nibool_interfaces_infinite, & diff --git a/src/meshfem3D/save_arrays_solver_hdf5.F90 b/src/meshfem3D/save_arrays_solver_hdf5.F90 new file mode 100644 index 000000000..d7a6d9faf --- /dev/null +++ b/src/meshfem3D/save_arrays_solver_hdf5.F90 @@ -0,0 +1,1153 @@ + subroutine save_arrays_solver_hdf5(idoubling,ibool,xstore,ystore,zstore, & + NSPEC2D_TOP,NSPEC2D_BOTTOM) + + use constants + + use meshfem_par, only: & + nspec + +#ifdef USE_HDF5 + use shared_parameters, only: LOCAL_PATH, ATT_F_C_SOURCE, H5_COL + use meshfem_models_par, only: & + OCEANS,TRANSVERSE_ISOTROPY,ANISOTROPIC_3D_MANTLE, & + ANISOTROPIC_INNER_CORE,ATTENUATION + use meshfem_par, only: & + nglob, iregion_code, & + NCHUNKS,ABSORBING_CONDITIONS, & + ROTATION,EXACT_MASS_MATRIX_FOR_ROTATION, & + NPROCTOT, & + ATT1,ATT2,ATT3 + use regions_mesh_par2, only: & + xixstore,xiystore,xizstore,etaxstore,etaystore,etazstore, & + gammaxstore,gammaystore,gammazstore, & + rhostore,kappavstore,kappahstore,muvstore,muhstore,eta_anisostore, & + c11store,c12store,c13store,c14store,c15store,c16store,c22store, & + c23store,c24store,c25store,c26store,c33store,c34store,c35store, & + c36store,c44store,c45store,c46store,c55store,c56store,c66store, & + mu0store, & + rmassx,rmassy,rmassz,rmass_ocean_load, & + b_rmassx,b_rmassy, & + ibelm_xmin,ibelm_xmax,ibelm_ymin,ibelm_ymax,ibelm_bottom,ibelm_top, & + normal_xmin,normal_xmax,normal_ymin,normal_ymax,normal_bottom,normal_top, & + jacobian2D_xmin,jacobian2D_xmax,jacobian2D_ymin,jacobian2D_ymax, & + jacobian2D_bottom,jacobian2D_top, & + rho_vp,rho_vs, & + nspec2D_xmin,nspec2D_xmax,nspec2D_ymin,nspec2D_ymax, & + ispec_is_tiso, & + tau_s_store,tau_e_store,Qmu_store, & + nglob_oceans, nglob_xy + + + + use manager_hdf5 +#endif + + implicit none + + ! doubling mesh flag + integer,dimension(nspec),intent(in) :: idoubling + integer,dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(in) :: ibool + + ! arrays with the mesh in double precision + double precision,dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(in) :: xstore,ystore,zstore + + ! boundary parameters locator + integer,intent(in) :: NSPEC2D_TOP,NSPEC2D_BOTTOM + +#ifdef USE_HDF5 + ! local parameters + integer :: i,j,k,ispec,iglob,ier + real(kind=CUSTOM_REAL),dimension(:),allocatable :: tmp_array + + ! MPI variables + integer :: info, comm + + ! processor dependent group names + character(len=64) :: gname_region + + ! offset arrays + integer, dimension(0:NPROCTOT-1) :: offset_nnodes + integer, dimension(0:NPROCTOT-1) :: offset_nnodes_xy + integer, dimension(0:NPROCTOT-1) :: offset_nnodes_oceans + integer, dimension(0:NPROCTOT-1) :: offset_nelems + + ! nspec2d_* arrays (for storing actual number of elements) + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_xmin + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_xmax + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_ymin + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_ymax + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_bottom + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2D_top + + ! sub array for nspec2d_* arrays (for storing the padded arrays) + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_xmin + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_xmax + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_ymin + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_ymax + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_bottom + integer, dimension(0:NPROCTOT-1) :: arr_nspec2D_top + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT) + + ! share the offset info + ! + ! offset_nnodes + ! offset_nelems + ! offset_n_elms_bounds + call gather_all_all_singlei(nglob, offset_nnodes, NPROCTOT) + call gather_all_all_singlei(nglob_xy, offset_nnodes_xy, NPROCTOT) + call gather_all_all_singlei(nglob_oceans, offset_nnodes_oceans, NPROCTOT) + call gather_all_all_singlei(nspec, offset_nelems, NPROCTOT) + + ! arr_nspec2D_* arrays + call gather_all_all_singlei(nspec2D_xmin, act_arr_nspec2D_xmin, NPROCTOT) + call gather_all_all_singlei(nspec2D_xmax, act_arr_nspec2D_xmax, NPROCTOT) + call gather_all_all_singlei(nspec2D_ymin, act_arr_nspec2D_ymin, NPROCTOT) + call gather_all_all_singlei(nspec2D_ymax, act_arr_nspec2D_ymax, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_BOTTOM, act_arr_nspec2D_bottom, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_TOP, act_arr_nspec2D_top, NPROCTOT) + call gather_all_all_singlei(size(ibelm_xmin), arr_nspec2D_xmin, NPROCTOT) + call gather_all_all_singlei(size(ibelm_xmax), arr_nspec2D_xmax, NPROCTOT) + call gather_all_all_singlei(size(ibelm_ymin), arr_nspec2D_ymin, NPROCTOT) + call gather_all_all_singlei(size(ibelm_ymax), arr_nspec2D_ymax, NPROCTOT) + call gather_all_all_singlei(size(ibelm_bottom), arr_nspec2D_bottom, NPROCTOT) + call gather_all_all_singlei(size(ibelm_top), arr_nspec2D_top, NPROCTOT) + + ! + ! prepare file, group and dataset by myrank == 0 + ! + if (myrank == 0) then + ! create and open solver_data.h5 + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'solver_data.h5' + if (iregion_code == 1) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + + ! create group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + call h5_create_group(gname_region) + + ! open group for iregion_code + call h5_open_group(gname_region) + + ! create dataset for offset arrays and store them by myrank == 0 + + call h5_create_dataset_gen_in_group('offset_nnodes', (/NPROCTOT/), 1, 1) ! nnode = nglob + call h5_create_dataset_gen_in_group('offset_nnodes_xy', (/NPROCTOT/), 1, 1) ! nnode = nglob_xy + call h5_create_dataset_gen_in_group('offset_nnodes_oceans', (/NPROCTOT/), 1, 1) ! nnode = nglob_oceans + call h5_create_dataset_gen_in_group('offset_nelems' , (/NPROCTOT/), 1, 1) ! nspec = nelem + + ! the other dataset need to be written by all ranks + ! so here we just create the dataset + call h5_create_dataset_gen_in_group('xstore', (/sum(offset_nnodes(:))/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('ystore', (/sum(offset_nnodes(:))/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('zstore', (/sum(offset_nnodes(:))/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('ibool', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, 1) + call h5_create_dataset_gen_in_group('idoubling', (/sum(offset_nelems(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ispec_is_tiso', (/sum(offset_nelems(:))/), 1, 0) + call h5_create_dataset_gen_in_group('xixstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('xiystore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('xizstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('etaxstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('etaystore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('etazstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('gammaxstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('gammaystore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('gammazstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('rhostore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('kappavstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + if (iregion_code /= IREGION_OUTER_CORE) then + call h5_create_dataset_gen_in_group('muvstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + + select case(iregion_code) + case (IREGION_CRUST_MANTLE) + ! crusl/mantle region mesh + ! save anisotropy in the mantle only + if (ANISOTROPIC_3D_MANTLE) then + call h5_create_dataset_gen_in_group('c11store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c12store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c13store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c14store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c15store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c16store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c22store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c23store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c24store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c25store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c26store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c33store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c34store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c35store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c36store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c44store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c45store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c46store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c55store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c56store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c66store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + else + if (TRANSVERSE_ISOTROPY) then + call h5_create_dataset_gen_in_group('kappahstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('muhstore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('eta_anisostore', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + endif + ! for azimutahl aniso kernels + call h5_create_dataset_gen_in_group('mu0store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + + case(IREGION_INNER_CORE) + ! inner core region mesh + if (ANISOTROPIC_INNER_CORE) then + call h5_create_dataset_gen_in_group('c11store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c12store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c13store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c33store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('c44store', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + end select + + ! Stacy + if (ABSORBING_CONDITIONS) then + if (iregion_code == IREGION_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group('rho_vp', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('rho_vs', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + else if (iregion_code == IREGION_OUTER_CORE) then + call h5_create_dataset_gen_in_group('rho_vp', (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + endif + + ! mass matrices + if (((NCHUNKS /= 6 .and. ABSORBING_CONDITIONS) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) .and. iregion_code == IREGION_INNER_CORE)) then + call h5_create_dataset_gen_in_group('rmassx', (/sum(offset_nnodes_xy(:))/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('rmassy', (/sum(offset_nnodes_xy(:))/), 1, CUSTOM_REAL) + endif + + call h5_create_dataset_gen_in_group('rmassz', (/sum(offset_nnodes(:))/), 1, CUSTOM_REAL) + + ! maxx matrices for backward simulation when ROTATION is .true. + if (ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) then + if (iregion_code == IREGION_CRUST_MANTLE .or. iregion_code == IREGION_INNER_CORE) then + call h5_create_dataset_gen_in_group('b_rmassx', (/sum(offset_nnodes_xy(:))/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('b_rmassy', (/sum(offset_nnodes_xy(:))/), 1, CUSTOM_REAL) + endif + endif + + ! ocean load mass matrix + if (OCEANS .and. iregion_code == IREGION_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group('rmass_ocean_load', (/sum(offset_nnodes_oceans(:))/), 1, CUSTOM_REAL) + endif + + ! close group for iregion_code + call h5_close_group() + ! close solver_data.h5 + call h5_close_file() + + ! create and open bounday.h5 + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'boundary.h5' + if (iregion_code == 1) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + + ! create group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + call h5_create_group(gname_region) + ! open group for iregion_code + call h5_open_group(gname_region) + + ! create node/element number dumps + call h5_create_dataset_gen_in_group('nspec2D_xmin', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('nspec2D_xmax', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('nspec2D_ymin', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('nspec2D_ymax', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('nspec2D_bottom', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('nspec2D_top', (/NPROCTOT/), 1, 1) + + call h5_create_dataset_gen_in_group('sub_nspec2D_xmin', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_nspec2D_xmax', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_nspec2D_ymin', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_nspec2D_ymax', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_nspec2D_bottom', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_nspec2D_top', (/NPROCTOT/), 1, 1) + + call h5_create_dataset_gen_in_group('ibelm_xmin', (/sum(arr_nspec2d_xmin(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_xmax', (/sum(arr_nspec2d_xmax(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_ymin', (/sum(arr_nspec2d_ymin(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_ymax', (/sum(arr_nspec2d_ymax(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_bottom', (/sum(arr_nspec2d_bottom(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_top', (/sum(arr_nspec2d_top(:))/), 1, 1) + + call h5_create_dataset_gen_in_group('normal_xmin', (/NDIM,NGLLY,NGLLZ,sum(arr_nspec2d_xmin(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_xmax', (/NDIM,NGLLY,NGLLZ,sum(arr_nspec2d_xmax(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_ymin', (/NDIM,NGLLX,NGLLZ,sum(arr_nspec2d_ymin(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_ymax', (/NDIM,NGLLX,NGLLZ,sum(arr_nspec2d_ymax(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_bottom', (/NDIM,NGLLX,NGLLY,sum(arr_nspec2d_bottom(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_top', (/NDIM,NGLLX,NGLLY,sum(arr_nspec2d_top(:))/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen_in_group('jacobian2D_xmin', (/NGLLY,NGLLZ,sum(arr_nspec2d_xmin(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('jacobian2D_xmax', (/NGLLY,NGLLZ,sum(arr_nspec2d_xmin(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('jacobian2D_ymin', (/NGLLX,NGLLZ,sum(arr_nspec2d_ymin(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('jacobian2D_ymax', (/NGLLX,NGLLZ,sum(arr_nspec2d_ymax(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('jacobian2D_bottom', (/NGLLX,NGLLY,sum(arr_nspec2d_bottom(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('jacobian2D_top', (/NGLLX,NGLLY,sum(arr_nspec2d_top(:))/), 3, CUSTOM_REAL) + + ! close group for iregion_code + call h5_close_group() + ! close boundary.h5 + call h5_close_file() + + ! create and open attenuation.h5 + if (ATTENUATION) then + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'attenuation.h5' + if (iregion_code == 1) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + + ! create group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + call h5_create_group(gname_region) + ! open group for iregion_code + call h5_open_group(gname_region) + + ! create dataset for attenuation + call h5_create_dataset_gen_in_group('tau_s_store', (/N_SLS*NPROCTOT/), 1, 8) + call h5_create_dataset_gen_in_group('tau_e_store', (/ATT1, ATT2, ATT3, N_SLS, sum(offset_nelems(:))/), 5, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('Qmu_store', (/ATT1, ATT2, ATT3, sum(offset_nelems(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('att_f_c_source', (/NPROCTOT/), 1, 8) + + call h5_close_group() + call h5_close_file() + endif + + endif ! myrank == 0 + + ! + ! write the data to the HDF5 file by all ranks + ! (except the number of nodes/elements, which is written by myrank == 0) + ! + ! create and open solver_data.h5 + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'solver_data.h5' + call h5_open_file_p_collect(name_database_hdf5) + + ! open group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + call h5_open_group(gname_region) + + call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes", (/offset_nnodes(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes_xy", (/offset_nnodes_xy(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("offset_nnodes_oceans", (/offset_nnodes_oceans(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("offset_nelems", (/offset_nelems(myrank)/), (/myrank/), H5_COL) + + allocate(tmp_array(nglob),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array for mesh topology') + + !--- x coordinate + tmp_array(:) = 0._CUSTOM_REAL + do ispec = 1,nspec + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + iglob = ibool(i,j,k,ispec) + ! distinguish between single and double precision for reals + tmp_array(iglob) = real(xstore(i,j,k,ispec), kind=CUSTOM_REAL) + enddo + enddo + enddo + enddo + + call h5_write_dataset_collect_hyperslab_in_group('xstore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) + !--- y coordinate + tmp_array(:) = 0._CUSTOM_REAL + do ispec = 1,nspec + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + iglob = ibool(i,j,k,ispec) + ! distinguish between single and double precision for reals + tmp_array(iglob) = real(ystore(i,j,k,ispec), kind=CUSTOM_REAL) + enddo + enddo + enddo + enddo + call h5_write_dataset_collect_hyperslab_in_group('ystore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) + !--- z coordinate + tmp_array(:) = 0._CUSTOM_REAL + do ispec = 1,nspec + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + iglob = ibool(i,j,k,ispec) + ! distinguish between single and double precision for reals + tmp_array(iglob) = real(zstore(i,j,k,ispec), kind=CUSTOM_REAL) + enddo + enddo + enddo + enddo + call h5_write_dataset_collect_hyperslab_in_group('zstore', tmp_array, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) + deallocate(tmp_array) + + call h5_write_dataset_collect_hyperslab_in_group('ibool', ibool, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('idoubling', idoubling, (/sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ispec_is_tiso', ispec_is_tiso, (/sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('xixstore', xixstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('xiystore', xiystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('xizstore', xizstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('etaxstore', etaxstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('etaystore', etaystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('etazstore', etazstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('gammaxstore', gammaxstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('gammaystore', gammaystore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('gammazstore', gammazstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('rhostore', rhostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('kappavstore', kappavstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + if (iregion_code /= IREGION_OUTER_CORE) then + call h5_write_dataset_collect_hyperslab_in_group('muvstore', muvstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + endif + + select case(iregion_code) + case (IREGION_CRUST_MANTLE) + ! crusl/mantle region mesh + ! save anisotropy in the mantle only + if (ANISOTROPIC_3D_MANTLE) then + call h5_write_dataset_collect_hyperslab_in_group('c11store', c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c12store', c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c13store', c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c14store', c14store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c15store', c15store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c16store', c16store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c22store', c22store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c23store', c23store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c24store', c24store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c25store', c25store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c26store', c26store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c33store', c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c34store', c34store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c35store', c35store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c36store', c36store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c44store', c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c45store', c45store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c46store', c46store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c55store', c55store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c56store', c56store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c66store', c66store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + else + if (TRANSVERSE_ISOTROPY) then + call h5_write_dataset_collect_hyperslab_in_group('kappahstore', kappahstore, & + (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('muhstore', muhstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('eta_anisostore', eta_anisostore, & + (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + endif + endif + ! for azimutahl aniso kernels + call h5_write_dataset_collect_hyperslab_in_group('mu0store', mu0store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) +!! + case(IREGION_INNER_CORE) + ! inner core region mesh + if (ANISOTROPIC_INNER_CORE) then + call h5_write_dataset_collect_hyperslab_in_group('c11store', c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c12store', c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c13store', c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c33store', c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('c44store', c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + endif + end select +!! + ! Stacy + if (ABSORBING_CONDITIONS) then + if (iregion_code == IREGION_CRUST_MANTLE) then + call h5_write_dataset_collect_hyperslab_in_group('rho_vp', rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('rho_vs', rho_vs, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + else if (iregion_code == IREGION_OUTER_CORE) then + call h5_write_dataset_collect_hyperslab_in_group('rho_vp', rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + endif + endif +!!! + ! mass matrices + if (((NCHUNKS /= 6 .and. ABSORBING_CONDITIONS) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) .and. iregion_code == IREGION_INNER_CORE)) then + call h5_write_dataset_collect_hyperslab_in_group('rmassx', rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('rmassy', rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) + endif + + call h5_write_dataset_collect_hyperslab_in_group('rmassz', rmassz, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) + + ! maxx matrices for backward simulation when ROTATION is .true. + if (ROTATION .and. EXACT_MASS_MATRIX_FOR_ROTATION) then + if (iregion_code == IREGION_CRUST_MANTLE .or. iregion_code == IREGION_INNER_CORE) then + call h5_write_dataset_collect_hyperslab_in_group('b_rmassx', b_rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('b_rmassy', b_rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) + endif + endif + + ! ocean load mass matrix + if (OCEANS .and. iregion_code == IREGION_CRUST_MANTLE) then + call h5_write_dataset_collect_hyperslab_in_group('rmass_ocean_load', rmass_ocean_load, & + (/sum(offset_nnodes_oceans(0:myrank-1))/), H5_COL) + endif + + ! close group for iregion_code + call h5_close_group() + ! close solver_data.h5 + call h5_close_file_p() + + ! create and open bounday.h5 + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'boundary.h5' + call h5_open_file_p_collect(name_database_hdf5) + + ! create group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + ! open group for iregion_code + call h5_open_group(gname_region) + + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_xmin', (/act_arr_nspec2d_xmin(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_xmax', (/act_arr_nspec2d_xmax(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_ymin', (/act_arr_nspec2d_ymin(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_ymax', (/act_arr_nspec2d_ymax(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_bottom', (/act_arr_nspec2d_bottom(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('nspec2D_top', (/act_arr_nspec2d_top(myrank)/), (/myrank/), H5_COL) + + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_xmin', (/arr_nspec2d_xmin(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_xmax', (/arr_nspec2d_xmax(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_ymin', (/arr_nspec2d_ymin(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_ymax', (/arr_nspec2d_ymax(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_bottom', (/arr_nspec2d_bottom(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_nspec2D_top', (/arr_nspec2d_top(myrank)/), (/myrank/), H5_COL) + + if (arr_nspec2d_xmin(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmin', ibelm_xmin, & + (/sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_xmin', normal_xmin, & + (/0,0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmin', jacobian2D_xmin, & + (/0,0,sum(arr_nspec2d_xmin(0:myrank-1))/), H5_COL) + endif + if (arr_nspec2d_xmax(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_xmax', ibelm_xmax, & + (/sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_xmax', normal_xmax, & + (/0,0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_xmax', jacobian2D_xmax, & + (/0,0,sum(arr_nspec2d_xmax(0:myrank-1))/), H5_COL) + endif + if (arr_nspec2d_ymin(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymin', ibelm_ymin, & + (/sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_ymin', normal_ymin, & + (/0,0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymin', jacobian2D_ymin, & + (/0,0,sum(arr_nspec2d_ymin(0:myrank-1))/), H5_COL) + endif + if (arr_nspec2d_ymax(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_ymax', ibelm_ymax, & + (/sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_ymax', normal_ymax, & + (/0,0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_ymax', jacobian2D_ymax, & + (/0,0,sum(arr_nspec2d_ymax(0:myrank-1))/), H5_COL) + endif + if (arr_nspec2d_bottom(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_bottom', ibelm_bottom, & + (/sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_bottom', normal_bottom, & + (/0,0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_bottom', jacobian2D_bottom, & + (/0,0,sum(arr_nspec2d_bottom(0:myrank-1))/), H5_COL) + endif + if (arr_nspec2d_top(myrank) /= 0) then + call h5_write_dataset_collect_hyperslab_in_group('ibelm_top', ibelm_top, & + (/sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_top', normal_top, & + (/0,0,0,sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('jacobian2D_top', jacobian2D_top, & + (/0,0,sum(arr_nspec2d_top(0:myrank-1))/), H5_COL) + endif + + ! close group for iregion_code + call h5_close_group() + ! close boundary.h5 + call h5_close_file_p() + + ! create and open attenuation.h5 + if (ATTENUATION) then + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'attenuation.h5' + call h5_open_file_p_collect(name_database_hdf5) + ! create group for iregion_code + write(gname_region, "('reg',i1)") iregion_code + ! open group for iregion_code + call h5_open_group(gname_region) + + ! create dataset for attenuation + call h5_write_dataset_collect_hyperslab_in_group('tau_s_store', tau_s_store, (/myrank*N_SLS/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('tau_e_store', tau_e_store, & + (/0,0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('Qmu_store', Qmu_store, & + (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('att_f_c_source', (/ATT_F_C_SOURCE/), (/myrank/), H5_COL) + + call h5_close_group() + call h5_close_file_p() + endif + +#else + ! no HDF5 compilation + + ! user output + print * + print *, "Error: HDF5 routine save_databases_hdf5() called without HDF5 Support." + print *, "To enable HDF5 support, reconfigure with --with-hdf5 flag." + print * + stop 'Error HDF5 save_databases_hdf5(): called without compilation support' + +#endif + +end subroutine save_arrays_solver_hdf5 + +! +!----------------------------------------------------------------------- +! + + subroutine save_arrays_boundary_hdf5() + +! saves arrays for boundaries such as MOHO, 400 and 670 discontinuities + +#ifdef USE_HDF5 + use shared_parameters, only: LOCAL_PATH, H5_COL + + use constants, only: myrank,SUPPRESS_CRUSTAL_MESH,CUSTOM_REAL, & + NDIM, NGLLX, NGLLY + + use meshfem_models_par, only: & + HONOR_1D_SPHERICAL_MOHO + + use meshfem_par, only: & + iregion_code, NPROCTOT + +! boundary kernels + use regions_mesh_par2, only: & + NSPEC2D_MOHO, NSPEC2D_400, NSPEC2D_670, & + ibelm_moho_top,ibelm_moho_bot,ibelm_400_top,ibelm_400_bot, & + ibelm_670_top,ibelm_670_bot,normal_moho,normal_400,normal_670, & + ispec2D_moho_top,ispec2D_moho_bot,ispec2D_400_top,ispec2D_400_bot, & + ispec2D_670_top,ispec2D_670_bot + + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + + ! arrays for Moho, 400, 670 + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_moho_top + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_moho_bot + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_400_top + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_400_bot + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_670_top + integer, dimension(0:NPROCTOT-1) :: arr_nspec2d_670_bot + + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_moho_top + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_moho_bot + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_400_top + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_400_bot + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_670_top + integer, dimension(0:NPROCTOT-1) :: act_arr_nspec2d_670_bot + + ! processor dependent group names + character(len=64) :: gname_region + + ! process rank + myrank = myrank + + ! first check the number of surface elements are the same for Moho, 400, 670 + if (.not. SUPPRESS_CRUSTAL_MESH .and. HONOR_1D_SPHERICAL_MOHO) then + if (ispec2D_moho_top /= NSPEC2D_MOHO .or. ispec2D_moho_bot /= NSPEC2D_MOHO) & + call exit_mpi(myrank, 'Not the same number of Moho surface elements') + endif + if (ispec2D_400_top /= NSPEC2D_400 .or. ispec2D_400_bot /= NSPEC2D_400) & + call exit_mpi(myrank,'Not the same number of 400 surface elements') + if (ispec2D_670_top /= NSPEC2D_670 .or. ispec2D_670_bot /= NSPEC2D_670) & + call exit_mpi(myrank,'Not the same number of 670 surface elements') + + ! gather the number of surface elements for Moho, 400, 670 + call gather_all_all_singlei(size(ibelm_moho_top), arr_nspec2d_moho_top, NPROCTOT) + call gather_all_all_singlei(size(ibelm_moho_bot), arr_nspec2d_moho_bot, NPROCTOT) + call gather_all_all_singlei(size(ibelm_400_top), arr_nspec2d_400_top, NPROCTOT) + call gather_all_all_singlei(size(ibelm_400_bot), arr_nspec2d_400_bot, NPROCTOT) + call gather_all_all_singlei(size(ibelm_670_top), arr_nspec2d_670_top, NPROCTOT) + call gather_all_all_singlei(size(ibelm_670_bot), arr_nspec2d_670_bot, NPROCTOT) + + call gather_all_all_singlei(NSPEC2D_MOHO, act_arr_nspec2d_moho_top, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_MOHO, act_arr_nspec2d_moho_bot, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_400, act_arr_nspec2d_400_top, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_400, act_arr_nspec2d_400_bot, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_670, act_arr_nspec2d_670_top, NPROCTOT) + call gather_all_all_singlei(NSPEC2D_670, act_arr_nspec2d_670_bot, NPROCTOT) + + ! file name + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'boundary_disc.h5' + ! group name + write(gname_region, "('reg',i1)") iregion_code + + ! create dataset + if (myrank == 0) then + ! create and open boundary.h5 + if (iregion_code == 1) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + + ! create group for iregion_code + call h5_create_group(gname_region) + + ! open group for iregion_code + call h5_open_group(gname_region) + + ! write the number of surface elements for Moho, 400, 670 + call h5_create_dataset_gen_in_group('NSPEC2D_MOHO', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('NSPEC2D_400', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('NSPEC2D_670', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_MOHO_top', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_MOHO_bot', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_400_top', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_400_bot', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_670_top', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('sub_NSPEC2D_670_bot', (/NPROCTOT/), 1, 1) + + ! create dataset for Moho, 400, 670 + call h5_create_dataset_gen_in_group('ibelm_moho_top', (/sum(arr_nspec2d_moho_top(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_moho_bot', (/sum(arr_nspec2d_moho_bot(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_400_top', (/sum(arr_nspec2d_400_top(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_400_bot', (/sum(arr_nspec2d_400_bot(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_670_top', (/sum(arr_nspec2d_670_top(:))/), 1, 1) + call h5_create_dataset_gen_in_group('ibelm_670_bot', (/sum(arr_nspec2d_670_bot(:))/), 1, 1) + call h5_create_dataset_gen_in_group('normal_moho', (/NDIM,NGLLX,NGLLY,sum(arr_nspec2d_moho_top(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_400', (/NDIM,NGLLX,NGLLY,sum(arr_nspec2d_400_top(:))/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('normal_670', (/NDIM,NGLLX,NGLLY,sum(arr_nspec2d_670_top(:))/), 4, CUSTOM_REAL) + + call h5_close_group() + call h5_close_file() + + endif + + ! + ! write datasets by all processors + ! + ! open file and group + call h5_open_file_p_collect(name_database_hdf5) + call h5_open_group(gname_region) + + ! write datasets + call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_MOHO', (/act_arr_nspec2d_moho_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_400', (/act_arr_nspec2d_400_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('NSPEC2D_670', (/act_arr_nspec2d_670_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_MOHO_top', (/act_arr_nspec2d_moho_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_MOHO_bot', (/act_arr_nspec2d_moho_bot(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_400_top', (/act_arr_nspec2d_400_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_400_bot', (/act_arr_nspec2d_400_bot(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_670_top', (/act_arr_nspec2d_670_top(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('sub_NSPEC2D_670_bot', (/act_arr_nspec2d_670_bot(myrank)/), (/myrank/), H5_COL) + + call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_top', ibelm_moho_top, & + (/sum(arr_nspec2d_moho_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_moho_bot', ibelm_moho_bot, & + (/sum(arr_nspec2d_moho_bot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_400_top', ibelm_400_top, (/sum(arr_nspec2d_400_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_400_bot', ibelm_400_bot, (/sum(arr_nspec2d_400_bot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_670_top', ibelm_670_top, (/sum(arr_nspec2d_670_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('ibelm_670_bot', ibelm_670_bot, (/sum(arr_nspec2d_670_bot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_moho', normal_moho, & + (/0,0,0,sum(arr_nspec2d_moho_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_400', normal_400, (/0,0,0,sum(arr_nspec2d_400_top(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('normal_670', normal_670, (/0,0,0,sum(arr_nspec2d_670_top(0:myrank-1))/), H5_COL) + + ! close group for iregion_code + call h5_close_group() + ! close boundary_disc.h5 + call h5_close_file_p() + +#else + ! no HDF5 compilation + + ! user output + print * + print *, "Error: HDF5 routine save_databases_hdf5() called without HDF5 Support." + print *, "To enable HDF5 support, reconfigure with --with-hdf5 flag." + print * + stop 'Error HDF5 save_databases_hdf5(): called without compilation support' +#endif + +end subroutine save_arrays_boundary_hdf5 + +! +!------------------------------------------------------------------------------------------------- +! + + subroutine save_MPI_arrays_hdf5(iregion_code,LOCAL_PATH, & + num_interfaces,max_nibool_interfaces, & + my_neighbors,nibool_interfaces, & + ibool_interfaces, & + nspec_inner,nspec_outer, & + num_phase_ispec,phase_ispec_inner, & + num_colors_outer,num_colors_inner, & + num_elem_colors) + use constants + +#ifdef USE_HDF5 + use shared_parameters, only: H5_COL + use meshfem_par, only: & + NPROCTOT + use manager_hdf5 +#endif + + implicit none + + integer,intent(in) :: iregion_code + + character(len=MAX_STRING_LEN),intent(in) :: LOCAL_PATH + + ! MPI interfaces + integer,intent(in) :: num_interfaces,max_nibool_interfaces + integer, dimension(num_interfaces),intent(in) :: my_neighbors + integer, dimension(num_interfaces),intent(in) :: nibool_interfaces + integer, dimension(max_nibool_interfaces,num_interfaces),intent(in) :: & + ibool_interfaces + + ! inner/outer elements + integer,intent(in) :: nspec_inner,nspec_outer + integer,intent(in) :: num_phase_ispec + integer,dimension(num_phase_ispec,2),intent(in) :: phase_ispec_inner + + ! mesh coloring + integer,intent(in) :: num_colors_outer,num_colors_inner + integer, dimension(num_colors_outer + num_colors_inner),intent(in) :: & + num_elem_colors + +#ifdef USE_HDF5 + ! local parameters + character(len=64) :: gname_region + + ! offset arrays + integer, dimension(0:NPROCTOT-1) :: offset_num_interfaces + integer, dimension(0:NPROCTOT-1) :: offset_max_nibool_interfaces + integer, dimension(0:NPROCTOT-1) :: offset_nspec_inner + integer, dimension(0:NPROCTOT-1) :: offset_nspec_outer + integer, dimension(0:NPROCTOT-1) :: offset_num_phase_ispec + integer, dimension(0:NPROCTOT-1) :: offset_num_colors_outer + integer, dimension(0:NPROCTOT-1) :: offset_num_colors_inner + + myrank = myrank + + ! gather the offsets + call gather_all_all_singlei(num_interfaces, offset_num_interfaces, NPROCTOT) + call gather_all_all_singlei(max_nibool_interfaces, offset_max_nibool_interfaces, NPROCTOT) + call gather_all_all_singlei(nspec_inner, offset_nspec_inner, NPROCTOT) + call gather_all_all_singlei(nspec_outer, offset_nspec_outer, NPROCTOT) + call gather_all_all_singlei(num_phase_ispec, offset_num_phase_ispec, NPROCTOT) + call gather_all_all_singlei(num_colors_outer, offset_num_colors_outer, NPROCTOT) + call gather_all_all_singlei(num_colors_inner, offset_num_colors_inner, NPROCTOT) + + ! file name + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'solver_data_mpi.h5' + ! group name + write(gname_region, "('reg',i1)") iregion_code + + if (myrank == 0) then + ! create and open solver_data_mpi.h5 + if (iregion_code == IREGION_CRUST_MANTLE) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + ! create group for iregion_code + call h5_create_group(gname_region) + ! open group for iregion_code + call h5_open_group(gname_region) + + ! create datasets + call h5_create_dataset_gen_in_group('offset_num_interfaces', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_max_nibool_interfaces', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_nspec_inner', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_nspec_outer', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_num_phase_ispec', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_num_colors_outer', (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group('offset_num_colors_inner', (/NPROCTOT/), 1, 1) + + if (sum(offset_num_interfaces) > 0) then + call h5_create_dataset_gen_in_group("max_nibool_interfaces", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("my_neighbors", (/sum(offset_num_interfaces(:))/), 1, 1) + call h5_create_dataset_gen_in_group("nibool_interfaces", (/sum(offset_num_interfaces(:))/), 1, 1) + call h5_create_dataset_gen_in_group("ibool_interfaces", & + (/maxval(offset_max_nibool_interfaces),sum(offset_num_interfaces(:))/), 2, 1) + else + ! dummy + call h5_create_dataset_gen_in_group("max_nibool_interfaces", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("my_neighbors", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("nibool_interfaces", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("ibool_interfaces", (/1,NPROCTOT/), 2, 1) + endif + + ! num_phase_ispec + if (sum(offset_num_phase_ispec) > 0) then + call h5_create_dataset_gen_in_group("phase_ispec_inner", (/sum(offset_num_phase_ispec(:)),2/), 2, 1) + else + ! dummy + call h5_create_dataset_gen_in_group("phase_ispec_inner", (/NPROCTOT,2/), 2, 1) + endif + + ! mesh coloring + if (sum(offset_num_colors_outer) + sum(offset_num_colors_inner) > 0) then + call h5_create_dataset_gen_in_group("num_elem_colors", & + (/sum(offset_num_colors_outer(:)) + sum(offset_num_colors_inner(:))/), 1, 1) + else + ! dummy + call h5_create_dataset_gen_in_group("num_elem_colors", (/NPROCTOT/), 1, 1) + endif + + ! close group for iregion_code + call h5_close_group() + ! close solver_data_mpi.h5 + call h5_close_file() + + endif + + ! + ! write datasets by all processors + ! + + ! open file and group + call h5_open_file_p_collect(name_database_hdf5) + call h5_open_group(gname_region) + + ! write datasets + call h5_write_dataset_collect_hyperslab_in_group('offset_num_interfaces', (/offset_num_interfaces(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_max_nibool_interfaces', & + (/offset_max_nibool_interfaces(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_nspec_inner', (/offset_nspec_inner(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_nspec_outer', (/offset_nspec_outer(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_phase_ispec', & + (/offset_num_phase_ispec(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_outer', & + (/offset_num_colors_outer(myrank)/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('offset_num_colors_inner', & + (/offset_num_colors_inner(myrank)/), (/myrank/), H5_COL) + + if (sum(offset_num_interfaces) > 0) then + call h5_write_dataset_collect_hyperslab_in_group("max_nibool_interfaces", (/max_nibool_interfaces/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("my_neighbors", my_neighbors, & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("nibool_interfaces", nibool_interfaces, & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("ibool_interfaces", ibool_interfaces, & + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + else + ! dummy + call h5_write_dataset_collect_hyperslab_in_group("max_nibool_interfaces", (/max_nibool_interfaces/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("my_neighbors", my_neighbors, (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("nibool_interfaces", nibool_interfaces, (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("ibool_interfaces", ibool_interfaces, (/0,myrank/), H5_COL) + endif + + ! num_phase_ispec + if (sum(offset_num_phase_ispec) > 0) then + call h5_write_dataset_collect_hyperslab_in_group("phase_ispec_inner", phase_ispec_inner, & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) + else + ! dummy + call h5_write_dataset_collect_hyperslab_in_group("phase_ispec_inner", phase_ispec_inner, (/myrank,0/), H5_COL) + endif + + ! mesh coloring + if (sum(offset_num_colors_outer) + sum(offset_num_colors_inner) > 0) then + call h5_write_dataset_collect_hyperslab_in_group("num_elem_colors", num_elem_colors, & + (/sum(offset_num_colors_outer(0:myrank-1)) + sum(offset_num_colors_inner(0:myrank-1))/), H5_COL) + else + ! dummy + call h5_write_dataset_collect_hyperslab_in_group("num_elem_colors", num_elem_colors, (/myrank/), H5_COL) + endif + + ! close group for iregion_code + call h5_close_group() + ! close solver_data_mpi.h5 + call h5_close_file_p() + +#else + ! no HDF5 compilation + print* , "Error: HDF5 routine save_MPI_arrays_hdf5() called without HDF5 Support." + print* , "To enable HDF5 support, reconfigure with --with-hdf5 flag." + stop 'Error HDF5 save_MPI_arrays_hdf5(): called without compilation support' +#endif + + end subroutine save_MPI_arrays_hdf5 + + + subroutine get_absorb_stacey_boundary_hdf5(iregion, num_abs_boundary_faces, & + abs_boundary_ispec,abs_boundary_npoin, & + abs_boundary_ijk,abs_boundary_normal,abs_boundary_jacobian2Dw) + + use constants, only: NDIM,NGLLX,NGLLY,NGLLSQUARE,CUSTOM_REAL,MAX_STRING_LEN,IREGION_CRUST_MANTLE + +#ifdef USE_HDF5 + use constants, only: myrank + use shared_parameters, only: H5_COL + use meshfem_par, only: LOCAL_PATH, NPROCTOT + use manager_hdf5 +#endif + + implicit none + + integer,intent(in) :: iregion + + ! absorbing boundary arrays + integer,intent(in) :: num_abs_boundary_faces + integer, dimension(num_abs_boundary_faces), intent(in) :: abs_boundary_ispec + integer, dimension(num_abs_boundary_faces), intent(in) :: abs_boundary_npoin + integer, dimension(3,NGLLSQUARE,num_abs_boundary_faces), intent(in) :: abs_boundary_ijk + real(kind=CUSTOM_REAL), dimension(NDIM,NGLLSQUARE,num_abs_boundary_faces), intent(in) :: abs_boundary_normal + real(kind=CUSTOM_REAL), dimension(NGLLSQUARE,num_abs_boundary_faces), intent(in) :: abs_boundary_jacobian2Dw + +#ifdef USE_HDF5 + ! local parameters + integer, dimension(0:NPROCTOT-1) :: offset_num_abs_boundary_faces + ! MPI parameters + integer :: comm, info + + ! dummy arrays + integer, dimension(1,1), parameter :: i2d_dummy = reshape((/0/),(/1,1/)) + integer, dimension(1,1,1), parameter :: i3d_dummy = reshape((/0/),(/1,1,1/)) + real(kind=CUSTOM_REAL), dimension(1,1), parameter :: r2d_dummy = reshape((/0.0/),(/1,1/)) + real(kind=CUSTOM_REAL), dimension(1,1,1), parameter :: r3d_dummy = reshape((/0.0/),(/1,1,1/)) + + ! variables for HDF5 + character(len=64) :: gname_region + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT) + + ! file name + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'stacey.h5' + ! group name + write(gname_region, "('reg',i1)") iregion + + ! get offset arrays + call gather_all_all_singlei(num_abs_boundary_faces, offset_num_abs_boundary_faces, NPROCTOT) + + ! create datasets by myrank=0 + if (myrank == 0) then + ! create and open stacey.h5 + if (iregion == IREGION_CRUST_MANTLE) then + call h5_create_file(name_database_hdf5) + else + call h5_open_file(name_database_hdf5) + endif + + ! create group for iregion_code + call h5_create_group(gname_region) + ! open group for iregion_code + call h5_open_group(gname_region) + + call h5_create_dataset_gen_in_group("num_abs_boundary_faces", (/NPROCTOT/), 1, 1) + + if (sum(offset_num_abs_boundary_faces) > 0) then + call h5_create_dataset_gen_in_group("abs_boundary_ispec", (/sum(offset_num_abs_boundary_faces(:))/), 1, 1) + call h5_create_dataset_gen_in_group("abs_boundary_npoin", (/sum(offset_num_abs_boundary_faces(:))/), 1, 1) + call h5_create_dataset_gen_in_group("abs_boundary_ijk", & + (/NDIM,NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("abs_boundary_jacobian2Dw", & + (/NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 2, CUSTOM_REAL) + + if (iregion == IREGION_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group("abs_boundary_normal", & + (/NDIM,NGLLSQUARE,sum(offset_num_abs_boundary_faces(:))/), 3, CUSTOM_REAL) + endif + else + ! dummy + call h5_create_dataset_gen_in_group("abs_boundary_ispec", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("abs_boundary_npoin", (/NPROCTOT/), 1, 1) + call h5_create_dataset_gen_in_group("abs_boundary_ijk", (/NDIM,NGLLSQUARE,NPROCTOT/), 3, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("abs_boundary_jacobian2Dw", (/NGLLSQUARE,NPROCTOT/), 2, CUSTOM_REAL) + + if (iregion == IREGION_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group("abs_boundary_normal", (/NDIM,NGLLSQUARE,NPROCTOT/), 3, CUSTOM_REAL) + endif + endif ! iregion == IREGION_CRUST_MANTEL + + ! close group for iregion_code + call h5_close_group() + ! close stacey.h5 + call h5_close_file() + endif ! myrank == 0 + + ! write datasets by all processors + call h5_open_file_p_collect(name_database_hdf5) + call h5_open_group(gname_region) + + ! write datasets + call h5_write_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", & + (/offset_num_abs_boundary_faces(myrank)/), (/myrank/), .true.) + + if (sum(offset_num_abs_boundary_faces) > 0) then + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk, & + (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw, & + (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + if (iregion == IREGION_CRUST_MANTLE) then + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal, & + (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + endif + else + ! dummy + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ispec", (/0/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_npoin", (/0/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_ijk", i3d_dummy, (/0,0,myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", r2d_dummy, (/0,myrank/), H5_COL) + if (iregion == IREGION_CRUST_MANTLE) then + call h5_write_dataset_collect_hyperslab_in_group("abs_boundary_normal", r3d_dummy, (/0,0,myrank/), H5_COL) + endif + endif + + ! close group for iregion_code + call h5_close_group() + ! close stacey.h5 + call h5_close_file_p() + +#else + ! no HDF5 compilation + print* , "Error: HDF5 routine get_absorb_stacey_boundary_hdf5() called without HDF5 Support." + print* , "To enable HDF5 support, reconfigure with --with-hdf5 flag." + stop 'Error HDF5 get_absorb_stacey_boundary_hdf5(): called without compilation support' +#endif + + + + + + + end subroutine get_absorb_stacey_boundary_hdf5 diff --git a/src/meshfem3D/save_model_meshfiles_hdf5.F90 b/src/meshfem3D/save_model_meshfiles_hdf5.F90 new file mode 100644 index 000000000..399253cbf --- /dev/null +++ b/src/meshfem3D/save_model_meshfiles_hdf5.F90 @@ -0,0 +1,277 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + + subroutine save_model_meshfiles_hdf5() + + use constants +#ifdef USE_HDF5 +! outputs model files in binary format + + use shared_parameters, only: R_PLANET,RHOAV,LOCAL_PATH,H5_COL + + use meshfem_par, only: nspec,iregion_code,NPROCTOT + + use meshfem_models_par, only: & + TRANSVERSE_ISOTROPY,ATTENUATION,ATTENUATION_3D,ATTENUATION_1D_WITH_3D_STORAGE, & + HETEROGEN_3D_MANTLE,ANISOTROPIC_3D_MANTLE + + use regions_mesh_par2, only: & + rhostore,kappavstore,kappahstore,muvstore,muhstore,eta_anisostore, & + Qmu_store,Gc_prime_store,Gs_prime_store,mu0store + + use model_heterogen_mantle_par + + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! local parameters + integer :: i,j,k,ispec,ier + real(kind=CUSTOM_REAL) :: scaleval1,scaleval2,scaleval,scale_GPa + real(kind=CUSTOM_REAL),dimension(:,:,:,:),allocatable :: temp_store + + ! dset_name and group_name + character(len=64) :: dset_name, gname_region + + ! offset for nspec + integer, dimension(0:NPROCTOT-1) :: offset_nelems + + ! gather nspec + call gather_all_all_singlei(nspec, offset_nelems, NPROCTOT) + + ! file name + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/'//'meshfile.h5' + ! group name + write(gname_region, "('reg',i1)") iregion_code + + if (myrank == 0) then + ! create the file + if (iregion_code == 1) then + call h5_create_file(name_database_hdf5) + endif + + ! open the file + call h5_open_file(name_database_hdf5) + ! create the group + call h5_create_group(gname_region) + ! open the group + call h5_open_group(gname_region) + + if (TRANSVERSE_ISOTROPY) then + ! vpv + dset_name = 'vpv' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! vph + dset_name = 'vph' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! vsv + dset_name = 'vsv' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! vsh + dset_name = 'vsh' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! rho + dset_name = 'rho' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! eta + dset_name = 'eta' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + + else + + ! isotropic model + ! vp + dset_name = 'vp' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! vs + dset_name = 'vs' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! rho + dset_name = 'rho' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + + endif ! TRANSVERSE_ISOTROPY + + ! anisotropic values + if (ANISOTROPIC_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then + ! Gc_prime + dset_name = 'Gc_prime' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! Gs_prime + dset_name = 'Gs_prime' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + ! mu0 + dset_name = 'mu0' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + + if (ATTENUATION) then + ! Qmu + dset_name = 'Qmu' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + + if (HETEROGEN_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then + ! dvp + dset_name = 'dvp' + call h5_create_dataset_gen_in_group(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nelems(:))/), 4, CUSTOM_REAL) + endif + + ! close the group + call h5_close_group() + ! close the file + call h5_close_file() + + endif ! myrank == 0 + + ! + ! write the data + ! + ! open the file + call h5_open_file_p(name_database_hdf5) + call h5_open_group(gname_region) + + ! scaling factors to re-dimensionalize units + scaleval1 = real(sqrt(PI*GRAV*RHOAV)*(R_PLANET/1000.0d0),kind=CUSTOM_REAL) + scaleval2 = real(RHOAV/1000.0d0,kind=CUSTOM_REAL) + + ! uses temporary array + allocate(temp_store(NGLLX,NGLLY,NGLLZ,nspec),stat=ier) + if (ier /= 0) stop 'Error allocating temp_store array' + temp_store(:,:,:,:) = 0._CUSTOM_REAL + + if (TRANSVERSE_ISOTROPY) then + + ! vpv + temp_store(:,:,:,:) = sqrt((kappavstore(:,:,:,:) + 4.0_CUSTOM_REAL * muvstore(:,:,:,:)/3.0_CUSTOM_REAL)/rhostore(:,:,:,:)) & + * scaleval1 + dset_name = 'vpv' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! vph + temp_store(:,:,:,:) = sqrt((kappahstore(:,:,:,:) + 4.0_CUSTOM_REAL * muhstore(:,:,:,:)/3.0_CUSTOM_REAL)/rhostore(:,:,:,:)) & + * scaleval1 + dset_name = 'vph' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! vsv + temp_store(:,:,:,:) = sqrt( muvstore(:,:,:,:)/rhostore(:,:,:,:) )*scaleval1 + dset_name = 'vsv' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! vsh + temp_store(:,:,:,:) = sqrt( muhstore(:,:,:,:)/rhostore(:,:,:,:) )*scaleval1 + dset_name = 'vsh' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! rho + temp_store(:,:,:,:) = rhostore(:,:,:,:) * scaleval2 + dset_name = 'rho' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! eta + dset_name = 'eta' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, eta_anisostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + + else + ! isotropic model + ! vp + temp_store(:,:,:,:) = sqrt((kappavstore(:,:,:,:) + 4.0_CUSTOM_REAL * muvstore(:,:,:,:)/3.0_CUSTOM_REAL)/rhostore(:,:,:,:)) & + * scaleval1 + dset_name = 'vp' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! vs + temp_store(:,:,:,:) = sqrt( muvstore(:,:,:,:)/rhostore(:,:,:,:) )*scaleval1 + dset_name = 'vs' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! rho + temp_store(:,:,:,:) = rhostore(:,:,:,:) * scaleval2 + dset_name = 'rho' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + + endif ! TRANSVERSE_ISOTROPY + + ! anisotropic values + if (ANISOTROPIC_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then + ! the scale of GPa--[g/cm^3][(km/s)^2] + scaleval = real(sqrt(PI*GRAV*RHOAV),kind=CUSTOM_REAL) + scale_GPa = real((RHOAV/1000.d0)*((R_PLANET*scaleval/1000.d0)**2),kind=CUSTOM_REAL) + + ! Gc_prime + dset_name = 'Gc_prime' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, Gc_prime_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! Gs_prime + dset_name = 'Gs_prime' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, Gs_prime_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! mu0 + temp_store(:,:,:,:) = mu0store(:,:,:,:) * scale_GPa + dset_name = 'mu0' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + + endif + + if (ATTENUATION) then + if (ATTENUATION_3D .or. ATTENUATION_1D_WITH_3D_STORAGE) then + temp_store(:,:,:,:) = Qmu_store(:,:,:,:) + else + do ispec = 1,nspec + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + temp_store(i,j,k,ispec) = Qmu_store(1,1,1,ispec) + end do + end do + end do + end do + endif + + ! Qmu + dset_name = 'Qmu' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, temp_store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + + endif ! ATTENUATION + + deallocate(temp_store) + + if (HETEROGEN_3D_MANTLE .and. iregion_code == IREGION_CRUST_MANTLE) then + ! dvp + dset_name = 'dvp' + call h5_write_dataset_collect_hyperslab_in_group(dset_name, dvpstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + endif + + ! close the group + call h5_close_group() + ! close the file + call h5_close_file() + +#else + + print *, 'Error: HDF5 support not enabled in this build' + stop + +#endif + + +end subroutine save_model_meshfiles_hdf5 diff --git a/src/shared/broadcast_computed_parameters.f90 b/src/shared/broadcast_computed_parameters.f90 index 1ce5aeb98..eabfe82e4 100644 --- a/src/shared/broadcast_computed_parameters.f90 +++ b/src/shared/broadcast_computed_parameters.f90 @@ -34,10 +34,10 @@ subroutine broadcast_computed_parameters() ! local parameters ! broadcast parameter arrays - integer, parameter :: nparam_i = 50 + integer, parameter :: nparam_i = 51 integer, dimension(nparam_i) :: bcast_integer - integer, parameter :: nparam_l = 74 + integer, parameter :: nparam_l = 77 logical, dimension(nparam_l) :: bcast_logical integer, parameter :: nparam_dp = 42 @@ -77,7 +77,8 @@ subroutine broadcast_computed_parameters() GPU_RUNTIME,NUMBER_OF_SIMULTANEOUS_RUNS, & MODEL_GLL_TYPE,USER_NSTEP, & NSTEP_STEADY_STATE,NTSTEP_BETWEEN_OUTPUT_SAMPLE, & - POISSON_SOLVER /) + POISSON_SOLVER, & + HDF5_IO_NODES /) bcast_logical = (/ & TRANSVERSE_ISOTROPY,ANISOTROPIC_3D_MANTLE,ANISOTROPIC_INNER_CORE, & @@ -111,7 +112,8 @@ subroutine broadcast_computed_parameters() OUTPUT_SEISMOS_3D_ARRAY, & REGIONAL_MESH_CUTOFF,REGIONAL_MESH_ADD_2ND_DOUBLING, & EMC_MODEL, & - FULL_GRAVITY /) + FULL_GRAVITY, & + HDF5_ENABLED, HDF5_FOR_MOVIES, OUTPUT_SEISMOS_HDF5 /) bcast_double_precision = (/ & DT, & @@ -284,6 +286,7 @@ subroutine broadcast_computed_parameters() NSTEP_STEADY_STATE = bcast_integer(48) NTSTEP_BETWEEN_OUTPUT_SAMPLE = bcast_integer(49) POISSON_SOLVER = bcast_integer(50) + HDF5_IO_NODES = bcast_integer(51) ! logicals TRANSVERSE_ISOTROPY = bcast_logical(1) @@ -360,6 +363,9 @@ subroutine broadcast_computed_parameters() REGIONAL_MESH_ADD_2ND_DOUBLING = bcast_logical(72) EMC_MODEL = bcast_logical(73) FULL_GRAVITY = bcast_logical(74) + HDF5_ENABLED = bcast_logical(75) + HDF5_FOR_MOVIES = bcast_logical(76) + OUTPUT_SEISMOS_HDF5 = bcast_logical(77) ! double precisions DT = bcast_double_precision(1) diff --git a/src/shared/hdf5_manager.F90 b/src/shared/hdf5_manager.F90 new file mode 100644 index 000000000..fc1a64495 --- /dev/null +++ b/src/shared/hdf5_manager.F90 @@ -0,0 +1,6065 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +!------------------------------------------------------------------------------- +!> Tools for parallel HDF5 support, +!> contains wrapper subroutines for common hdf5 calls +! +! The idea for these HDF5 tools is to convert all database files to a HDF5-format +! for better scalability to very large simulations. +! It is complementary to ADIOS2 support in case those libraries are not installed/wanted. +! These HDF5 files should avoid bottlenecks on HPC clusters due to the possible higher number of file outputs +! in our simulations (database files, seismograms, visualization snapshots, etc.). + +! note: HDF5 library calls use a format like "h5**do_something***_f()" +! our own wrapper functions in this module will rather use something like "h5_***do_something_***()", +! and higher-level routines called in other Fortran routines like "***do_something_h5***()", +! to better distinguish between library functions and wrappers. +! +! version info: +! - initial version, April 2023: +! Masaru Nagaso (main developer, all initial hdf5 functionality) +! Daniel Peter (just adding Masaru-san's original implementation, sometimes renaming things to more specfem-coherent ways) +! +!------------------------------------------------------------------------------- + +! +! parallel HDF5 file I/O routines +! + +module manager_hdf5 + +! class-like module for HDF5 routines + + use constants, only: CUSTOM_REAL, MAX_STRING_LEN + +#if defined(USE_HDF5) + use hdf5 +#endif + + implicit none + + private + + ! public routines + ! also to act as empty stubs (for missing HDF5 compilation support) + public :: write_attenuation_file_hdf5, read_attenuation_file_hdf5 + public :: write_checkmesh_data_hdf5, write_checkmesh_xdmf_hdf5 + + ! functional interface + public :: h5_initialize + +#if defined(USE_HDF5) + ! only w/ HDF5 compilation + + public :: & + h5_finalize + + public :: & + h5_create_file, & + h5_create_or_open_file, & + h5_open_file, & + h5_close_file, & + h5_create_group, & + h5_open_group, & + h5_close_group, & + h5_create_subgroup, & + h5_open_subgroup, & + h5_open_or_create_group, & + h5_close_subgroup + + public :: & + h5_open_dataset, & + h5_open_dataset2, & + h5_close_dataset + + public :: & + h5_add_attribute_i, & + h5_set_mpi_info, & + h5_set_buffer_size, & + h5_set_sieve_buffer_size, & + h5_set_group_name, & + h5_gather_dsetsize + + public :: & + h5_create_file_p, & + h5_create_file_p_collect, & + h5_open_file_p, & + h5_open_file_p_collect, & + h5_close_file_p, & + h5_create_file_prop_list, & + h5_close_prop_list, & + h5_close_prop_list_nocheck, & + h5_open_group_prop_list, & + h5_create_group_prop_list, & + h5_close_group_prop_list, & + h5_create_dataset_prop_list, & + h5_create_dataset_gen, & + h5_create_dataset_gen_in_group, & + h5_check_dataset_exists, & + h5_create_group_p, & + h5_open_group_p + + public :: & + h5_read_attribute_p, & + h5_read_dataset_p, & + h5_read_dataset_p_scalar, & + h5_read_dataset_scalar_collect_hyperslab, & + h5_read_dataset_scalar_collect_hyperslab_in_group, & + h5_read_dataset_collect_hyperslab_in_group, & + h5_read_dataset_collect_hyperslab + + public :: & + i2c + + public :: & + h5_write_dataset, & + h5_write_dataset_no_group, & + h5_write_dataset_p, & + h5_write_dataset_p_a, & + h5_write_dataset_1d_to_2d_r_collect_hyperslab, & + h5_write_dataset_2d_to_3d_r_collect_hyperslab, & + h5_write_dataset_collect_hyperslab_in_group, & + h5_write_dataset_collect_hyperslab + + ! private routines + + private :: & + h5_check_arr_dim, & + h5_check_collective, & + h5_check_group + + private :: & + create_dataset_collect, & + bool_array2integer, & + int_array2bool + + ! generic interface to read dataset + interface h5_read_dataset_p + module procedure h5_read_dataset_p_1d_l ! logical + module procedure h5_read_dataset_p_1d_i ! integer + module procedure h5_read_dataset_p_2d_i + module procedure h5_read_dataset_p_3d_i + module procedure h5_read_dataset_p_4d_i + module procedure h5_read_dataset_p_1d_r ! real + module procedure h5_read_dataset_p_2d_r + module procedure h5_read_dataset_p_3d_r + module procedure h5_read_dataset_p_4d_r + module procedure h5_read_dataset_p_5d_r + module procedure h5_read_dataset_p_2d_d ! double + module procedure h5_read_dataset_p_2d_c ! char + end interface h5_read_dataset_p + + ! generic interface to read scalar dataset + interface h5_read_dataset_p_scalar + module procedure h5_read_dataset_p_scalar_i + module procedure h5_read_dataset_p_scalar_r + end interface h5_read_dataset_p_scalar + + ! generic interface to read dataset in collective mode + interface h5_read_dataset_scalar_collect_hyperslab + module procedure h5_read_dataset_scalar_i_collect_hyperslab + module procedure h5_read_dataset_scalar_r_collect_hyperslab + end interface h5_read_dataset_scalar_collect_hyperslab + + interface h5_read_dataset_scalar_collect_hyperslab_in_group + module procedure h5_read_dataset_scalar_i_collect_hyperslab_in_group + end interface h5_read_dataset_scalar_collect_hyperslab_in_group + + ! generic interface to read dataset in collective mode + interface h5_read_dataset_collect_hyperslab_in_group + module procedure h5_read_dataset_1d_l_collect_hyperslab_in_group ! logical + module procedure h5_read_dataset_1d_i_collect_hyperslab_in_group ! integer + module procedure h5_read_dataset_1d_r_collect_hyperslab_in_group + module procedure h5_read_dataset_1d_d_collect_hyperslab_in_group ! double + module procedure h5_read_dataset_2d_i_collect_hyperslab_in_group + module procedure h5_read_dataset_2d_r_collect_hyperslab_in_group + module procedure h5_read_dataset_3d_i_collect_hyperslab_in_group + module procedure h5_read_dataset_3d_r_collect_hyperslab_in_group + module procedure h5_read_dataset_4d_i_collect_hyperslab_in_group + module procedure h5_read_dataset_4d_r_collect_hyperslab_in_group + module procedure h5_read_dataset_5d_r_collect_hyperslab_in_group + end interface h5_read_dataset_collect_hyperslab_in_group + + ! generic interface to read dataset in collective mode + interface h5_read_dataset_collect_hyperslab + module procedure h5_read_dataset_1d_l_collect_hyperslab ! logical + module procedure h5_read_dataset_1d_i_collect_hyperslab ! integer + module procedure h5_read_dataset_2d_i_collect_hyperslab + module procedure h5_read_dataset_3d_i_collect_hyperslab + module procedure h5_read_dataset_4d_i_collect_hyperslab + module procedure h5_read_dataset_1d_r_collect_hyperslab ! real + module procedure h5_read_dataset_2d_r_collect_hyperslab + module procedure h5_read_dataset_3d_r_collect_hyperslab + module procedure h5_read_dataset_4d_r_collect_hyperslab + module procedure h5_read_dataset_5d_r_collect_hyperslab + module procedure h5_read_dataset_2d_d_collect_hyperslab ! double + end interface h5_read_dataset_collect_hyperslab + + ! generic interface to write dataset + interface h5_write_dataset + module procedure h5_write_dataset_1d_i ! integer + module procedure h5_write_dataset_2d_i + module procedure h5_write_dataset_1d_d ! double + module procedure h5_write_dataset_2d_d + module procedure h5_write_dataset_1d_c ! char + module procedure h5_write_dataset_2d_c + module procedure h5_write_dataset_2d_r ! real + module procedure h5_write_dataset_4d_r + end interface h5_write_dataset + + interface h5_write_dataset_no_group + module procedure h5_write_dataset_1d_i_no_group + module procedure h5_write_dataset_1d_d_no_group + module procedure h5_write_dataset_1d_c_no_group + module procedure h5_write_dataset_2d_r_no_group + end interface h5_write_dataset_no_group + + ! generic interface to write dataset + interface h5_write_dataset_p + module procedure h5_write_dataset_p_1d_l ! logical + module procedure h5_write_dataset_p_1d_i ! integer + module procedure h5_write_dataset_p_2d_i + module procedure h5_write_dataset_p_3d_i + module procedure h5_write_dataset_p_4d_i + module procedure h5_write_dataset_p_1d_r ! real + module procedure h5_write_dataset_p_2d_r + module procedure h5_write_dataset_p_3d_r + module procedure h5_write_dataset_p_4d_r + module procedure h5_write_dataset_p_5d_r + module procedure h5_write_dataset_p_2d_d ! double + end interface h5_write_dataset_p + + interface h5_write_dataset_p_a + module procedure h5_write_dataset_p_1d_ia + module procedure h5_write_dataset_p_2d_ia + end interface h5_write_dataset_p_a + + ! generic interface to write dataset in collective mode + interface h5_write_dataset_collect_hyperslab_in_group + module procedure h5_write_dataset_1d_l_collect_hyperslab_in_group ! logical + module procedure h5_write_dataset_1d_i_collect_hyperslab_in_group + module procedure h5_write_dataset_2d_i_collect_hyperslab_in_group + module procedure h5_write_dataset_3d_i_collect_hyperslab_in_group + module procedure h5_write_dataset_4d_i_collect_hyperslab_in_group + module procedure h5_write_dataset_1d_r_collect_hyperslab_in_group ! real + module procedure h5_write_dataset_2d_r_collect_hyperslab_in_group + module procedure h5_write_dataset_3d_r_collect_hyperslab_in_group + module procedure h5_write_dataset_4d_r_collect_hyperslab_in_group + module procedure h5_write_dataset_5d_r_collect_hyperslab_in_group + module procedure h5_write_dataset_1d_d_collect_hyperslab_in_group ! double + module procedure h5_write_dataset_2d_d_collect_hyperslab_in_group ! double + end interface h5_write_dataset_collect_hyperslab_in_group + + ! generic interface to write dataset in collective mode + interface h5_write_dataset_collect_hyperslab + module procedure h5_write_dataset_1d_l_collect_hyperslab ! logical + module procedure h5_write_dataset_1d_i_collect_hyperslab ! integer + module procedure h5_write_dataset_2d_i_collect_hyperslab + module procedure h5_write_dataset_3d_i_collect_hyperslab + module procedure h5_write_dataset_4d_i_collect_hyperslab + module procedure h5_write_dataset_1d_r_collect_hyperslab ! real + module procedure h5_write_dataset_2d_r_collect_hyperslab + module procedure h5_write_dataset_3d_r_collect_hyperslab + module procedure h5_write_dataset_4d_r_collect_hyperslab + module procedure h5_write_dataset_5d_r_collect_hyperslab + module procedure h5_write_dataset_2d_d_collect_hyperslab ! double + end interface h5_write_dataset_collect_hyperslab + + ! object-oriented interface + ! (Fortran 2003 standard style) + ! + ! usage example: + ! subroutine ** + ! .. + ! use manager_hdf5, only: h5io + ! type(h5io) :: h5 + ! .. + ! h5 = h5io() ! calls the object constructor, i.e. h5io_constructor() + ! call h5%open(fname) ! object function call + ! ! or in a single call: + ! ! h5 = h5io(fname) + ! call h5%write("x",store_val_x_all) + ! call h5%close() + ! .. + ! end subroutine ! h5 goes out of scope, will call the object destructor, i.e., h5io_destructor() + ! + type, public :: h5io + private + logical :: is_initialized = .false. + integer(HID_T) :: f_id = -1, g_id = -1, d_id = -1 + character(len=MAX_STRING_LEN) :: filename = "" + contains + procedure :: open => h5io_open_file + procedure :: close => h5io_close_file + generic :: write => h5io_write_dataset_i, h5io_write_dataset_r + procedure, private :: h5io_write_dataset_i + procedure, private :: h5io_write_dataset_r + final :: h5io_destructor + end type h5io + ! object constructor + ! called by: h5 = h5io() + interface h5io + module procedure :: h5io_constructor + module procedure :: h5io_constructor_from_file + end interface h5io + + ! module parameters + ! ids + integer(HID_T) :: file_id, group_id, parent_group_id, dataset_id + integer(HID_T) :: mem_dspace_id, file_dspace_id ! for collective IO + + ! parallel process + integer(HID_T) :: plist_id, fplist_id, gplist_id + + ! string array + ! initial string length for network/station names (to be determined later) + integer(SIZE_T) :: str_len = 16 + integer(HID_T) :: str_type + ! maximum string length for passing character array undef_mat_prop + integer(SIZE_T), parameter :: str_len_max = MAX_STRING_LEN + integer(HID_T) :: str_type_max + + type(c_ptr) :: f_ptr + + ! MPI info + integer :: this_rank, this_info, this_comm, total_proc + + ! io unit + integer, parameter :: xdmf_mesh = 190 + + ! mesh nodes + public :: xdmf_mesh_nnodes + integer :: xdmf_mesh_nnodes + + ! function call errors + integer :: error + + ! class-wide private variables + character(len=256) :: file_path + character(len=256) :: store_group_name ! groupname for debug + + public :: name_database_hdf5 + + ! store HDF5 output file name + character(len=MAX_STRING_LEN) :: name_database_hdf5 + +#endif + +contains + +!------------------------------------------------------------------------------- +! +! public HDF5 wrapper routines (also available without hdf5 compilation support) +! +!------------------------------------------------------------------------------- + + subroutine h5_initialize() + +#if defined(USE_HDF5) + use constants, only: MAX_LENGTH_NETWORK_NAME, MAX_LENGTH_STATION_NAME +#endif + + implicit none + +#if defined(USE_HDF5) + ! initialize Fortran interface + call h5open_f(error) + call check_error() + + ! prepare string array type + if (MAX_LENGTH_STATION_NAME >= MAX_LENGTH_NETWORK_NAME) then + str_len = MAX_LENGTH_STATION_NAME + else + str_len = MAX_LENGTH_NETWORK_NAME + endif + + call h5tcopy_f(H5T_Fortran_S1, str_type, error) + call check_error() + + call h5tset_size_f(str_type, str_len, error) + call check_error() + + ! string array type for undef_mat_prop array + call h5tcopy_f(H5T_Fortran_S1, str_type_max, error) + call check_error() + + call h5tset_size_f(str_type_max, str_len_max, error) + call check_error() +#else + ! no HDF5 compilation support + + ! compilation without HDF5 support + print * + print *, "Error: HDF5 routine h5_initialize() called without HDF5 Support." + print *, "To enable HDF5 support, reconfigure with --with-hdf5 flag." + print * + + ! safety stop + stop 'Error HDF5 manager: h5_initialize() intitialization called without compilation support' + +#endif + + end subroutine h5_initialize + +!------------------------------------------------------------------------------- +! +! higher level utilities +! +!------------------------------------------------------------------------------- + + subroutine write_attenuation_file_hdf5(factor_common, scale_factor, factor_common_kappa, scale_factor_kappa) + +#if defined(USE_HDF5) + use shared_parameters, only: NPROC, LOCAL_PATH + use constants, only: myrank,N_SLS,NGLLX,NGLLY,NGLLZ +#endif + + implicit none + + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:,:) :: factor_common + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:) :: scale_factor + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:,:) :: factor_common_kappa + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:) :: scale_factor_kappa + +#if defined(USE_HDF5) + integer, dimension(4) :: dims + integer :: nspec + + ! offset arrays + integer, dimension(0:NPROC-1) :: offset_nspec + + ! hdf5 valiables + character(len=64) :: filename, dset_name, tempstr + + dims = shape(scale_factor) + nspec = dims(4) + + ! prepare offset arrays + call gather_all_all_singlei(nspec,offset_nspec,NPROC) ! n spec in each proc + + tempstr = "/external_mesh.h5" + filename = LOCAL_PATH(1:len_trim(LOCAL_PATH))//trim(tempstr) + + ! initialize h5 object + call h5_initialize() + + ! prepare dataset + if (myrank == 0) then + call h5_open_file(filename) + dset_name = "scale_factor" + call h5_create_dataset_gen(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec)/), 4, CUSTOM_REAL) + dset_name = "scale_factor_kappa" + call h5_create_dataset_gen(dset_name, (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec)/), 4, CUSTOM_REAL) + dset_name = "factor_common" + call h5_create_dataset_gen(dset_name, (/N_SLS,NGLLX,NGLLY,NGLLZ,sum(offset_nspec)/), 5, CUSTOM_REAL) + dset_name = "factor_common_kappa" + call h5_create_dataset_gen(dset_name, (/N_SLS,NGLLX,NGLLY,NGLLZ,sum(offset_nspec)/), 5, CUSTOM_REAL) + call h5_close_file() + endif + + call synchronize_all() + + ! write + call h5_open_file_p_collect(filename) + dset_name = "scale_factor" + call h5_write_dataset_4d_r_collect_hyperslab(dset_name, & + scale_factor,(/0,0,0,sum(offset_nspec(0:myrank-1))/),.true.) + dset_name = "scale_factor_kappa" + call h5_write_dataset_4d_r_collect_hyperslab(dset_name, & + scale_factor_kappa,(/0,0,0,sum(offset_nspec(0:myrank-1))/),.true.) + dset_name = "factor_common" + call h5_write_dataset_5d_r_collect_hyperslab(dset_name, & + factor_common,(/0,0,0,0,sum(offset_nspec(0:myrank-1))/),.true.) + dset_name = "factor_common_kappa" + call h5_write_dataset_5d_r_collect_hyperslab(dset_name, & + factor_common_kappa,(/0,0,0,0,sum(offset_nspec(0:myrank-1))/),.true.) + + call h5_close_file_p() + call h5_finalize() +#else + ! no compilation support + ! to avoid compiler warnings + real(kind=CUSTOM_REAL) :: dummy + + dummy = factor_common(1,1,1,1,1) + dummy = factor_common_kappa(1,1,1,1,1) + dummy = scale_factor(1,1,1,1) + dummy = scale_factor_kappa(1,1,1,1) + + ! safety stop + stop 'Error HDF5 manager: write_attenuation_file_hdf5() called without compilation support' +#endif + + end subroutine write_attenuation_file_hdf5 + +! +!------------------------------------------------------------------------------- +! + + subroutine read_attenuation_file_hdf5(factor_common, scale_factor, factor_common_kappa, scale_factor_kappa) + +#if defined(USE_HDF5) + use shared_parameters, only: NPROC, LOCAL_PATH + use constants, only: myrank +#endif + + implicit none + + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:,:) :: factor_common + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:) :: scale_factor + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:,:) :: factor_common_kappa + real(kind=CUSTOM_REAL), allocatable, dimension(:,:,:,:) :: scale_factor_kappa + +#if defined(USE_HDF5) + ! offset array + integer, dimension(0:NPROC-1) :: offset_nspec + + ! hdf5 valiables + character(len=64) :: fname, tempstr + + tempstr = "/external_mesh.h5" + fname = LOCAL_PATH(1:len_trim(LOCAL_PATH))//trim(tempstr) + + ! initialize h5 object + call h5_initialize() + + ! open file + call h5_open_file_p_collect(fname) + ! read offset array + call h5_read_dataset_1d_i_collect_hyperslab("offset_nspec",offset_nspec, (/0/), .true.) + + call h5_read_dataset_4d_r_collect_hyperslab("scale_factor", scale_factor, & + (/0,0,0,sum(offset_nspec(0:myrank-1))/), .true.) + call h5_read_dataset_4d_r_collect_hyperslab("scale_factor_kappa", scale_factor_kappa, & + (/0,0,0,sum(offset_nspec(0:myrank-1))/), .true.) + call h5_read_dataset_5d_r_collect_hyperslab("factor_common", factor_common, & + (/0,0,0,0,sum(offset_nspec(0:myrank-1))/), .true.) + call h5_read_dataset_5d_r_collect_hyperslab("factor_common_kappa", factor_common_kappa, & + (/0,0,0,0,sum(offset_nspec(0:myrank-1))/), .true.) + + call h5_close_file_p() + call h5_finalize() +#else + ! no compilation support + ! to avoid compiler warnings + real(kind=CUSTOM_REAL) :: dummy + + dummy = factor_common(1,1,1,1,1) + dummy = factor_common_kappa(1,1,1,1,1) + dummy = scale_factor(1,1,1,1) + dummy = scale_factor_kappa(1,1,1,1) + + ! safety stop + stop 'Error HDF5 manager: read_attenuation_file_hdf5() called without compilation support' +#endif + + end subroutine read_attenuation_file_hdf5 + +! +!------------------------------------------------------------------------------- +! + + subroutine write_checkmesh_data_hdf5(dset_name,dump_array) + +#if defined(USE_HDF5) + use shared_parameters, only: LOCAL_PATH, NPROC + use constants, only: myrank +#endif + + implicit none + + real(kind=CUSTOM_REAL),dimension(:), intent(in) :: dump_array + character(len=MAX_STRING_LEN), intent(in) :: dset_name + +#if defined(USE_HDF5) + character(len=MAX_STRING_LEN) :: filename + integer, dimension(0:NPROC-1) :: offset + + ! MPI variables + integer :: info, comm + + ! hdf5 valiables + character(len=64) :: tempstr + + ! flag if dataset exists + logical :: exists = .false. + + ! saves mesh file external_mesh.h5 + tempstr = "/external_mesh.h5" + filename = LOCAL_PATH(1:len_trim(LOCAL_PATH))//trim(tempstr) + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize h5 object + call h5_initialize() + + call h5_set_mpi_info(comm, info, myrank, NPROC) + + ! get offset info + call gather_all_all_singlei(size(dump_array), offset, NPROC) + + ! make dataset + if (myrank == 0) then + call h5_open_file(filename) + ! check if dataset exists + call h5_check_dataset_exists(dset_name, exists) + if (.not. exists) then + call h5_create_dataset_gen(dset_name, (/sum(offset(:))/), 1, CUSTOM_REAL) + endif + call h5_close_file() + endif + call synchronize_all() + + ! open file + call h5_open_file_p_collect(filename) + call h5_write_dataset_1d_r_collect_hyperslab(dset_name, dump_array, (/sum(offset(0:myrank-1))/),.true.) + call h5_close_file_p() + + call h5_finalize() +#else + ! no compilation support + ! to avoid compiler warnings + real(kind=CUSTOM_REAL) :: dummy + character(len=1) :: c_dummy + + dummy = dump_array(1) + c_dummy = dset_name(1:1) + + ! safety stop + stop 'Error HDF5 manager: write_checkmesh_data_hdf5() called without compilation support' +#endif + + end subroutine write_checkmesh_data_hdf5 + +! +!------------------------------------------------------------------------------- +! + + subroutine write_checkmesh_xdmf_hdf5(NSPEC_AB) + +#if defined(USE_HDF5) + use constants, only: myrank + use shared_parameters +#endif + implicit none + + integer, intent(in) :: NSPEC_AB + +#if defined(USE_HDF5) + character(len=MAX_STRING_LEN) :: fname_xdmf_checkmesh,fname_h5_database_xdmf,fname_h5_extmesh_xdmf + integer, dimension(0:NPROC-1) :: nelms, nnodes + character(len=20) :: type_str,nelm_str,nnode_str + + ! gather number of elements in each proc + call gather_all_singlei(NSPEC_AB, nelms, NPROC) + + ! count and gather the number of control nodes in each proc + call gather_all_singlei(xdmf_mesh_nnodes, nnodes, NPROC) + + if (myrank == 0) then + ! writeout xdmf file for surface movie + fname_xdmf_checkmesh = trim(LOCAL_PATH)//"/checkmesh.xmf" + fname_h5_database_xdmf = "./Database.h5" ! relative to checkmesh.xmf file + fname_h5_extmesh_xdmf = "./external_mesh.h5" + + open(unit=xdmf_mesh, file=trim(fname_xdmf_checkmesh), recl=512) + + ! definition of topology and geometry + ! refer only control nodes (8 or 27) as a coarse output + ! data array need to be extracted from full data array on GLL points + write(xdmf_mesh,'(a)') '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + !write(xdmf_mesh,*) '' + + nelm_str = i2c(sum(nelms(:))) + nnode_str = i2c(sum(nnodes(:))) + + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) ' '//trim(fname_h5_database_xdmf)//':/elm_conn_xdmf' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) ' '//trim(fname_h5_database_xdmf)//':/nodes_coords' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + + type_str = "res_Courant_number" + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) ' '//trim(fname_h5_extmesh_xdmf)//':/'//trim(type_str) + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + + type_str = "res_minimum_period" + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) ' '//trim(fname_h5_extmesh_xdmf)//':/'//trim(type_str) + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + + !write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + write(xdmf_mesh,*) '' + + close(xdmf_mesh) + endif + +#else + ! no compilation support + ! to avoid compiler warnings + integer :: dummy + + dummy = NSPEC_AB + + ! safety stop + stop 'Error HDF5 manager: write_checkmesh_xdmf_hdf5() called without compilation support' +#endif + + end subroutine write_checkmesh_xdmf_hdf5 + + +!------------------------------------------------------------------------------- +! +! HDF5 wrapper routines (only available with HDF5 compilation support) +! +!------------------------------------------------------------------------------- + +#if defined(USE_HDF5) +! only available with HDF5 compilation support + + + function i2c(k) result(str) + ! "Convert an integer to string." + implicit none + integer, intent(in) :: k + character(len=20) str + write (str, "(i20)") k + str = adjustl(str) + end function i2c + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_finalize() + implicit none + ! close Fortran interface + call h5close_f(error) + call check_error() + end subroutine h5_finalize + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_check_collective(dataset_name) + implicit none + character(len=*), intent(in) :: dataset_name + integer :: io_mode + call h5pget_mpio_actual_io_mode_f(plist_id,io_mode,error) + if (error /= 0) write(*,*) 'hdf5 get_mpio_actual_io_mode failed for ',trim(dataset_name) + !if (io_mode == H5D_MPIO_NO_COLLECTIVE_F) print *, & + ! "collective read/write not possible for dataset: ", dataset_name + call check_error() + end subroutine h5_check_collective + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_check_arr_dim(dim) + implicit none + integer(kind=HSIZE_T), dimension(:), intent(in) :: dim + integer :: i + ! if one of the dimension is 0, cancel space_id and file_id + ! loop all elements in dim + do i = 1, size(dim) + if (dim(i) == 0) then + call h5sselect_none_f(mem_dspace_id, error) + call check_error() + call h5sselect_none_f(file_dspace_id, error) + call check_error() + endif + enddo + end subroutine h5_check_arr_dim + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_file(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5fcreate_f(trim(file_path), H5F_ACC_TRUNC_F, file_id, error) + if (error /= 0) then + print *,'Error file create: ', trim(file_path) + print * + print *,'check if path exists: ', trim(file_path) + stop 'Error file create h5' + endif + end subroutine h5_create_file + +! +!------------------------------------------------------------------------------- +! + subroutine h5_create_or_open_file(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + logical :: file_exists + + ! set the target file path + file_path = fpath_in + + ! check if file exists + inquire(file=trim(file_path), exist=file_exists) + if (file_exists) then + call h5fopen_f(trim(file_path), H5F_ACC_RDWR_F, file_id, error) + if (error /= 0) then + print *,'Error file open: ', trim(file_path) + stop 'Error file open h5' + endif + else + call h5fcreate_f(trim(file_path), H5F_ACC_TRUNC_F, file_id, error) + if (error /= 0) then + print *,'Error file create: ', trim(file_path) + print * + print *,'check if path exists: ', trim(file_path) + stop 'Error file create h5' + endif + endif + end subroutine h5_create_or_open_file + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_file(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5fopen_f(trim(file_path), H5F_ACC_RDWR_F, file_id, error) + if (error /= 0) then + print *, 'Error file open: ', trim(file_path) + stop 'Error file open h5' + endif + end subroutine h5_open_file + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_file() + implicit none + call h5fclose_f(file_id, error) + if (error /= 0) write(*,*) 'error while closing a file.' + call check_error() + end subroutine h5_close_file + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_file_p() + implicit none + call h5_close_file_prop_list() + call h5fclose_f(file_id, error) + if (error /= 0) write(*,*) 'error while closing a file.' + call check_error() + end subroutine h5_close_file_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_group(group_name) + implicit none + character(len=*), intent(in) :: group_name + call h5gcreate_f(file_id, trim(group_name), group_id, error) + if (error /= 0) write(*,*) 'error while creating a group, ', group_name + call check_error() + call h5gclose_f(group_id, error) + if (error /= 0) write(*,*) 'error while closing a group, ' , group_name + call check_error() + end subroutine h5_create_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_subgroup(group_name) + implicit none + character(len=*), intent(in) :: group_name + integer(HID_T) :: temp_group_id + + call h5gcreate_f(group_id, trim(group_name), temp_group_id, error) + if (error /= 0) write(*,*) 'error while creating a subgroup, ', group_name + call check_error() + call h5gclose_f(temp_group_id, error) + if (error /= 0) write(*,*) 'error while closing a subgroup, ' , group_name + call check_error() + end subroutine h5_create_subgroup + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_group(group_name) + implicit none + character(len=*), intent(in) :: group_name + ! open group + call h5gopen_f(file_id, trim(group_name), group_id, error) ! group open + if (error /= 0) write(*,*) 'hdf5 open group failed for ', trim(group_name) + call check_error() + ! stores name + store_group_name = group_name + end subroutine h5_open_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_subgroup(group_name) + implicit none + character(len=*), intent(in) :: group_name + integer(HID_T) :: temp_group_id + ! open subgroup + call h5gopen_f(group_id, trim(group_name), temp_group_id, error) ! group open + if (error /= 0) write(*,*) 'hdf5 open subgroup failed for ', trim(group_name) + call check_error() + ! put the group id of the first level becomes parent group + ! only while the group at second level exists + parent_group_id = group_id + group_id = temp_group_id + end subroutine h5_open_subgroup + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_or_create_group(group_name) + ! Check if group with the given name exists. Create it if it doesn't, + ! open it if it does. + implicit none + character(len=*), intent(in) :: group_name + ! Variable for checking if a group exists or not + logical :: group_exists + ! check group + call h5_check_group(group_name, group_exists) + ! open or create + if (group_exists) then + call h5gopen_f(file_id, group_name, group_id, error) + else + call h5gcreate_f(file_id, group_name, group_id, error) + endif + call check_error() + end subroutine h5_open_or_create_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_check_group(group_name,group_exists) + ! Check if group with the given name exists. Create it if it doesn't, + ! open it if it does. + implicit none + character(len=*), intent(in) :: group_name + logical, intent(out) :: group_exists + ! Variable for checking if a group exists or not + group_exists = .false. + ! check + call h5lexists_f(file_id, group_name, group_exists, error) + call check_error() + end subroutine h5_check_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_group() + implicit none + call h5gclose_f(group_id, error) ! group open + if (error /= 0) write(*,*) 'hdf5 close group failed' + call check_error() + end subroutine h5_close_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_subgroup() + implicit none + call h5gclose_f(group_id, error) ! group open + if (error /= 0) write(*,*) 'hdf5 close group failed' + call check_error() + ! stores group + group_id = parent_group_id + end subroutine h5_close_subgroup + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_set_group_name(group_name) + implicit none + character(len=*), intent(in) :: group_name + store_group_name = group_name + end subroutine h5_set_group_name + +! +!------------------------------------------------------------------------------- +! + + ! open dataset. group need to be opened before. + subroutine h5_open_dataset(dataset_name) + implicit none + character(len=*), intent(in) :: dataset_name + + call h5dopen_f(group_id, trim(dataset_name), dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 open dataset failed for ', dataset_name + call check_error() + end subroutine h5_open_dataset + +! +!------------------------------------------------------------------------------- +! + + ! open dataset without open group + subroutine h5_open_dataset2(dataset_name) + implicit none + character(len=*), intent(in) :: dataset_name + + call h5dopen_f(file_id, trim(dataset_name), dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 open dataset2 failed for ', dataset_name + call check_error() + end subroutine h5_open_dataset2 + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_dataset() + implicit none + call h5dclose_f(dataset_id, error) ! group open + if (error /= 0) write(*,*) 'hdf5 close dataset failed' + call check_error() + end subroutine h5_close_dataset + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_check_dataset_exists(dataset_name, exists) + implicit none + character(len=*), intent(in) :: dataset_name + logical, intent(out) :: exists + + call h5lexists_f(file_id, dataset_name, exists, error) + call check_error() + end subroutine h5_check_dataset_exists + + +!------------------------------------------------------------------------------- +! +! serial write routines +! +!------------------------------------------------------------------------------- + + ! dataset writer for 1d integer array + subroutine h5_write_dataset_1d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + dim = size(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_1d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_1d_i_no_group(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + dim = size(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + ! closes dataset + call h5_close_dataset() + end subroutine h5_write_dataset_1d_i_no_group + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 1d custom real array + subroutine h5_write_dataset_1d_d(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + dim = size(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + if (CUSTOM_REAL == 4) then + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + else + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + endif + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_1d_d + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 1d custom real array without grouping + subroutine h5_write_dataset_1d_d_no_group(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + dim = size(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + if (CUSTOM_REAL == 4) then + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + else + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + endif + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + ! closes dataset + call h5_close_dataset() + end subroutine h5_write_dataset_1d_d_no_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_1d_c(dataset_name, data_in) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), dimension(:), intent(in) :: data_in + integer(HID_T) :: dspace_id ! dataspace id is local. + character(len=str_len), dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer :: i + + dim = shape(data_in) + + allocate(data(dim(1)),stat=error) + ! fill blancs after each string to have the same length of strings + do i = 1, dim(1) + data(i) = data_in(i)//repeat(' ',str_len-len(data_in(i))) + enddo + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(group_id, trim(dataset_name), str_type, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, str_type, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + deallocate(data, stat=error) + + end subroutine h5_write_dataset_1d_c + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_1d_c_no_group(dataset_name, data_in) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), dimension(:), intent(in) :: data_in + integer(HID_T) :: dspace_id ! dataspace id is local. + character(len=str_len), dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer :: i + + dim = shape(data_in) + + allocate(data(dim(1)),stat=error) + ! fill blancs after each string to have the same length of strings + do i = 1, dim(1) + data(i) = data_in(i)//repeat(' ',str_len-len(data_in(i))) + enddo + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(file_id, trim(dataset_name), str_type, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, str_type, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + ! closes dataset + call h5_close_dataset() + deallocate(data, stat=error) + end subroutine h5_write_dataset_1d_c_no_group + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 2d integer array + subroutine h5_write_dataset_2d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_2d_i + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 2d double array + subroutine h5_write_dataset_2d_d(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_2d_d + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 2d double array + subroutine h5_write_dataset_2d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + if (CUSTOM_REAL == 4) then + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + else + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + endif + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_2d_r + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 2d double array + subroutine h5_write_dataset_2d_r_no_group(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + if (CUSTOM_REAL == 4) then + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + else + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + endif + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + ! closes dataset + call h5_close_dataset() + end subroutine h5_write_dataset_2d_r_no_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_4d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + if (CUSTOM_REAL == 4) then + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + else + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + endif + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_4d_r + +! +!------------------------------------------------------------------------------- +! + + ! dataset writer for 2d character array + subroutine h5_write_dataset_2d_c(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), dimension(:,:), intent(in) :: data + integer(HID_T) :: dspace_id ! dataspace id is local. + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + ! write string array for undef_mat_prop + dim = shape(data) + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5dcreate_f(group_id, trim(dataset_name), str_type_max, dspace_id, dataset_id, error) + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5dwrite_f(dataset_id, str_type_max, data, dim, error) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', dataset_name + call check_error() + end subroutine h5_write_dataset_2d_c + +! +!------------------------------------------------------------------------------- +! + + ! set attribute to a dataset + subroutine h5_add_attribute_i(attribute_name, data) + implicit none + integer(HID_T) :: aspace_id, atype_id, attr_id ! attribute id + integer :: rank = 1 ! attribute rank + integer, dimension(:), intent(in) :: data + integer(HSIZE_T) :: taglen + character(len=*), intent(in) :: attribute_name + integer(HSIZE_T), dimension(1) :: dim + + dim = shape(data) + taglen = len(trim(attribute_name)) + + call h5screate_simple_f(rank, dim, aspace_id, error) + if (error /= 0) write(*,*) 'hdf5 screate failed for attribute, ', attribute_name + call check_error() + call h5tcopy_f(H5T_NATIVE_INTEGER, atype_id, error) ! for a string tag of attribute value + if (error /= 0) write(*,*) 'hdf5 tcopy failed for attribute, ', attribute_name + call check_error() + call h5tset_size_f(atype_id, taglen, error) + if (error /= 0) write(*,*) 'hdf5 set_size failed for attribute, ', attribute_name + call check_error() + ! here the attribute is written on the current openning dataset + call h5acreate_f(dataset_id, trim(attribute_name), atype_id, aspace_id, attr_id, error) + if (error /= 0) write(*,*) 'hdf5 acreate failed for attribute, ', attribute_name + call check_error() + call h5awrite_f(attr_id, atype_id, data, dim, error) ! write + if (error /= 0) write(*,*) 'hdf5 awrite failed for attribute, ', attribute_name + call check_error() + call h5sclose_f(aspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace closing failed for ', attribute_name + call check_error() + call h5aclose_f(attr_id, error) + if (error /= 0) write(*,*) 'hdf5 aclose failed for ', attribute_name + call check_error() + call h5tclose_f(atype_id, error) + if (error /= 0) write(*,*) 'hdf5 tclose failed for ', attribute_name + call check_error() + end subroutine h5_add_attribute_i + + +!------------------------------------------------------------------------------- +! +! parallel routines +! +!------------------------------------------------------------------------------- + + subroutine h5_set_mpi_info(comm, info, rank, nproc) + implicit none + integer, intent(in) :: comm, info, rank, nproc + + this_comm = comm + this_info = info + this_rank = rank + total_proc = nproc + end subroutine h5_set_mpi_info + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_set_sieve_buffer_size() + implicit none + integer(hsize_t) :: buf_size = 1*1024*1024 ! 1 MB + integer(hsize_t) :: alig_size = 1*1024*1024 ! 1 MB + call h5pset_sieve_buf_size_f(fplist_id, buf_size, error) ! buf_size may vary depending on machiens + call check_error() + call h5pset_alignment_f(fplist_id, buf_size, alig_size, error) + call check_error() + end subroutine h5_set_sieve_buffer_size + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_set_buffer_size() + implicit none + integer(hsize_t) :: buf_size = 1*1024*1024 ! 1 MB + call h5pset_buffer_f(plist_id, buf_size, error) + call check_error() + end subroutine h5_set_buffer_size + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_file_prop_list(if_collective) + implicit none + logical, intent(in) :: if_collective + + ! Setup file access property list with parallel I/O access. + call h5pcreate_f(H5P_FILE_ACCESS_F, fplist_id, error) + if (error /= 0) write(*,*) 'hdf5 create plist failed.' + call check_error() + call h5pset_all_coll_metadata_ops_f(fplist_id, if_collective, error) + call check_error() + call h5pset_coll_metadata_write_f(fplist_id, if_collective, error) + call check_error() + call h5pset_fapl_mpio_f(fplist_id, this_comm, this_info, error) + if (error /= 0) write(*,*) 'hdf5 set_fapl failed.' + call check_error() + end subroutine h5_create_file_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_group_prop_list() + implicit none + logical, parameter :: if_collective = .true. + ! Setup file access property list with parallel I/O access. + call h5pcreate_f(H5P_GROUP_ACCESS_F, gplist_id, error) + if (error /= 0) write(*,*) 'hdf5 create group plist failed.' + call check_error() + call h5pset_all_coll_metadata_ops_f(gplist_id, if_collective, error) + call check_error() + call h5pset_coll_metadata_write_f(gplist_id, if_collective, error) + if (error /= 0) write(*,*) 'hdf5 group set_all_coll_metadata_ops failed.' + call check_error() + end subroutine h5_open_group_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_group_prop_list() + implicit none + ! Setup file access property list with parallel I/O access. + call h5pcreate_f(H5P_GROUP_CREATE_F, gplist_id, error) + if (error /= 0) write(*,*) 'hdf5 create group plist failed.' + call check_error() + end subroutine h5_create_group_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_dataset_prop_list(if_collective) + implicit none + logical, intent(in) :: if_collective + ! Setup file access property list with parallel I/O access. + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + if (error /= 0) write(*,*) 'hdf5 create dataset plist failed.' + call check_error() + ! use larger buffer size from the default + call h5_set_buffer_size() + if (if_collective) then + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + else + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_INDEPENDENT_F, error) + endif + if (error /= 0) write(*,*) 'hdf5 dataset set_dxpl_mpio failed.' + call check_error() + end subroutine h5_create_dataset_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_prop_list(dataset_name) + implicit none + character(len=*), intent(in) :: dataset_name + !print *,"DEBUG closing dataset prop: ", dataset_name + ! check if the latest dataset transfer has been done in collective mode + call h5_check_collective(dataset_name) + call h5pclose_f(plist_id, error) ! property list can be closed soon + if (error /= 0) write(*,*) 'hdf5 close_prop_list failed for ',trim(dataset_name) + call check_error() + end subroutine h5_close_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_prop_list_nocheck(dataset_name) + implicit none + character(len=*), intent(in) :: dataset_name + !print *,"DEBUG closing dataset prop: ", dataset_name + ! check if the latest dataset transfer has been done in collective mode + call h5pclose_f(plist_id, error) ! property list can be closed soon + if (error /= 0) write(*,*) 'hdf5 close_prop_list failed for ',trim(dataset_name) + call check_error() + end subroutine h5_close_prop_list_nocheck + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_file_prop_list() + implicit none + call h5pclose_f(fplist_id, error) ! property list can be closed soon + if (error /= 0) write(*,*) 'hdf5 close_file_prop_list failed.' + call check_error() + end subroutine h5_close_file_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_close_group_prop_list() + implicit none + call h5pclose_f(gplist_id, error) ! property list can be closed soon + if (error /= 0) write(*,*) 'hdf5 close_prop_list failed.' + call check_error() + end subroutine h5_close_group_prop_list + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_file_p(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5_create_file_prop_list(.false.) + call h5fcreate_f(trim(file_path), H5F_ACC_TRUNC_F, file_id, error, creation_prp=H5P_DEFAULT_F, access_prp=fplist_id) + if (error /= 0) write(*,*) 'hdf5 create file p failed.' + call check_error() + end subroutine h5_create_file_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_file_p_collect(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5_create_file_prop_list(.true.) + call h5fcreate_f(trim(file_path), H5F_ACC_TRUNC_F, file_id, error, creation_prp=H5P_DEFAULT_F, access_prp=fplist_id) + if (error /= 0) write(*,*) 'hdf5 create file p failed.' + call check_error() + end subroutine h5_create_file_p_collect + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_file_p(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5_create_file_prop_list(.false.) + call h5fopen_f(trim(file_path), H5F_ACC_RDWR_F, file_id, error, access_prp=fplist_id) + if (error /= 0) write(*,*) 'hdf5 open file p failed.' + call check_error() + end subroutine h5_open_file_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_file_p_collect(fpath_in) + implicit none + character(len=*), intent(in) :: fpath_in + + ! set the target file path + file_path = fpath_in + + call h5_create_file_prop_list(.true.) + call h5_set_sieve_buffer_size() + + call h5fopen_f(trim(file_path), H5F_ACC_RDWR_F, file_id, error, access_prp=fplist_id) + if (error /= 0) write(*,*) 'hdf5 open file p failed.' + call check_error() + end subroutine h5_open_file_p_collect + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_dataset_gen(dataset_name, dim_in, rank, dtype_id) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in) :: dim_in + + integer(HSIZE_T), dimension(size(dim_in)) :: dim + integer, intent(in) :: dtype_id ! 1:int, 4:real4, 8:real8, + integer, intent(in) :: rank + + integer(HID_T) :: dspace_id + !logical :: if_chunk = .true. + !integer :: i + + dim = dim_in ! convert data type + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5pcreate_f(H5P_DATASET_CREATE_F, plist_id, error) + call check_error() + + ! chunk size setting + !do i = 1, rank + ! if (dim(i) <= 0) then + ! if_chunk = .false. + ! print *, "dataset not chunk set: ", dataset_name + ! endif + !enddo + !if (if_chunk) call h5pset_chunk_f(plist_id,rank,dim,error) + + if (dtype_id == 0) then ! bool uses integer + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 1) then ! integer + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 2) then ! character + call h5dcreate_f(file_id, trim(dataset_name), str_type, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 4) then ! real + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 8) then ! double + call h5dcreate_f(file_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else + print *, "specified dtype_id is not implemented yet for hdf5 io. aborting..." + stop 'Invalid dtype_id, not implemented yet in h5_create_dataset_gen() routine' + endif + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + + call h5_close_prop_list_nocheck(dataset_name) + + call h5dclose_f(dataset_id,error) + if (error /= 0) write(*,*) 'hdf5 dataset close failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace close failed for ', dataset_name + call check_error() + + end subroutine h5_create_dataset_gen + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_dataset_gen_in_group(dataset_name, dim_in, rank, dtype_id) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in) :: dim_in + + integer(HSIZE_T), dimension(size(dim_in)) :: dim + integer, intent(in) :: dtype_id ! 1:int, 4:real4, 8:real8, + integer, intent(in) :: rank + + integer(HID_T) :: dspace_id + + dim = dim_in ! convert data type + + call h5screate_simple_f(rank, dim, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5pcreate_f(H5P_DATASET_CREATE_F, plist_id, error) + call check_error() + if (dtype_id == 0) then ! logial uses integer + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 1) then ! integer + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 2) then ! character + call h5dcreate_f(group_id, trim(dataset_name), str_type, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 4) then ! real + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 8) then ! double + call h5dcreate_f(group_id, trim(dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else + print *, "Error: specified dtype_id is not implemented yet for hdf5 io. aborting..." + stop 'Invalid dtype_id, not implemented yet in h5_create_dataset_gen_in_group()' + endif + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + + call h5_close_prop_list_nocheck(dataset_name) + + call h5dclose_f(dataset_id,error) + if (error /= 0) write(*,*) 'hdf5 dataset close failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace close failed for ', dataset_name + call check_error() + end subroutine h5_create_dataset_gen_in_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_create_group_p(group_name) + implicit none + character(len=*), intent(in) :: group_name + + call h5_create_group_prop_list() + call h5gcreate_f(file_id, trim(group_name), group_id, error, gcpl_id=gplist_id) + if (error /= 0) write(*,*) 'error while creating a group, ', group_name + call check_error() + call h5gclose_f(group_id, error) + if (error /= 0) write(*,*) 'error while closing a group, ' , group_name + call check_error() + call h5_close_group_prop_list() + end subroutine h5_create_group_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_open_group_p(group_name) + implicit none + character(len=*), intent(in) :: group_name + call h5_open_group_prop_list() + call h5gopen_f(file_id, trim(group_name), group_id, error, gapl_id=gplist_id) + if (error /= 0) write(*,*) 'hdf5 open group failed for ', group_name + call check_error() + call h5_close_group_prop_list() + store_group_name = group_name + end subroutine h5_open_group_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_scalar_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, intent(out) :: data + integer, dimension(1) :: rdata + integer(HSIZE_T), dimension(1) :: dim + dim = shape(rdata) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, rdata, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + data = rdata(1) + end subroutine h5_read_dataset_p_scalar_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_scalar_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), intent(out) :: data + real(kind=CUSTOM_REAL), dimension(1) :: rdata + integer(HSIZE_T), dimension(1) :: dim + dim = shape(rdata) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, rdata, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, rdata, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + data = rdata(1) + end subroutine h5_read_dataset_p_scalar_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_1d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(inout) :: data + integer(HSIZE_T), dimension(1) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_1d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_1d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(inout) :: data + integer(HSIZE_T), dimension(1) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_1d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_1d_l(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(inout) :: data + integer, dimension(:), allocatable :: ldata + integer(HSIZE_T), dimension(1) :: dim + integer :: lsize + dim = shape(data) + lsize = size(data) + allocate(ldata(lsize),stat=error) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, ldata, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + call int_array2bool(ldata, data) + + deallocate(ldata, stat=error) + end subroutine h5_read_dataset_p_1d_l + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_2d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(inout) :: data + integer(HSIZE_T), dimension(2) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_2d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_2d_d(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(inout) :: data + integer(HSIZE_T), dimension(2) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_2d_d + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_2d_c(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), dimension(:,:), intent(inout) :: data + integer(HSIZE_T), dimension(2) :: dim + + ! reads in string array for undef_mat_prop + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, str_type_max, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_2d_c + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_2d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(inout) :: data + integer(HSIZE_T), dimension(2) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_2d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_3d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(inout) :: data + integer(HSIZE_T), dimension(3) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_3d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_3d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(inout) :: data + integer(HSIZE_T), dimension(3) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_3d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_4d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(inout) :: data + integer(HSIZE_T), dimension(4) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_4d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_5d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(inout) :: data + integer(HSIZE_T), dimension(5) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, xfer_prp=plist_id) + endif + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_5d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_attribute_p(attribute_name, dataset_name, data) + implicit none + character(len=*), intent(in) :: attribute_name + character(len=*), intent(in) :: dataset_name + integer(HID_T) :: attr_id + integer, dimension(:), intent(inout) :: data + integer(HSIZE_T), dimension(1) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5aopen_f(dataset_id, attribute_name, attr_id, error) + call check_error() + call h5aread_f(attr_id, H5T_NATIVE_INTEGER, data, dim, error) + call check_error() + call h5aclose_f(attr_id,error) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_attribute_p + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_p_4d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(inout) :: data + integer(HSIZE_T), dimension(4) :: dim + dim = shape(data) + + call h5dopen_f(group_id, dataset_name, dataset_id, error) + call check_error() + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, xfer_prp=plist_id) + call check_error() + call h5pclose_f(plist_id, error) + call check_error() + call h5dclose_f(dataset_id, error) + call check_error() + end subroutine h5_read_dataset_p_4d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_scalar_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + !dim = shape(data) + dim = (/1/) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_scalar_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_scalar_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + !dim = shape(data) + dim = (/1/) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_scalar_i_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_scalar_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + !dim = shape(data) + dim = (/1/) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_scalar_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_1d_l_collect_hyperslab(dataset_name, data_out, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(inout), target :: data_out + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer, dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data_out) + offset = offset_in ! convert data type + allocate(data(dim(1))) + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + f_ptr = c_loc(data(1)) + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + + call int_array2bool(data, data_out) + deallocate(data) + end subroutine h5_read_dataset_1d_l_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_1d_l_collect_hyperslab_in_group(dataset_name, data_out, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(inout), target :: data_out + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer, dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data_out) + offset = offset_in ! convert data type + allocate(data(dim(1))) + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + f_ptr = c_loc(data(1)) + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + + call int_array2bool(data, data_out) + deallocate(data) + end subroutine h5_read_dataset_1d_l_collect_hyperslab_in_group + +! +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_1d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + call h5_check_arr_dim(dim) + + ! write array using Fortran pointer + !call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! use F2003 API + f_ptr = c_loc(data(1)) + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_1d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_1d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + call h5_check_arr_dim(dim) + + ! write array using Fortran pointer + !call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! use F2003 API + f_ptr = c_loc(data(1)) + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_1d_i_collect_hyperslab_in_group + +! +! +!------------------------------------------------------------------------------- +! + + ! store local 1d array to global 1d array + subroutine h5_read_dataset_1d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_1d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 1d array to global 1d array + subroutine h5_read_dataset_1d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_1d_r_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + ! store local 1d array to global 1d array + subroutine h5_read_dataset_1d_d_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_1d_d_collect_hyperslab_in_group + +! +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_2d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_2d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_2d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_2d_i_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + ! store local 2d array to global 2d array + subroutine h5_read_dataset_2d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1)! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_2d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 2d array to global 2d array + subroutine h5_read_dataset_2d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1)! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_2d_r_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_2d_d_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1)! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_2d_d_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_3d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_3d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_3d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_3d_i_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_3d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_3d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_3d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_3d_r_collect_hyperslab_in_group + +! +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_4d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_4d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_4d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1)) + + ! write array using Fortran pointer + call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_4d_i_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_4d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_4d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_4d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_4d_r_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_5d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 5 + integer(HSIZE_T), dimension(5) :: dim + integer(HSIZE_T), dimension(5) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(5) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + count(5) = dim(5) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_5d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_read_dataset_5d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(inout), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 5 + integer(HSIZE_T), dimension(5) :: dim + integer(HSIZE_T), dimension(5) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(5) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + count(4) = dim(4) ! NSTEP partial + count(5) = dim(5) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + f_ptr = c_loc(data(1,1,1,1,1)) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dread_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dread_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_read_dataset_5d_r_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! +! parallel write routines +! +!------------------------------------------------------------------------------- + +! +! collective writers +! + + subroutine h5_write_dataset_p_1d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in), target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + ! dummy array for generate 0 length dataset + integer, dimension(1) :: dummy_1d_array = (/0/) + + dim = shape(data) + + ! add dummy 0 for no_element array + if (size(data) == 0) then + dim = 1 + endif + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + if (size(data) == 0) then + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, dummy_1d_array, dim, error,xfer_prp=plist_id) + else + f_ptr = c_loc(data(1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_1d_i + +! +!------------------------------------------------------------------------------- +! + + ! write 1d integer and attribute + subroutine h5_write_dataset_p_1d_ia(dataset_name, data, attribute_name, attr_data) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), intent(in) :: attribute_name + integer, dimension(:), intent(in), target :: data + integer, intent(in) :: attr_data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + ! dummy array for generate 0 length dataset + integer, dimension(1) :: dummy_1d_array = (/0/) + + dim = shape(data) + + ! add dummy 0 for no_element array + if (size(data) == 0) then + dim = 1 + endif + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + if (size(data) == 0) then + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, dummy_1d_array, dim, error,xfer_prp=plist_id) + else + f_ptr = c_loc(data(1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + ! add attribute + call h5_add_attribute_i(attribute_name, (/attr_data/)) + + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_1d_ia + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_1d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in), target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, CUSTOM_REAL, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1)) + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_1d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_2d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(in), target :: data + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_2d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_2d_ia(dataset_name, data, attribute_name, attr_data) + implicit none + character(len=*), intent(in) :: dataset_name + character(len=*), intent(in) :: attribute_name + integer, dimension(:,:), intent(in), target :: data + integer, intent(in) :: attr_data + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + ! add attribute + call h5_add_attribute_i(attribute_name, (/attr_data/)) + + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_2d_ia + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_2d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in), target :: data + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, CUSTOM_REAL, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1)) + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_2d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_2d_d(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(in), target :: data + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 8, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_2d_d + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_3d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(in), target :: data + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1,1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_3d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_3d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(in), target :: data + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, CUSTOM_REAL, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1,1)) + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_3d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_4d_i(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(in), target :: data + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1,1,1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_4d_i + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_4d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(in), target :: data + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, CUSTOM_REAL, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1,1,1)) + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_4d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_5d_r(dataset_name, data) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(in), target :: data + integer :: rank = 5 + integer(HSIZE_T), dimension(5) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, CUSTOM_REAL, file_id) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + + ! write array using Fortran pointer + f_ptr = c_loc(data(1,1,1,1,1)) + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, f_ptr, error, & + xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, f_ptr, error, & + xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + end subroutine h5_write_dataset_p_5d_r + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_p_1d_l(dataset_name, data_in) + ! writer for logical array + ! logical array will be converted to integer array before write + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(in) :: data_in + integer, dimension(:), allocatable, target :: data + + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + dim = shape(data_in) + + ! create dataset + call create_dataset_collect(dataset_name, dim, rank, 1, file_id) + + allocate(data(size(data_in)), stat=error) + call bool_array2integer(data_in, data) + + write(tempstr, "(i6.6)") this_rank + group_name = gname_proc_head // trim(tempstr) + + !! create datasets of all processes for independent write + call h5_open_group(group_name) + call h5_open_dataset(trim(dataset_name)) + + call h5pcreate_f(H5P_DATASET_XFER_F, plist_id, error) + call check_error() + call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + !call h5pset_dxpl_mpio_f(plist_id, H5FD_MPIO_COLLECTIVE_F, error) + call check_error() + ! write array using Fortran pointer + f_ptr = c_loc(data(1)) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, & + xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5_close_dataset() + call h5_close_group() + deallocate(data) + end subroutine h5_write_dataset_p_1d_l + +! +!------------------------------------------------------------------------------- +! + + ! from 1d local array to 2d global array + subroutine h5_write_dataset_1d_to_2d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in), target :: data + integer, dimension(2), intent(in) :: offset_in ! the position where the datablock is inserted + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset + + dim = size(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! size of data array inserted. + count(1) = dim(1) + count(2) = 1 + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_to_2d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 2d array to global 3d array + subroutine h5_write_dataset_2d_to_3d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = 3! NDIM + count(2) = dim(2) ! NSTEP partial + count(3) = 1 ! nrec + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_to_3d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_1d_l_collect_hyperslab(dataset_name, data_in, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(in), target :: data_in + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer, dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data_in) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! if_collective == .false., this function gather all data from procs then write in a file at once + + ! select a place where data is inserted. + count(1) = dim(1) + allocate(data(dim(1))) + + ! convert logical array to integer array + call bool_array2integer(data_in, data) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + deallocate(data) + + end subroutine h5_write_dataset_1d_l_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_1d_l_collect_hyperslab_in_group(dataset_name, data_in, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + logical, dimension(:), intent(in), target :: data_in + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer, dimension(:), allocatable, target :: data + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data_in) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! if_collective == .false., this function gather all data from procs then write in a file at once + + ! select a place where data is inserted. + count(1) = dim(1) + allocate(data(dim(1))) + + ! convert logical array to integer array + call bool_array2integer(data_in, data) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + deallocate(data) + + end subroutine h5_write_dataset_1d_l_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + + ! store local 1d array to global 1d array + subroutine h5_write_dataset_1d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + call h5_check_arr_dim(dim) + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! use F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 1d array to global 1d array + subroutine h5_write_dataset_1d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + call h5_check_arr_dim(dim) + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! use F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_i_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + + subroutine h5_write_dataset_1d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + !call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1)),error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 1d array to global 1d array + subroutine h5_write_dataset_1d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + !call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_r_collect_hyperslab_in_group +! +!------------------------------------------------------------------------------- +! + + +! subroutine h5_write_dataset_1d_d_collect_hyperslab(dataset_name, data, offset_in, if_collective) +! implicit none +! character(len=*), intent(in) :: dataset_name +! double precision, dimension(:), intent(in), target :: data +! integer, dimension(:), intent(in) :: offset_in +! logical, intent(in) :: if_collective +! ! local parameters +! integer :: rank = 1 +! integer(HSIZE_T), dimension(1) :: dim +! integer(HSIZE_T), dimension(1) :: count ! size of hyperslab +! integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted +! +! dim = shape(data) +! offset = offset_in ! convert data type +! +! ! open dataset +! call h5_open_dataset2(trim(dataset_name)) +! +! ! select a place where data is inserted. +! count(1) = dim(1) +! +! ! select hyperslab in the file +! call h5screate_simple_f(rank,count, mem_dspace_id, error) +! call check_error() +! call h5dget_space_f(dataset_id, file_dspace_id, error) +! call check_error() +! call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) +! call check_error() +! call h5_create_dataset_prop_list(if_collective) +! +! ! write array using Fortran pointer +! !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & +! ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) +! call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1)), error, & +! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) +! +! if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name +! call check_error() +! call h5_close_prop_list(dataset_name) +! call h5sclose_f(mem_dspace_id, error) +! call check_error() +! call h5sclose_f(file_dspace_id, error) +! call check_error() +! call h5_close_dataset() +! end subroutine h5_write_dataset_1d_d_collect_hyperslab + + +! +!------------------------------------------------------------------------------- +! + subroutine h5_write_dataset_1d_d_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 1 + integer(HSIZE_T), dimension(1) :: dim + integer(HSIZE_T), dimension(1) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(1) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_1d_d_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + + ! store local 2d array to global 2d array + subroutine h5_write_dataset_2d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_2d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_i_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + ! store local 2d array to global 2d array + subroutine h5_write_dataset_2d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + !call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_r_collect_hyperslab +! +!------------------------------------------------------------------------------- +! + + ! store local 2d array to global 2d array + subroutine h5_write_dataset_2d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + !call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + !call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_r_collect_hyperslab_in_group + +! +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_2d_d_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_d_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_2d_d_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + double precision, dimension(:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 2 + integer(HSIZE_T), dimension(2) :: dim + integer(HSIZE_T), dimension(2) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(2) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! nrec + count(2) = dim(2) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_2d_d_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_3d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_3d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_3d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + !call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, & + ! file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + ! F2003 API + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_3d_i_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + ! store local 3d array to global 3d array + subroutine h5_write_dataset_3d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_3d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + ! store local 3d array to global 3d array + subroutine h5_write_dataset_3d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 3 + integer(HSIZE_T), dimension(3) :: dim + integer(HSIZE_T), dimension(3) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(3) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) ! NDIM + count(2) = dim(2) ! nrec + count(3) = dim(3) ! NSTEP partial + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_3d_r_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + + subroutine h5_write_dataset_4d_i_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_4d_i_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_4d_i_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + integer, dimension(:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + call h5dwrite_f(dataset_id, H5T_NATIVE_INTEGER, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_4d_i_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + + subroutine h5_write_dataset_4d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_4d_r_collect_hyperslab + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_4d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 4 + integer(HSIZE_T), dimension(4) :: dim + integer(HSIZE_T), dimension(4) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(4) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_4d_r_collect_hyperslab_in_group + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_5d_r_collect_hyperslab_in_group(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 5 + integer(HSIZE_T), dimension(5) :: dim + integer(HSIZE_T), dimension(5) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(5) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + count(5) = dim(5) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_5d_r_collect_hyperslab_in_group + + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_write_dataset_5d_r_collect_hyperslab(dataset_name, data, offset_in, if_collective) + implicit none + character(len=*), intent(in) :: dataset_name + real(kind=CUSTOM_REAL), dimension(:,:,:,:,:), intent(in), target :: data + integer, dimension(:), intent(in) :: offset_in + logical, intent(in) :: if_collective + ! local parameters + integer :: rank = 5 + integer(HSIZE_T), dimension(5) :: dim + integer(HSIZE_T), dimension(5) :: count ! size of hyperslab + integer(HSSIZE_T), dimension(5) :: offset ! the position where the datablock is inserted + + dim = shape(data) + offset = offset_in ! convert data type + + ! open dataset + call h5_open_dataset2(trim(dataset_name)) + + ! select a place where data is inserted. + count(1) = dim(1) + count(2) = dim(2) + count(3) = dim(3) + count(4) = dim(4) + count(5) = dim(5) + + ! select hyperslab in the file + call h5screate_simple_f(rank,count, mem_dspace_id, error) + call check_error() + call h5dget_space_f(dataset_id, file_dspace_id, error) + call check_error() + call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error) + call check_error() + call h5_create_dataset_prop_list(if_collective) + + ! write array using Fortran pointer + if (CUSTOM_REAL == 4) then + call h5dwrite_f(dataset_id, H5T_NATIVE_REAL, c_loc(data(1,1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + else + call h5dwrite_f(dataset_id, H5T_NATIVE_DOUBLE, c_loc(data(1,1,1,1,1)), error, & + file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5sclose_f(mem_dspace_id, error) + call check_error() + call h5sclose_f(file_dspace_id, error) + call check_error() + call h5_close_dataset() + end subroutine h5_write_dataset_5d_r_collect_hyperslab + +!------------------------------------------------------------------------------- +! +! other utilities +! +!------------------------------------------------------------------------------- + + subroutine check_error() + implicit none + if (error /= 0) then + print *,'Error: HDF5 routine call got error ',error + stop 'HDF5 error' + endif + end subroutine check_error + +! +!------------------------------------------------------------------------------- +! + + subroutine bool_array2integer(boolarray, intarray) + implicit none + logical, dimension(:), intent(in) :: boolarray + integer, dimension(:), intent(out) :: intarray + integer :: array_size, i + + array_size = size(boolarray) + intarray(:) = 0 + do i = 1, array_size + if (boolarray(i)) then + intarray(i) = 1 + endif + enddo + end subroutine bool_array2integer + +! +!------------------------------------------------------------------------------- +! + + subroutine int_array2bool(intarray, boolarray) + implicit none + integer, dimension(:), intent(in) :: intarray + logical, dimension(:), intent(out) :: boolarray + integer :: array_size,i + + array_size = size(intarray) + boolarray(:) = .false. + + do i = 1, array_size + if (intarray(i) /= 0) then + boolarray(i) = .true. + endif + enddo + end subroutine int_array2bool + +! +!------------------------------------------------------------------------------- +! + + subroutine h5_gather_dsetsize(dimin, data_rank, all_dim) + + implicit none + integer, intent(in) :: data_rank + integer(HSIZE_T), dimension(:), intent(in) :: dimin + integer, dimension(data_rank) :: dim + integer, dimension(data_rank,0:total_proc-1), intent(out) :: all_dim + + dim = dimin ! convert integer 8(HSIZE_T) to 4 + + if (data_rank == 1) then + call gather_all_all_singlei(dim(1),all_dim,total_proc) + !debug + !if (this_rank == 0) then + ! print *, "rank: ", this_rank & + ! ,"dimin", dimin, ", kind: ", kind(dimin) & + ! ,"dim", dim, ", kind: ", kind(dim) & + ! ,"shape dimin", shape(dimin) & + ! ,"data_rank", data_rank & + ! ,"all dim", all_dim & + ! ,"shape all dim", shape(all_dim) & + ! ,"kind all_dim", kind(all_dim(1,0)) & + ! ,"total proc " , total_proc & + ! ,"kind mpi_integer ", kind(MPI_INTEGER) + !endif + else + ! gather only on main rank + !call gather_all_i(dim, data_rank, all_dim, data_rank, total_proc) + ! gather on all processes + call gather_all_all_i(dim, data_rank, all_dim, data_rank, total_proc) + endif + + call synchronize_all() + end subroutine + +! +!------------------------------------------------------------------------------- +! + + subroutine create_dataset_collect(dataset_name, dim, data_rank, dtype_id, base_id) + implicit none + character(len=*), intent(in) :: dataset_name + integer(HSIZE_T), dimension(:), intent(in) :: dim + integer, intent(in) :: dtype_id ! 1:int, 4:real4, 8:real8, + integer, intent(in) :: data_rank + integer(HID_T), intent(in) :: base_id ! base (file_id or group_id) of dataset creation + + integer, dimension(data_rank,0:total_proc-1) :: all_dim + integer(HSIZE_T), dimension(data_rank) :: dim_h5 + integer :: iproc + character(len=128) :: group_and_dataset_name + integer(HID_T) :: dspace_id + character(len=10) :: tempstr + character(len=5) :: gname_proc_head = "proc_" + character(len=64) :: group_name + + ! gather dataset dimension of other processors + call h5_gather_dsetsize(dim, data_rank, all_dim) + ! create all datasets of all process by rank0 + + !print *, dataset_name, ": debug returned all dim", all_dim + do iproc = 0, total_proc -1 + write(tempstr, "(i6.6)") iproc + group_name = gname_proc_head // trim(tempstr) + group_and_dataset_name = trim(group_name) //"/"// dataset_name + + dim_h5 = all_dim(:,iproc) + + call h5screate_simple_f(data_rank, dim_h5, dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace create failed for ', dataset_name + call check_error() + call h5pcreate_f(H5P_DATASET_CREATE_F, plist_id, error) + call check_error() + ! This is required for this data pattern + call H5Pset_alloc_time_f(plist_id, H5D_ALLOC_TIME_EARLY_F, error) + call check_error() + if (dtype_id == 1) then + call h5dcreate_f(base_id, trim(group_and_dataset_name), H5T_NATIVE_INTEGER, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else if (dtype_id == 4) then + call h5dcreate_f(base_id, trim(group_and_dataset_name), H5T_NATIVE_REAL, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + else + call h5dcreate_f(base_id, trim(group_and_dataset_name), H5T_NATIVE_DOUBLE, dspace_id, dataset_id, error, & + dcpl_id=plist_id) + endif + if (error /= 0) write(*,*) 'hdf5 dataset create failed for ', dataset_name + call check_error() + call h5_close_prop_list(dataset_name) + call h5dclose_f(dataset_id,error) + if (error /= 0) write(*,*) 'hdf5 dataset close failed for ', dataset_name + call check_error() + call h5sclose_f(dspace_id, error) + if (error /= 0) write(*,*) 'hdf5 dataspace close failed for ', dataset_name + call check_error() + enddo + + call synchronize_all() + end subroutine create_dataset_collect + +!------------------------------------------------------------------------------- +! +! object-oriented interface +! +!------------------------------------------------------------------------------- +! Fortran 2003 standard +! +! this interface is mainly for testing purposes, and for checking if compilers by now support these features :( +! +! in the past, we had issues with compilers due to keywords like class(). +! here, we use strictly Fortran 2003 features, like polymorphism with 'class()', object destructor procedure by 'final ::', +! and type binding by a pass statement like 'procedure :: . => ..' +! +! for a status report on compiler support, see Fortran+2003+status on: +! https://Fortranwiki.org/ +! Cray, GNU, IBM and Intel compilers should well support these features. + + ! constructors + + function h5io_constructor() result(this) + implicit none + type(h5io) :: this + call h5_initialize() + this%is_initialized = .true. + end function h5io_constructor + + function h5io_constructor_from_file(filename) result(this) + implicit none + type(h5io) :: this + character(*), intent(in) :: filename + call h5_initialize() + this%is_initialized = .true. + call this%open(filename) + this%filename = trim(filename) + end function h5io_constructor_from_file + + ! destructor + + subroutine h5io_destructor(this) + implicit none + type(h5io) :: this + call h5_finalize() + this%is_initialized = .false. + end subroutine h5io_destructor + + ! object procedures + + ! note: with the pass statement "procedure :: open => h5io_open_file", the routine must have at least one argument and + ! the argument 'this' must be defined as polymorphic, i.e., by class() and not type() as above + + subroutine h5io_open_file(this,filename,status) + implicit none + class(h5io) :: this + character(*), intent(in) :: filename + character(*), intent(in), optional :: status + ! checks if initialized + if (.not. this%is_initialized) then + call h5_initialize() + this%is_initialized = .true. + endif + ! only main process creates file if needed + if (present(status)) then + if (status == 'new') call h5_create_file(filename) + endif + ! opens file + call h5_open_file(filename) + this%f_id = file_id + this%filename = trim(filename) + end subroutine h5io_open_file + + subroutine h5io_close_file(this) + implicit none + class(h5io) :: this + call h5_close_file() + this%f_id = file_id + end subroutine h5io_close_file + + subroutine h5io_write_dataset_i(this,dname,data_i,group) + implicit none + class(h5io) :: this + character(len=*), intent(in) :: dname + integer, dimension(:), intent(in) :: data_i + character(len=*), intent(in), optional :: group + ! writes either with or without group + if (present(group)) then + call h5_open_or_create_group(group) + call h5_write_dataset(dname,data_i) + else + call h5_write_dataset_no_group(dname,data_i) + endif + this%d_id = dataset_id + end subroutine h5io_write_dataset_i + + subroutine h5io_write_dataset_r(this,dname,data_r,group) + implicit none + class(h5io) :: this + character(len=*), intent(in) :: dname + real(kind=CUSTOM_REAL), dimension(:), intent(in) :: data_r + character(len=*), intent(in), optional :: group + ! writes either with or without group + if (present(group)) then + call h5_open_or_create_group(group) + call h5_write_dataset(dname,data_r) + else + call h5_write_dataset_no_group(dname,data_r) + endif + this%d_id = dataset_id + end subroutine h5io_write_dataset_r + +#endif + +end module manager_hdf5 + +!----------------------------------------------------- + +#if defined(USE_HDF5) + +! test function for object-oriented interface + + subroutine test_io_hdf5() + use constants, only: myrank + use manager_hdf5, only: h5io + implicit none + type(h5io) :: h5 + integer :: store_x(10) + integer :: i + + ! serial test by main process only + if (myrank /= 0) return + + ! initialize + store_x(:) = (/(i,i = 1,10)/) + + ! hdf5 + ! calls the object constructor, i.e. h5io_constructor() + h5 = h5io() + + ! open file + call h5%open("tmp_test.h5",status='new') ! object function call + + ! write out data + call h5%write("x",store_x) + + ! close file + call h5%close() + + ! h5 goes out of scope, will call the object destructor, i.e., h5io_destructor() + end subroutine test_io_hdf5 + +#endif diff --git a/src/shared/parallel.f90 b/src/shared/parallel.f90 index 45abc0741..ee51ef6f0 100644 --- a/src/shared/parallel.f90 +++ b/src/shared/parallel.f90 @@ -71,6 +71,7 @@ module my_mpi ! my MPI group for simultaneous runs integer :: my_local_mpi_comm_world integer :: my_local_mpi_comm_for_bcast + integer :: my_local_mpi_comm_inter ! MPI subgroup for hdf5 i/o server end module my_mpi @@ -1527,6 +1528,28 @@ subroutine gather_all_i(sendbuf, sendcnt, recvbuf, recvcount, NPROC) end subroutine gather_all_i +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gather_all_all_i(sendbuf, sendcnt, recvbuf, recvcount, NPROC) + + use my_mpi + + implicit none + + integer :: sendcnt, recvcount, NPROC + integer, dimension(sendcnt) :: sendbuf + integer, dimension(recvcount,0:NPROC-1) :: recvbuf + + integer :: ier + + call MPI_ALLGATHER(sendbuf,sendcnt,MPI_INTEGER, & + recvbuf,recvcount,MPI_INTEGER, & + my_local_mpi_comm_world,ier) + + end subroutine gather_all_all_i + ! !------------------------------------------------------------------------------------------------- ! @@ -1549,6 +1572,28 @@ subroutine gather_all_singlei(sendbuf, recvbuf, NPROC) end subroutine gather_all_singlei +! +!------------------------------------------------------------------------------------------------- +! + + subroutine gather_all_all_singlei(sendbuf, recvbuf, NPROC) + + use my_mpi + + implicit none + + integer :: NPROC + integer :: sendbuf + integer, dimension(0:NPROC-1) :: recvbuf + + integer :: ier + + call MPI_ALLGATHER(sendbuf,1,MPI_INTEGER, & + recvbuf,1,MPI_INTEGER, & + my_local_mpi_comm_world,ier) + + end subroutine gather_all_all_singlei + ! !------------------------------------------------------------------------------------------------- ! diff --git a/src/shared/read_parameter_file.F90 b/src/shared/read_parameter_file.F90 index bbbb9477e..b30892b62 100644 --- a/src/shared/read_parameter_file.F90 +++ b/src/shared/read_parameter_file.F90 @@ -245,8 +245,10 @@ subroutine read_parameter_file() call read_value_logical(OUTPUT_SEISMOS_SAC_BINARY, 'OUTPUT_SEISMOS_SAC_BINARY', ier) if (ier /= 0) stop 'an error occurred while reading the parameter file: OUTPUT_SEISMOS_SAC_BINARY' call read_value_logical(OUTPUT_SEISMOS_ASDF, 'OUTPUT_SEISMOS_ASDF', ier) - if (ier /= 0) stop 'an error occurred while reading the parameter file: OUTPUT_ASDF' + if (ier /= 0) stop 'an error occurred while reading the parameter file: OUTPUT_HDF5' call read_value_logical(OUTPUT_SEISMOS_3D_ARRAY, 'OUTPUT_SEISMOS_3D_ARRAY', ier) + if (ier /= 0) stop 'an error occurred while reading the parameter file: OUTPUT_ASDF' + call read_value_logical(OUTPUT_SEISMOS_HDF5, 'OUTPUT_SEISMOS_HDF5', ier) if (ier /= 0) stop 'an error occurred while reading the parameter file: OUTPUT_3D_ARRAY' call read_value_logical(ROTATE_SEISMOGRAMS_RT, 'ROTATE_SEISMOGRAMS_RT', ier) if (ier /= 0) stop 'an error occurred while reading the parameter file: ROTATE_SEISMOGRAMS_RT' @@ -376,6 +378,17 @@ subroutine read_parameter_file() call read_value_double_precision(FILESYSTEM_IO_BANDWIDTH, 'FILESYSTEM_IO_BANDWIDTH', ier); ier = 0 endif + ! HDF5 file I/O + ! (optional) hdf5 database io flag + call read_value_logical(HDF5_ENABLED, 'HDF5_ENABLED', ier); ier = 0 + ! HDF file I/O server + if (HDF5_ENABLED) then + ! (optional) movie outputs + call read_value_logical(HDF5_FOR_MOVIES, 'HDF5_FOR_MOVIES', ier); ier = 0 + ! (optional) number of io dedicated nodes + call read_value_integer(HDF5_IO_NODES, 'HDF5_IO_NODES', ier); ier = 0 + endif + ! closes parameter file call close_parameter_file() @@ -409,5 +422,20 @@ subroutine read_parameter_file() endif #endif + ! checks HDF5 compilation support +#if !defined(USE_HDF5) + if (HDF5_ENABLED) then + print * + print *,'**************' + print *,'**************' + print *,'HDF5 is enabled in parameter file but the code was not compiled with HDF5' + print *,'See --with-hdf5 configure options.' + print *,'**************' + print *,'**************' + print * + stop 'an error occurred while reading the parameter file: HDF5 is enabled but code not built with HDF5' + endif +#endif + end subroutine read_parameter_file diff --git a/src/shared/rules.mk b/src/shared/rules.mk index 9334a8f4f..1a6a2dab3 100644 --- a/src/shared/rules.mk +++ b/src/shared/rules.mk @@ -57,6 +57,7 @@ shared_OBJECTS = \ $O/get_model_parameters.shared.o \ $O/get_timestep_and_layers.shared.o \ $O/gll_library.shared.o \ + $O/hdf5_manager.shared_hdf5_module.o \ $O/heap_sort.shared.o \ $O/hex_nodes.shared.o \ $O/init_openmp.shared.o \ @@ -149,6 +150,9 @@ else shared_OBJECTS += $(adios_shared_STUBS) endif +## HDF5 file i/o + + ## ## ASDF ## @@ -279,6 +283,19 @@ $O/%.shared_asdf.o: $S/%.f90 $O/%.cc.o: $S/%.c ${SETUP}/config.h ${CC} -c $(CPPFLAGS) $(CFLAGS) -o $@ $< +## HDF5 +$O/%.shared_hdf5_module.o: $S/%.f90 $O/shared_par.shared_module.o + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< + +$O/%.shared_hdf5_module.o: $S/%.F90 $O/shared_par.shared_module.o + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< + +$O/%.shared_hdf5.o: $S/%.f90 $O/hdf5_manager.shared_hdf5_module.o + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< + +$O/%.shared_hdf5.o: $S/%.F90 $O/hdf5_manager.shared_hdf5_module.o + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} -c -o $@ $< + ## c++ files $O/%.shared.o: $S/%.cpp ${SETUP}/config.h ${CXX} -c $(CXXFLAGS) $(PARALLEL_STL_DEF) -o $@ $< diff --git a/src/shared/shared_par.f90 b/src/shared/shared_par.f90 index ab85596e7..a5081c66a 100644 --- a/src/shared/shared_par.f90 +++ b/src/shared/shared_par.f90 @@ -65,7 +65,7 @@ module shared_input_parameters logical :: RECEIVERS_CAN_BE_BURIED logical :: OUTPUT_SEISMOS_ASCII_TEXT,OUTPUT_SEISMOS_SAC_ALPHANUM,OUTPUT_SEISMOS_SAC_BINARY, & - OUTPUT_SEISMOS_ASDF,OUTPUT_SEISMOS_3D_ARRAY, & + OUTPUT_SEISMOS_ASDF,OUTPUT_SEISMOS_3D_ARRAY,OUTPUT_SEISMOS_HDF5, & ROTATE_SEISMOGRAMS_RT,WRITE_SEISMOGRAMS_BY_MAIN, & SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE,READ_ADJSRC_ASDF @@ -208,6 +208,21 @@ module shared_input_parameters logical :: SHIFT_SIMULTANEOUS_RUNS = .false. double precision :: FILESYSTEM_IO_BANDWIDTH = 0.d0 + ! HDF5 file i/o + logical :: HDF5_ENABLED = .false. ! for all databases i/o in hdf5 + logical :: HDF5_FOR_MOVIES = .false. ! for movies (shakemap, surface movies, volume movies) + + ! HDF5 IO server + ! number of io dedicated nodes + integer :: HDF5_IO_NODES = 0 + + ! HDF5 IO mode (collective or independent) + logical :: H5_COL = .true. + + ! flag for io-dedicated/compute node. + logical :: IO_storage_task = .false. + logical :: IO_compute_task = .true. + ! UCB Source time function parameters logical :: STF_IS_UCB_HEAVISIDE = .false. ! source-time function is a UCB-style filtered heaviside double precision :: UCB_SOURCE_T1 = 400.d0, UCB_SOURCE_T2 = 250.d0, UCB_SOURCE_T3 = 53.d0, UCB_SOURCE_T4 = 40.d0 diff --git a/src/specfem3D/SIEM_compute_kernels.F90 b/src/specfem3D/SIEM_compute_kernels.F90 index 94aebc21c..a92badadc 100644 --- a/src/specfem3D/SIEM_compute_kernels.F90 +++ b/src/specfem3D/SIEM_compute_kernels.F90 @@ -675,6 +675,12 @@ end subroutine write_ensight_perelementAS endif endif + if (HDF5_ENABLED) then + if (myrank == 0) then + print *,'Full gravity kernels: option HDF5_ENABLED not implemented yet, saving them as binary files' + endif + endif + ! scaling factors ! kernel unit [ s / km^3 ] scale_kl = real(scale_t * scale_displ_inv * 1.d9,kind=CUSTOM_REAL) diff --git a/src/specfem3D/finalize_simulation.F90 b/src/specfem3D/finalize_simulation.F90 index 1ee16f937..323f7a490 100644 --- a/src/specfem3D/finalize_simulation.F90 +++ b/src/specfem3D/finalize_simulation.F90 @@ -292,10 +292,16 @@ subroutine finalize_simulation_cleanup() if (MOVIE_SURFACE) then deallocate(store_val_ux,store_val_uy,store_val_uz) deallocate(store_val_ux_all,store_val_uy_all,store_val_uz_all) + if (HDF5_ENABLED) then + call movie_surface_finalize_hdf5() + endif endif if (MOVIE_VOLUME) then deallocate(nu_3dmovie) deallocate(mask_3dmovie,muvstore_crust_mantle_3dmovie) + if (HDF5_ENABLED) then + call movie_volume_finalize_hdf5() + endif endif ! noise simulations diff --git a/src/specfem3D/get_attenuation.f90 b/src/specfem3D/get_attenuation.f90 index 5810b1eec..8d1e83b2f 100644 --- a/src/specfem3D/get_attenuation.f90 +++ b/src/specfem3D/get_attenuation.f90 @@ -36,7 +36,7 @@ subroutine get_attenuation_model_3D(iregion_code, & use constants_solver - use shared_parameters, only: ATT_F_C_SOURCE + use shared_parameters, only: ATT_F_C_SOURCE, HDF5_ENABLED use specfem_par, only: ATTENUATION_VAL,ADIOS_FOR_ARRAYS_SOLVER,LOCAL_PATH, & scale_t_inv @@ -97,7 +97,9 @@ subroutine get_attenuation_model_3D(iregion_code, & ! use the filename to determine the actual contents of the read if (ADIOS_FOR_ARRAYS_SOLVER) then ! ADIOS format - call read_attenuation_adios(iregion_code,factor_common, factor_scale, tau_s, vnspec, f_c_source) + call read_attenuation_adios(iregion_code, factor_common, factor_scale, tau_s, vnspec, f_c_source) + else if (HDF5_ENABLED) then + call read_attenuation_hdf5(iregion_code, factor_common, factor_scale, tau_s, vnspec, f_c_source) else ! binary format ! opens corresponding databases file diff --git a/src/specfem3D/initialize_simulation.F90 b/src/specfem3D/initialize_simulation.F90 index 7f45c3a72..02cffe2db 100644 --- a/src/specfem3D/initialize_simulation.F90 +++ b/src/specfem3D/initialize_simulation.F90 @@ -577,6 +577,8 @@ subroutine initialize_simulation_check() call exit_mpi(myrank,'SAVE_AZIMUTHAL_ANISO_KL_ONLY needs anisotropic kernel flag ANISOTROPIC_KL set to .true.') if (SAVE_REGULAR_KL .and. SAVE_AZIMUTHAL_ANISO_KL_ONLY) & call exit_mpi(myrank,'SAVE_AZIMUTHAL_ANISO_KL_ONLY not implemented yet for SAVE_REGULAR_KL kernels') + if (SAVE_REGULAR_KL .and. (HDF5_ENABLED .or. ADIOS_FOR_KERNELS)) & + call exit_mpi(myrank,'SAVE_REGULAR_KL not implemented yet for HDF5 or ADIOS output') endif ! check for GPU runs diff --git a/src/specfem3D/noise_tomography.f90 b/src/specfem3D/noise_tomography.f90 index c55b18b52..faaacb301 100644 --- a/src/specfem3D/noise_tomography.f90 +++ b/src/specfem3D/noise_tomography.f90 @@ -1182,6 +1182,8 @@ subroutine save_kernels_strength_noise() ! kernel file output if (ADIOS_FOR_KERNELS) then call write_kernels_strength_noise_adios() + else if (HDF5_ENABLED) then + call write_kernels_strength_noise_hdf5() else ! binary file output call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH) diff --git a/src/specfem3D/prepare_attenuation.f90 b/src/specfem3D/prepare_attenuation.f90 index 12a1ae1bd..9c776f308 100644 --- a/src/specfem3D/prepare_attenuation.f90 +++ b/src/specfem3D/prepare_attenuation.f90 @@ -448,6 +448,8 @@ subroutine prepare_attenuation() if (ADIOS_FOR_SOLVER_MESHFILES) then ! adios file output call save_forward_model_at_shifted_frequency_adios(factor_scale_relaxed_crust_mantle,factor_scale_relaxed_inner_core) + else if (HDF5_ENABLED) then + call save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_crust_mantle,factor_scale_relaxed_inner_core) else ! outputs model files in binary format call save_forward_model_at_shifted_frequency(factor_scale_relaxed_crust_mantle,factor_scale_relaxed_inner_core) diff --git a/src/specfem3D/prepare_movie.f90 b/src/specfem3D/prepare_movie.f90 index 168d85a3b..32b05a546 100644 --- a/src/specfem3D/prepare_movie.f90 +++ b/src/specfem3D/prepare_movie.f90 @@ -65,14 +65,18 @@ subroutine prepare_movie_surface() ! note: for noise tomography, must NOT be coarse (have to be saved on all GLL points) if (MOVIE_COARSE) then ! checks setup - if (NGLLX /= NGLLY) & + if (NGLLX /= NGLLY .and. .not. HDF5_ENABLED) & call exit_MPI(myrank,'MOVIE_COARSE together with MOVIE_SURFACE requires NGLLX=NGLLY') ! number of points nmovie_points = 2 * 2 * NSPEC2D_TOP(IREGION_CRUST_MANTLE) NIT = NGLLX - 1 else ! number of points - nmovie_points = NGLLX * NGLLY * NSPEC2D_TOP(IREGION_CRUST_MANTLE) + if (.not. HDF5_ENABLED) then + nmovie_points = NGLLX * NGLLY * NSPEC2D_TOP(IREGION_CRUST_MANTLE) + else ! HDF5 + nmovie_points = 4 * (NGLLX-1) * (NGLLY-1) * NSPEC2D_TOP(IREGION_CRUST_MANTLE) + end if NIT = 1 endif @@ -81,7 +85,11 @@ subroutine prepare_movie_surface() ! those arrays are not necessary for noise tomography, so only allocate them in MOVIE_SURFACE case ! writes out movie point locations to file - call write_movie_surface_mesh() + if (HDF5_ENABLED) then + call write_movie_surface_mesh_hdf5() + else + call write_movie_surface_mesh() + endif ! allocates movie surface arrays for wavefield values allocate(store_val_ux(nmovie_points), & @@ -90,6 +98,7 @@ subroutine prepare_movie_surface() if (ier /= 0 ) call exit_MPI(myrank,'Error allocating movie surface arrays') ! allocates arrays for gathering wavefield values + ! TODO ADD HDF5 (store_val_*_all is not used) if (myrank == 0) then ! only main needs full arrays allocate(store_val_ux_all(nmovie_points,0:NPROCTOT_VAL-1), & @@ -184,8 +193,13 @@ subroutine prepare_movie_volume() allocate(nu_3dmovie(3,3,npoints_3dmovie),stat=ier) if (ier /= 0 ) call exit_MPI(myrank,'Error allocating nu for 3D movie') - call write_movie_volume_mesh(nu_3dmovie,num_ibool_3dmovie,mask_3dmovie,mask_ibool_3dmovie, & - muvstore_crust_mantle_3dmovie,npoints_3dmovie) + if (HDF5_ENABLED) then + call write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovie,mask_ibool_3dmovie, & + muvstore_crust_mantle_3dmovie,npoints_3dmovie,nspecel_3dmovie) + else + call write_movie_volume_mesh(nu_3dmovie,num_ibool_3dmovie,mask_3dmovie,mask_ibool_3dmovie, & + muvstore_crust_mantle_3dmovie,npoints_3dmovie) + endif if (myrank == 0) then write(IMAIN,*) ' Writing to movie3D*** files on local disk databases directory' diff --git a/src/specfem3D/read_arrays_solver_hdf5.F90 b/src/specfem3D/read_arrays_solver_hdf5.F90 new file mode 100644 index 000000000..e33ac481e --- /dev/null +++ b/src/specfem3D/read_arrays_solver_hdf5.F90 @@ -0,0 +1,1380 @@ + subroutine read_arrays_solver_hdf5(iregion_code, & + nspec,nglob,nglob_xy, & + nspec_iso,nspec_tiso,nspec_ani, & + rho_vp,rho_vs, & + xstore,ystore,zstore, & + xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz, & + rhostore, kappavstore,muvstore,kappahstore,muhstore,eta_anisostore, & + c11store,c12store,c13store,c14store,c15store,c16store,c22store, & + c23store,c24store,c25store,c26store,c33store,c34store,c35store, & + c36store,c44store,c45store,c46store,c55store,c56store,c66store, & + mu0store, & + ibool,idoubling,ispec_is_tiso, & + rmassx,rmassy,rmassz, & + nglob_oceans,rmass_ocean_load, & + b_rmassx,b_rmassy) + + + use constants_solver + +#ifdef USE_HDF5 + use shared_parameters, only: H5_COL + use specfem_par, only: & + ABSORBING_CONDITIONS, & + LOCAL_PATH,ABSORBING_CONDITIONS + use manager_hdf5 +#endif + + implicit none + + integer,intent(in) :: iregion_code + integer,intent(in) :: nspec,nglob,nglob_xy + integer,intent(in) :: nspec_iso,nspec_tiso,nspec_ani + + ! Stacey + real(kind=CUSTOM_REAL),dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(inout) :: rho_vp,rho_vs + + real(kind=CUSTOM_REAL), dimension(nglob),intent(inout) :: xstore,ystore,zstore + + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(inout) :: & + xix,xiy,xiz,etax,etay,etaz,gammax,gammay,gammaz + + ! material properties + real(kind=CUSTOM_REAL),dimension(NGLLX,NGLLY,NGLLZ,nspec_iso),intent(inout) :: & + rhostore,kappavstore,muvstore + + ! additional arrays for anisotropy stored only where needed to save memory + real(kind=CUSTOM_REAL),dimension(NGLLX,NGLLY,NGLLZ,nspec_tiso),intent(inout) :: & + kappahstore,muhstore,eta_anisostore + + ! additional arrays for full anisotropy + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nspec_ani),intent(inout) :: & + c11store,c12store,c13store,c14store,c15store,c16store, & + c22store,c23store,c24store,c25store,c26store,c33store,c34store, & + c35store,c36store,c44store,c45store,c46store,c55store,c56store,c66store + + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(inout) :: mu0store + + ! global addressing + integer,dimension(NGLLX,NGLLY,NGLLZ,nspec),intent(inout) :: ibool + integer, dimension(nspec),intent(inout) :: idoubling + logical, dimension(nspec),intent(inout) :: ispec_is_tiso + + ! mass matrices and additional ocean load mass matrix + real(kind=CUSTOM_REAL), dimension(nglob_xy),intent(inout) :: rmassx,rmassy + real(kind=CUSTOM_REAL), dimension(nglob_xy),intent(inout) :: b_rmassx,b_rmassy + + real(kind=CUSTOM_REAL), dimension(nglob),intent(inout) :: rmassz + + integer,intent(in) :: nglob_oceans + real(kind=CUSTOM_REAL), dimension(nglob_oceans),intent(inout) :: rmass_ocean_load + +#ifdef USE_HDF5 + + ! local parameters + integer :: lnspec,lnglob + ! group, dataset name + character(len=64) :: gname_region + + ! MPI variables + integer :: info, comm + + ! offset arrays + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nnodes + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nnodes_xy + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nnodes_oceans + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nelems + + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/solver_data.h5' + ! group name + write(gname_region, "('reg',i1)") iregion_code + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the hdf5 file + call h5_open_file_p_collect(name_database_hdf5) + ! open the group + call h5_open_group(gname_region) + + call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes", offset_nnodes, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes_xy", offset_nnodes_xy, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("offset_nnodes_oceans", offset_nnodes_oceans, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("offset_nelems", offset_nelems, (/0/), H5_COL) + + lnspec = offset_nelems(myrank) + lnglob = offset_nnodes(myrank) + + ! checks dimensions + if (lnspec /= nspec) then + call h5_close_group() + call h5_close_file() + print *, 'Error at rank ', myrank + print *,'Error file dimension: nspec in file = ',lnspec,' but nspec desired:',nspec + print *,'please check file ', name_database_hdf5 + call exit_mpi(myrank,'Error dimensions in solver_data.h5') + endif + if (lnglob /= nglob) then + close(IIN) + print *, 'Error at rank ', myrank + print *,'Error file dimension: nglob in file = ',lnglob,' but nglob desired:',nglob + print *,'please check file ', name_database_hdf5 + call exit_mpi(myrank,'Error dimensions in solver_data.h5') + endif + + ! read xstore + call h5_read_dataset_collect_hyperslab_in_group("xstore", xstore, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) + ! read ystore + call h5_read_dataset_collect_hyperslab_in_group("ystore", ystore, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) + ! read zstore + call h5_read_dataset_collect_hyperslab_in_group("zstore", zstore, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) + ! ibool + call h5_read_dataset_collect_hyperslab_in_group("ibool", ibool, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! idoubling + call h5_read_dataset_collect_hyperslab_in_group("idoubling", idoubling, (/sum(offset_nelems(0:myrank-1))/), H5_COL) + ! ispec_is_tiso + call h5_read_dataset_collect_hyperslab_in_group("ispec_is_tiso", ispec_is_tiso, (/sum(offset_nelems(0:myrank-1))/), H5_COL) + ! xix + call h5_read_dataset_collect_hyperslab_in_group("xixstore", xix, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! xiy + call h5_read_dataset_collect_hyperslab_in_group("xiystore", xiy, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! xiz + call h5_read_dataset_collect_hyperslab_in_group("xizstore", xiz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! etax + call h5_read_dataset_collect_hyperslab_in_group("etaxstore", etax, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! etay + call h5_read_dataset_collect_hyperslab_in_group("etaystore", etay, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! etaz + call h5_read_dataset_collect_hyperslab_in_group("etazstore", etaz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! gammax + call h5_read_dataset_collect_hyperslab_in_group("gammaxstore", gammax, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! gammay + call h5_read_dataset_collect_hyperslab_in_group("gammaystore", gammay, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! gammaz + call h5_read_dataset_collect_hyperslab_in_group("gammazstore", gammaz, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + + if (iregion_code == IREGION_TRINFINITE .or. iregion_code == IREGION_INFINITE) then + call h5_close_group() + call h5_close_file() + return + endif + + ! rhostore + call h5_read_dataset_collect_hyperslab_in_group("rhostore", rhostore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! kappavstore + call h5_read_dataset_collect_hyperslab_in_group("kappavstore", kappavstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + + if (iregion_code /= IREGION_OUTER_CORE) then + ! muvstore + call h5_read_dataset_collect_hyperslab_in_group("muvstore", muvstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + endif + + select case (iregion_code) + case (IREGION_CRUST_MANTLE) + ! crust/mantle + if (ANISOTROPIC_3D_MANTLE_VAL) then + ! c11store + call h5_read_dataset_collect_hyperslab_in_group("c11store", c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c12store + call h5_read_dataset_collect_hyperslab_in_group("c12store", c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c13store + call h5_read_dataset_collect_hyperslab_in_group("c13store", c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c14store + call h5_read_dataset_collect_hyperslab_in_group("c14store", c14store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c15store + call h5_read_dataset_collect_hyperslab_in_group("c15store", c15store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c16store + call h5_read_dataset_collect_hyperslab_in_group("c16store", c16store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c22store + call h5_read_dataset_collect_hyperslab_in_group("c22store", c22store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c23store + call h5_read_dataset_collect_hyperslab_in_group("c23store", c23store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c24store + call h5_read_dataset_collect_hyperslab_in_group("c24store", c24store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c25store + call h5_read_dataset_collect_hyperslab_in_group("c25store", c25store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c26store + call h5_read_dataset_collect_hyperslab_in_group("c26store", c26store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c33store + call h5_read_dataset_collect_hyperslab_in_group("c33store", c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c34store + call h5_read_dataset_collect_hyperslab_in_group("c34store", c34store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c35store + call h5_read_dataset_collect_hyperslab_in_group("c35store", c35store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c36store + call h5_read_dataset_collect_hyperslab_in_group("c36store", c36store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c44store + call h5_read_dataset_collect_hyperslab_in_group("c44store", c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c45store + call h5_read_dataset_collect_hyperslab_in_group("c45store", c45store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c46store + call h5_read_dataset_collect_hyperslab_in_group("c46store", c46store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c55store + call h5_read_dataset_collect_hyperslab_in_group("c55store", c55store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c56store + call h5_read_dataset_collect_hyperslab_in_group("c56store", c56store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c66store + call h5_read_dataset_collect_hyperslab_in_group("c66store", c66store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + else + if (TRANSVERSE_ISOTROPY_VAL) then + ! kappahstore + call h5_read_dataset_collect_hyperslab_in_group("kappahstore", kappahstore, & + (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! muhstore + call h5_read_dataset_collect_hyperslab_in_group("muhstore", muhstore, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! eta_anisostore + call h5_read_dataset_collect_hyperslab_in_group("eta_anisostore", eta_anisostore, & + (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + endif + endif + + ! mu0store + call h5_read_dataset_collect_hyperslab_in_group("mu0store", mu0store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + + case (IREGION_INNER_CORE) + ! inner core + if (ANISOTROPIC_INNER_CORE_VAL) then + ! c11store + call h5_read_dataset_collect_hyperslab_in_group("c11store", c11store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c12store + call h5_read_dataset_collect_hyperslab_in_group("c12store", c12store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c13store + call h5_read_dataset_collect_hyperslab_in_group("c13store", c13store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c33store + call h5_read_dataset_collect_hyperslab_in_group("c33store", c33store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! c44store + call h5_read_dataset_collect_hyperslab_in_group("c44store", c44store, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + endif + end select + + ! Stacey + if (ABSORBING_CONDITIONS) then + if (iregion_code == IREGION_CRUST_MANTLE) then + ! rho_vp + call h5_read_dataset_collect_hyperslab_in_group("rho_vp", rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! rho_vs + call h5_read_dataset_collect_hyperslab_in_group("rho_vs", rho_vs, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + else if (iregion_code == IREGION_OUTER_CORE) then + ! rho_vp + call h5_read_dataset_collect_hyperslab_in_group("rho_vp", rho_vp, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + endif + endif + + ! mass matrices + if (((NCHUNKS_VAL /= 6 .and. ABSORBING_CONDITIONS) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_INNER_CORE)) then + ! rmassx + call h5_read_dataset_collect_hyperslab_in_group("rmassx", rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) + ! rmassy + call h5_read_dataset_collect_hyperslab_in_group("rmassy", rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) + endif + + ! rmassz + call h5_read_dataset_collect_hyperslab_in_group("rmassz", rmassz, (/sum(offset_nnodes(0:myrank-1))/), H5_COL) + + if (((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_CRUST_MANTLE) .or. & + ((ROTATION_VAL .and. EXACT_MASS_MATRIX_FOR_ROTATION_VAL) .and. iregion_code == IREGION_INNER_CORE)) then + ! b_rmassx + call h5_read_dataset_collect_hyperslab_in_group("b_rmassx", b_rmassx, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) + ! b_rmassy + call h5_read_dataset_collect_hyperslab_in_group("b_rmassy", b_rmassy, (/sum(offset_nnodes_xy(0:myrank-1))/), H5_COL) + endif + + ! read additional ocean load mass matrix + if (OCEANS_VAL .and. iregion_code == IREGION_CRUST_MANTLE) then + ! rmass_ocean_load + call h5_read_dataset_collect_hyperslab_in_group("rmass_ocean_load", rmass_ocean_load, & + (/sum(offset_nnodes_oceans(0:myrank-1))/), H5_COL) + endif + + ! close group + call h5_close_group() + ! close file + call h5_close_file_p() + +#else + print* + print*,'ERROR: HDF5 support not enabled' + print*, 'Please recompile with HDF5 support with the --with-hdf5 option' + print* + stop +#endif + + +end subroutine read_arrays_solver_hdf5 + + +subroutine read_mesh_databases_MPI_hdf5(iregion_code) + + use specfem_par + use specfem_par_crustmantle + use specfem_par_outercore + use specfem_par_innercore + + use specfem_par_trinfinite + use specfem_par_infinite + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + integer, intent(in) :: iregion_code +#ifdef USE_HDF5 + + ! local parameters + integer :: ierr + integer :: num_interfaces,max_nibool_interfaces, & + num_phase_ispec,num_colors_outer,num_colors_inner + integer :: nspec_inner,nspec_outer + + ! group, dataset name + character(len=64) :: gname_region + ! MPI variables + integer :: info, comm + + ! offset arrays + integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_interfaces + integer, dimension(0:NPROCTOT_VAL-1) :: offset_max_nibool_interfaces + integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_phase_ispec + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_inner + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_outer + integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_colors_outer + integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_colors_inner + + + call h5_initialize() + + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/solver_data_mpi.h5' + write(gname_region, "('reg',i1)") iregion_code + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the hdf5 file + call h5_open_file_p_collect(name_database_hdf5) + ! open the group + call h5_open_group(gname_region) + + call h5_read_dataset_collect_hyperslab_in_group('offset_num_interfaces', offset_num_interfaces, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_max_nibool_interfaces', offset_max_nibool_interfaces, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_num_phase_ispec', offset_num_phase_ispec, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_nspec_inner', offset_nspec_inner, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_nspec_outer', offset_nspec_outer, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_num_colors_outer', offset_num_colors_outer, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group('offset_num_colors_inner', offset_num_colors_inner, (/0/), H5_COL) + + num_interfaces = offset_num_interfaces(myrank) + max_nibool_interfaces = offset_max_nibool_interfaces(myrank) + num_phase_ispec = offset_num_phase_ispec(myrank) + nspec_inner = offset_nspec_inner(myrank) + nspec_outer = offset_nspec_outer(myrank) + num_colors_outer = offset_num_colors_outer(myrank) + num_colors_inner = offset_num_colors_inner(myrank) + + select case(iregion_code) + case (IREGION_CRUST_MANTLE) + + num_interfaces_crust_mantle = num_interfaces + max_nibool_interfaces_cm = max_nibool_interfaces + num_phase_ispec_crust_mantle = num_phase_ispec + nspec_inner_crust_mantle = nspec_inner + nspec_outer_crust_mantle = nspec_outer + num_colors_outer_crust_mantle = num_colors_outer + num_colors_inner_crust_mantle = num_colors_inner + + allocate(my_neighbors_crust_mantle(num_interfaces), & + nibool_interfaces_crust_mantle(num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating my_neighbors_crust_mantle etc.') + if (num_interfaces > 0) then + call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & + my_neighbors_crust_mantle(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & + nibool_interfaces_crust_mantle(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + end if + + allocate(ibool_interfaces_crust_mantle(max_nibool_interfaces,num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating ibool_interfaces_crust_mantle etc.') + if (num_interfaces > 0) then + ibool_interfaces_crust_mantle(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & + ibool_interfaces_crust_mantle(1:max_nibool_interfaces,1:num_interfaces), & + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + end if + + allocate(phase_ispec_inner_crust_mantle(num_phase_ispec,2), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating phase_ispec_inner_crust_mantle etc.') + if (num_phase_ispec > 0) then + phase_ispec_inner_crust_mantle(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & + phase_ispec_inner_crust_mantle(1:num_phase_ispec,1:2), & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) + end if + + allocate(num_elem_colors_crust_mantle(num_colors_outer+num_colors_inner), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating num_elem_colors_crust_mantle etc.') + if (USE_MESH_COLORING_GPU) then + call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & + num_elem_colors_crust_mantle(1:(num_colors_outer+num_colors_inner)), & + (/sum(offset_num_colors_outer(0:myrank-1))/), H5_COL) + end if + + case (IREGION_OUTER_CORE) + + num_interfaces_outer_core = num_interfaces + max_nibool_interfaces_oc = max_nibool_interfaces + num_phase_ispec_outer_core = num_phase_ispec + nspec_inner_outer_core = nspec_inner + nspec_outer_outer_core = nspec_outer + num_colors_outer_outer_core = num_colors_outer + num_colors_inner_outer_core = num_colors_inner + + allocate(my_neighbors_outer_core(num_interfaces), & + nibool_interfaces_outer_core(num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating my_neighbors_outer_core etc.') + if (num_interfaces > 0) then + call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & + my_neighbors_outer_core(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & + nibool_interfaces_outer_core(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + end if + + allocate(ibool_interfaces_outer_core(max_nibool_interfaces,num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating ibool_interfaces_outer_core etc.') + if (num_interfaces > 0) then + ibool_interfaces_outer_core(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & + ibool_interfaces_outer_core(1:max_nibool_interfaces,1:num_interfaces), & + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + end if + + allocate(phase_ispec_inner_outer_core(num_phase_ispec,2), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating phase_ispec_inner_outer_core etc.') + if (num_phase_ispec > 0) then + phase_ispec_inner_outer_core(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & + phase_ispec_inner_outer_core(1:num_phase_ispec,1:2), & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) + end if + + allocate(num_elem_colors_outer_core(num_colors_outer+num_colors_inner), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating num_elem_colors_outer_core etc.') + if (USE_MESH_COLORING_GPU) then + call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & + num_elem_colors_outer_core(1:(num_colors_outer+num_colors_inner)), & + (/sum(offset_num_colors_outer(0:myrank-1))/), H5_COL) + end if + + case (IREGION_INNER_CORE) + + num_interfaces_inner_core = num_interfaces + max_nibool_interfaces_ic = max_nibool_interfaces + num_phase_ispec_inner_core = num_phase_ispec + nspec_inner_inner_core = nspec_inner + nspec_outer_inner_core = nspec_outer + num_colors_outer_inner_core = num_colors_outer + num_colors_inner_inner_core = num_colors_inner + + allocate(my_neighbors_inner_core(num_interfaces), & + nibool_interfaces_inner_core(num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating my_neighbors_inner_core etc.') + if (num_interfaces > 0) then + call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & + my_neighbors_inner_core(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & + nibool_interfaces_inner_core(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + end if + + allocate(ibool_interfaces_inner_core(max_nibool_interfaces,num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating ibool_interfaces_inner_core etc.') + if (num_interfaces > 0) then + ibool_interfaces_inner_core(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & + ibool_interfaces_inner_core(1:max_nibool_interfaces,1:num_interfaces), & + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + end if + + allocate(phase_ispec_inner_inner_core(num_phase_ispec,2), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating phase_ispec_inner_inner_core etc.') + if (num_phase_ispec > 0) then + phase_ispec_inner_inner_core(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & + phase_ispec_inner_inner_core(1:num_phase_ispec,1:2), & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) + end if + + allocate(num_elem_colors_inner_core(num_colors_outer+num_colors_inner), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating num_elem_colors_inner_core etc.') + if (USE_MESH_COLORING_GPU) then + call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & + num_elem_colors_inner_core(1:(num_colors_outer+num_colors_inner)), & + (/sum(offset_num_colors_outer(0:myrank-1))/), H5_COL) + end if + + case (IREGION_TRINFINITE) + + num_interfaces_trinfinite = num_interfaces + max_nibool_interfaces_trinfinite = max_nibool_interfaces + num_phase_ispec_trinfinite = num_phase_ispec + nspec_inner_trinfinite = nspec_inner + nspec_outer_trinfinite = nspec_outer + num_colors_outer_trinfinite = num_colors_outer + num_colors_inner_trinfinite = num_colors_inner + + allocate(my_neighbors_trinfinite(num_interfaces), & + nibool_interfaces_trinfinite(num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating my_neighbors_trinfinite etc.') + if (num_interfaces > 0) then + call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & + my_neighbors_trinfinite(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & + nibool_interfaces_trinfinite(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + end if + + allocate(ibool_interfaces_trinfinite(max_nibool_interfaces,num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating ibool_interfaces_trinfinite etc.') + if (num_interfaces > 0) then + ibool_interfaces_trinfinite(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & + ibool_interfaces_trinfinite(1:max_nibool_interfaces,1:num_interfaces), & + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + end if + + allocate(phase_ispec_inner_trinfinite(num_phase_ispec,2), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating phase_ispec_inner_trinfinite etc.') + if (num_phase_ispec > 0) then + phase_ispec_inner_trinfinite(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & + phase_ispec_inner_trinfinite(1:num_phase_ispec,1:2), & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) + end if + + allocate(num_elem_colors_trinfinite(num_colors_outer+num_colors_inner), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating num_elem_colors_trinfinite etc.') + if (USE_MESH_COLORING_GPU) then + call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & + num_elem_colors_trinfinite(1:(num_colors_outer+num_colors_inner)), & + (/sum(offset_num_colors_outer(0:myrank-1))/), H5_COL) + end if + + case (IREGION_INFINITE) + + num_interfaces_infinite = num_interfaces + max_nibool_interfaces_infinite = max_nibool_interfaces + num_phase_ispec_infinite = num_phase_ispec + nspec_inner_infinite = nspec_inner + nspec_outer_infinite = nspec_outer + num_colors_outer_infinite = num_colors_outer + num_colors_inner_infinite = num_colors_inner + + allocate(my_neighbors_infinite(num_interfaces), & + nibool_interfaces_infinite(num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating my_neighbors_infinite etc.') + if (num_interfaces > 0) then + call h5_read_dataset_collect_hyperslab_in_group("my_neighbors", & + my_neighbors_infinite(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("nibool_interfaces", & + nibool_interfaces_infinite(1:num_interfaces), & + (/sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + end if + + allocate(ibool_interfaces_infinite(max_nibool_interfaces,num_interfaces), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating ibool_interfaces_infinite etc.') + if (num_interfaces > 0) then + ibool_interfaces_infinite(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("ibool_interfaces", & + ibool_interfaces_infinite(1:max_nibool_interfaces,1:num_interfaces), & + (/0,sum(offset_num_interfaces(0:myrank-1))/), H5_COL) + end if + + allocate(phase_ispec_inner_infinite(num_phase_ispec,2), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating phase_ispec_inner_infinite etc.') + if (num_phase_ispec > 0) then + phase_ispec_inner_infinite(:,:) = 0 + call h5_read_dataset_collect_hyperslab_in_group("phase_ispec_inner", & + phase_ispec_inner_infinite(1:num_phase_ispec,1:2), & + (/sum(offset_num_phase_ispec(0:myrank-1)),0/), H5_COL) + end if + + allocate(num_elem_colors_infinite(num_colors_outer+num_colors_inner), stat=ierr) + if (ierr /= 0) call exit_mpi(myrank,'Error allocating num_elem_colors_infinite etc.') + if (USE_MESH_COLORING_GPU) then + call h5_read_dataset_collect_hyperslab_in_group("num_elem_colors", & + num_elem_colors_infinite(1:(num_colors_outer+num_colors_inner)), & + (/sum(offset_num_colors_outer(0:myrank-1))/), H5_COL) + end if + + case default + print*, 'ERROR: unknown region code' + stop + end select + + ! close group + call h5_close_group() + ! close file + call h5_close_file_p() + +#else + print* + print*,'ERROR: HDF5 support not enabled' + print*, 'Please recompile with HDF5 support with the --with-hdf5 option' + print* + stop +#endif + +end subroutine read_mesh_databases_MPI_hdf5 + + + +subroutine read_mesh_databases_coupling_hdf5() + +#ifdef USE_HDF5 + use constants + + use meshfem_par, only: & + myrank, LOCAL_PATH + + !use meshfem_models_par, only: & + ! HONOR_1D_SPHERICAL_MOHO + !SAVE_BOUNDARY_MESH,HONOR_1D_SPHERICAL_MOHO,SUPPRESS_CRUSTAL_MESH + + !use regions_mesh_par2, only: & + ! NSPEC2D_MOHO, NSPEC2D_400, NSPEC2D_670, & + ! ibelm_moho_top,ibelm_moho_bot,ibelm_400_top,ibelm_400_bot, & + ! ibelm_670_top,ibelm_670_bot,normal_moho,normal_400,normal_670, & + ! ispec2D_moho_top,ispec2D_moho_bot,ispec2D_400_top,ispec2D_400_bot, & + ! ispec2D_670_top,ispec2D_670_bot ! prname + + use shared_parameters, only: FULL_GRAVITY + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + + use specfem_par_trinfinite + use specfem_par_infinite + + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + + character(len=64) :: gname_region + + ! MPI variables + integer :: info, comm + + ! offset arrays + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_xmin + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_xmax + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_ymin + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_ymax + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_top + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_bottom + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_moho_top + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_moho_bottom + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_400_top + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_400_bottom + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_670_top + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2D_670_bottom + !integer, dimension(0:NPROCTOT_VAL-1) :: tmp_array + + + ! dump integers + integer :: tmp_nspec2d_moho, tmp_nspec2d_400, tmp_nspec2d_670 + + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize hdf5 + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the hdf5 file + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/boundary.h5' + call h5_open_file_p_collect(name_database_hdf5) + + + if (NSPEC_CRUST_MANTLE > 0) then + ! open the group + write(gname_region, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_open_group(gname_region) + + ! read actual number of elements + ! nspec2D_xmin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_crust_mantle, (/myrank/), H5_COL) + ! nspec2D_xmax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_crust_mantle, (/myrank/), H5_COL) + ! nspec2D_ymin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_crust_mantle, (/myrank/), H5_COL) + ! nspec2D_ymax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_crust_mantle, (/myrank/), H5_COL) + + ! read offset arrays (stored length) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) + + ! ibelm_xmin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_crust_mantle, & + (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + ! ibelm_xmax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_crust_mantle, & + (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + ! ibelm_ymin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_crust_mantle, & + (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + ! ibelm_ymax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_crust_mantle, & + (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + ! ibelm_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_crust_mantle, & + (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + ! ibelm_bottom + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_crust_mantle, & + (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + + ! normal_xmin + call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + ! normal_xmax + call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + ! normal_ymin + call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + ! normal_ymax + call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + ! normal_top + call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + ! normal_bottom + call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_crust_mantle, & + (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + + ! jacobian_xmin + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_crust_mantle, & + (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + ! jacobian_xmax + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_crust_mantle,& + (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + ! jacobian_ymin + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_crust_mantle, & + (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + ! jacobian_ymax + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_crust_mantle, & + (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + ! jacobian_top + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_crust_mantle, & + (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + ! jacobian_bottom + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_crust_mantle, & + (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + + ! close group + call h5_close_group() + end if ! NSPEC_CRUST_MANTLE > 0 + + if (NSPEC_OUTER_CORE > 0) then + ! change group name + write(gname_region, "('reg',i1)") IREGION_OUTER_CORE + call h5_open_group(gname_region) + + ! read actual number of elements + ! nspec2D_xmin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_outer_core, (/myrank/), H5_COL) + ! nspec2D_xmax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_outer_core, (/myrank/), H5_COL) + ! nspec2D_ymin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_outer_core, (/myrank/), H5_COL) + ! nspec2D_ymax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_outer_core, (/myrank/), H5_COL) + + ! read offset arrays (stored length) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) + + ! ibelm_xmin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_outer_core, & + (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + ! ibelm_xmax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_outer_core, & + (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + ! ibelm_ymin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_outer_core, & + (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + ! ibelm_ymax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_outer_core, & + (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + ! ibelm_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_outer_core, & + (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + ! ibelm_bottom + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_outer_core, & + (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + + ! normal_xmin + call h5_read_dataset_collect_hyperslab_in_group("normal_xmin", normal_xmin_outer_core, & + (/0,0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + ! normal_xmax + call h5_read_dataset_collect_hyperslab_in_group("normal_xmax", normal_xmax_outer_core, & + (/0,0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + ! normal_ymin + call h5_read_dataset_collect_hyperslab_in_group("normal_ymin", normal_ymin_outer_core, & + (/0,0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + ! normal_ymax + call h5_read_dataset_collect_hyperslab_in_group("normal_ymax", normal_ymax_outer_core, & + (/0,0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + ! normal_top + call h5_read_dataset_collect_hyperslab_in_group("normal_top", normal_top_outer_core, & + (/0,0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + ! normal_bot + call h5_read_dataset_collect_hyperslab_in_group("normal_bottom", normal_bottom_outer_core, & + (/0,0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + + ! jacobian_xmin + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmin", jacobian2D_xmin_outer_core, & + (/0,0,sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + ! jacobian_xmax + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_xmax", jacobian2D_xmax_outer_core, & + (/0,0,sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + ! jacobian_ymin + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymin", jacobian2D_ymin_outer_core, & + (/0,0,sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + ! jacobian_ymax + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_ymax", jacobian2D_ymax_outer_core, & + (/0,0,sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + ! jacobian_top + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_top", jacobian2D_top_outer_core, & + (/0,0,sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + ! jacobian_bottom + call h5_read_dataset_collect_hyperslab_in_group("jacobian2D_bottom", jacobian2D_bottom_outer_core, & + (/0,0,sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + + ! close group + call h5_close_group() + end if ! NSPEC_OUTER_CORE > 0 + + if (NSPEC_INNER_CORE > 0) then + + ! change group name + write(gname_region, "('reg',i1)") IREGION_INNER_CORE + call h5_open_group(gname_region) + + ! read actual number of elements + ! nspec2D_xmin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_inner_core, (/myrank/), H5_COL) + ! nspec2D_xmax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_inner_core, (/myrank/), H5_COL) + ! nspec2D_ymin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_inner_core, (/myrank/), H5_COL) + ! nspec2D_ymax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_inner_core, (/myrank/), H5_COL) + + ! read offset arrays (stored length) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) + + ! ibelm_xmin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_inner_core, & + (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + ! ibelm_xmax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_inner_core, & + (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + ! ibelm_ymin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_inner_core, & + (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + ! ibelm_ymax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_inner_core, & + (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + ! ibelm_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_inner_core, & + (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + ! ibelm_bottom + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_inner_core, & + (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + + ! close group + call h5_close_group() + end if ! NSPEC_INNER_CORE > 0 + + if (FULL_GRAVITY) then + if (ADD_TRINF) then + if (NSPEC_TRINFINITE > 0) then + ! change group name + write(gname_region, "('reg',i1)") IREGION_TRINFINITE + call h5_open_group(gname_region) + + ! read actual number of elements + ! nspec2D_xmin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_trinfinite, (/myrank/), H5_COL) + ! nspec2D_xmax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_trinfinite, (/myrank/), H5_COL) + ! nspec2D_ymin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_trinfinite, (/myrank/), H5_COL) + ! nspec2D_ymax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_trinfinite, (/myrank/), H5_COL) + + ! read offset arrays (stored length) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) + + ! ibelm_xmin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_trinfinite, & + (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + ! ibelm_xmax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_trinfinite, & + (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + ! ibelm_ymin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_trinfinite, & + (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + ! ibelm_ymax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_trinfinite, & + (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + ! ibelm_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_trinfinite, & + (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + ! ibelm_bottom + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_trinfinite, & + (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + + ! close group + call h5_close_group() + end if ! NSPEC_TRINFINITE > 0 + end if ! ADD_TRINF + + if (NSPEC_INFINITE > 0) then + ! change group name + write(gname_region, "('reg',i1)") IREGION_INFINITE + call h5_open_group(gname_region) + + ! read actual number of elements + ! nspec2D_xmin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmin", nspec2D_xmin_infinite, (/myrank/), H5_COL) + ! nspec2D_xmax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_xmax", nspec2D_xmax_infinite, (/myrank/), H5_COL) + ! nspec2D_ymin + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymin", nspec2D_ymin_infinite, (/myrank/), H5_COL) + ! nspec2D_ymax + call h5_read_dataset_scalar_collect_hyperslab_in_group("nspec2D_ymax", nspec2D_ymax_infinite, (/myrank/), H5_COL) + + ! read offset arrays (stored length) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmin", offset_nspec2D_xmin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_xmax", offset_nspec2D_xmax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymin", offset_nspec2D_ymin, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_ymax", offset_nspec2D_ymax, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_top", offset_nspec2D_top, (/0/), H5_COL) + call h5_read_dataset_collect_hyperslab_in_group("sub_nspec2D_bottom", offset_nspec2D_bottom, (/0/), H5_COL) + + ! ibelm_xmin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmin", ibelm_xmin_infinite, & + (/sum(offset_nspec2D_xmin(0:myrank-1))/), H5_COL) + ! ibelm_xmax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_xmax", ibelm_xmax_infinite, & + (/sum(offset_nspec2D_xmax(0:myrank-1))/), H5_COL) + ! ibelm_ymin + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymin", ibelm_ymin_infinite, & + (/sum(offset_nspec2D_ymin(0:myrank-1))/), H5_COL) + ! ibelm_ymax + call h5_read_dataset_collect_hyperslab_in_group("ibelm_ymax", ibelm_ymax_infinite, & + (/sum(offset_nspec2D_ymax(0:myrank-1))/), H5_COL) + ! ibelm_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_top", ibelm_top_infinite, & + (/sum(offset_nspec2D_top(0:myrank-1))/), H5_COL) + ! ibelm_bottom + call h5_read_dataset_collect_hyperslab_in_group("ibelm_bottom", ibelm_bottom_infinite, & + (/sum(offset_nspec2D_bottom(0:myrank-1))/), H5_COL) + + ! close group + call h5_close_group() + end if ! NSPEC_INFINITE > 0 + end if ! FULL_GRAVITY + + ! close file + call h5_close_file_p() + + ! boundary mesh for crust and mantle + if (SAVE_BOUNDARY_MESH .and. SIMULATION_TYPE == 3) then + ! open boundary_disc.h5 file + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/boundary_disc.h5' + call h5_open_file_p_collect(name_database_hdf5) + + if (NSPEC_CRUST_MANTLE > 0) then + write(gname_region, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_open_group(gname_region) + + ! read NSPEC2D_MOHO + call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_MOHO", tmp_nspec2d_moho, (/myrank/), H5_COL) + ! read NSPEC2D_400 + call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_400", tmp_nspec2d_400, (/myrank/), H5_COL) + ! read NSPEC2D_670 + call h5_read_dataset_scalar_collect_hyperslab_in_group("NSPEC2D_670", tmp_nspec2d_670, (/myrank/), H5_COL) + + ! checks setup + if (tmp_nspec2d_moho /= NSPEC2D_MOHO .or. tmp_nspec2d_400 /= NSPEC2D_400 .or. tmp_nspec2d_670 /= NSPEC2D_670) then + print *,'Error: invalid NSPEC2D values read in for solver: ',& + tmp_nspec2d_moho,tmp_nspec2d_400,tmp_nspec2d_670,'(boundary_disc.h5)' + print *,' should be MOHO/400/670 : ',NSPEC2D_MOHO,NSPEC2D_400,NSPEC2D_670,'(mesh_parameters.h5)' + call exit_mpi(myrank, 'Error reading boundary_disc.h5 file') + endif + + ! read the actual number of elements + ! sub_NSPEC2D_MOHO_top + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_MOHO_top", offset_nspec2D_moho_top, (/0/), H5_COL) + ! sub_NSPEC2D_MOHO_bottom + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_MOHO_bottom", offset_nspec2D_moho_bottom, (/0/), H5_COL) + ! sub_NSPEC2D_400_top + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_400_top", offset_nspec2D_400_top, (/0/), H5_COL) + ! sub_NSPEC2D_400_bottom + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_400_bottom", offset_nspec2D_400_bottom, (/0/), H5_COL) + ! sub_NSPEC2D_670_top + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_670_top", offset_nspec2D_670_top, (/0/), H5_COL) + ! sub_NSPEC2D_670_bottom + call h5_read_dataset_collect_hyperslab_in_group("sub_NSPEC2D_670_bot", offset_nspec2D_670_bottom, (/0/), H5_COL) + + ! ibelm_moho_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_top", ibelm_moho_top, & + (/sum(offset_nspec2D_moho_top(0:myrank-1))/), H5_COL) + ! ibelm_moho_bot + call h5_read_dataset_collect_hyperslab_in_group("ibelm_moho_bot", ibelm_moho_bot, & + (/sum(offset_nspec2D_moho_bottom(0:myrank-1))/), H5_COL) + ! ibelm_400_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_top", ibelm_400_top, & + (/sum(offset_nspec2D_400_top(0:myrank-1))/), H5_COL) + ! ibelm_400_bot + call h5_read_dataset_collect_hyperslab_in_group("ibelm_400_bot", ibelm_400_bot, & + (/sum(offset_nspec2D_400_bottom(0:myrank-1))/), H5_COL) + ! ibelm_670_top + call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_top", ibelm_670_top, & + (/sum(offset_nspec2D_670_top(0:myrank-1))/), H5_COL) + ! ibelm_670_bot + call h5_read_dataset_collect_hyperslab_in_group("ibelm_670_bot", ibelm_670_bot, & + (/sum(offset_nspec2D_670_bottom(0:myrank-1))/), H5_COL) + ! normal_moho + call h5_read_dataset_collect_hyperslab_in_group("normal_moho", normal_moho, & + (/0,0,0,sum(offset_nspec2D_moho_top(0:myrank-1))/), H5_COL) + ! normal_400 + call h5_read_dataset_collect_hyperslab_in_group("normal_400", normal_400, & + (/0,0,0,sum(offset_nspec2D_400_top(0:myrank-1))/), H5_COL) + ! normal_670 + call h5_read_dataset_collect_hyperslab_in_group("normal_670", normal_670, & + (/0,0,0,sum(offset_nspec2D_670_top(0:myrank-1))/), H5_COL) + + ! close group + call h5_close_group() + end if ! NSPEC_CRUST_MANTLE > 0 + end if ! SAVE_BOUNDARY_MESH .and. SIMULATION_TYPE == 3 + + + + +#else + print* + print*,'ERROR: HDF5 support not enabled' + print*, 'Please recompile with HDF5 support with the --with-hdf5 option' + print* + stop +#endif + +end subroutine read_mesh_databases_coupling_hdf5 + + +subroutine read_mesh_databases_stacey_hdf5 + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + + ! offset + integer, dimension(0:NPROCTOT_VAL-1) :: offset_num_abs_boundary_faces + + ! hdf5 variables + integer :: comm, info, ier + character(len=64) :: gname_region + + + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize hdf5 + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! name of the file and group + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/stacey.h5' + + ! open the hdf5 file + call h5_open_file_p_collect(name_database_hdf5) + + if (NSPEC_CRUST_MANTLE > 0) then + + ! open the group + write(gname_region, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_open_group(gname_region) + + ! read the offset_num_abs_boundary_faces + call h5_read_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", offset_num_abs_boundary_faces, (/0/), H5_COL) + ! for this process + num_abs_boundary_faces_crust_mantle = offset_num_abs_boundary_faces(myrank) + + if (num_abs_boundary_faces_crust_mantle > 0) then + ! allocates absorbing boundary arrays + allocate(abs_boundary_ispec_crust_mantle(num_abs_boundary_faces_crust_mantle),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ispec') + allocate(abs_boundary_ijk_crust_mantle(3,NGLLSQUARE,num_abs_boundary_faces_crust_mantle),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ijk') + allocate(abs_boundary_jacobian2Dw_crust_mantle(NGLLSQUARE,num_abs_boundary_faces_crust_mantle),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_jacobian2Dw') + allocate(abs_boundary_normal_crust_mantle(NDIM,NGLLSQUARE,num_abs_boundary_faces_crust_mantle),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_normal') + allocate(abs_boundary_npoin_crust_mantle(num_abs_boundary_faces_crust_mantle),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_npoin') + if (ier /= 0) stop 'Error allocating array abs_boundary_ispec etc.' + + ! abs_boundary_ispec + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_crust_mantle, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + ! abs_boundary_npoin + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_crust_mantle, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + ! abs_boundary_ijk + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_crust_mantle, & + (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + ! abs_boundary_jacobian2Dw + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_crust_mantle, & + (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + ! abs_boundary_normal + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_crust_mantle, & + (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + + else ! dummy + ! dummy arrays + allocate(abs_boundary_ispec_crust_mantle(1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ispec') + allocate(abs_boundary_ijk_crust_mantle(1,1,1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ijk') + allocate(abs_boundary_jacobian2Dw_crust_mantle(1,1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_jacobian2Dw') + allocate(abs_boundary_normal_crust_mantle(1,1,1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_normal') + allocate(abs_boundary_npoin_crust_mantle(1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_npoin') + abs_boundary_ispec_crust_mantle(:) = 0; abs_boundary_npoin_crust_mantle(:) = 0 + abs_boundary_ijk_crust_mantle(:,:,:) = 0 + abs_boundary_jacobian2Dw_crust_mantle(:,:) = 0.0; abs_boundary_normal_crust_mantle(:,:,:) = 0.0 + + endif ! num_abs_boundary_faces_crust_mantle > 0 + + ! close group + call h5_close_group() + end if ! NSPEC_CRUST_MANTLE > 0 + + if (NSPEC_OUTER_CORE > 0) then + ! open the group + write(gname_region, "('reg',i1)") IREGION_OUTER_CORE + call h5_open_group(gname_region) + + ! read the offset_num_abs_boundary_faces + call h5_read_dataset_collect_hyperslab_in_group("num_abs_boundary_faces", offset_num_abs_boundary_faces, (/0/), H5_COL) + ! for this process + num_abs_boundary_faces_outer_core = offset_num_abs_boundary_faces(myrank) + + if (num_abs_boundary_faces_outer_core > 0) then + ! allocates absorbing boundary arrays + allocate(abs_boundary_ispec_outer_core(num_abs_boundary_faces_outer_core),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ispec') + allocate(abs_boundary_ijk_outer_core(3,NGLLSQUARE,num_abs_boundary_faces_outer_core),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ijk') + allocate(abs_boundary_jacobian2Dw_outer_core(NGLLSQUARE,num_abs_boundary_faces_outer_core),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_jacobian2Dw') + !allocate(abs_boundary_normal_outer_core(NDIM,NGLLSQUARE,num_abs_boundary_faces_outer_core),stat=ier) + !if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_normal') + allocate(abs_boundary_npoin_outer_core(num_abs_boundary_faces_outer_core),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_npoin') + if (ier /= 0) stop 'Error allocating array abs_boundary_ispec etc.' + + ! abs_boundary_ispec + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ispec", abs_boundary_ispec_outer_core, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + ! abs_boundary_npoin + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_npoin", abs_boundary_npoin_outer_core, & + (/sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + ! abs_boundary_ijk + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_ijk", abs_boundary_ijk_outer_core, & + (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + ! abs_boundary_jacobian2Dw + call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_jacobian2Dw", abs_boundary_jacobian2Dw_outer_core, & + (/0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + ! abs_boundary_normal + !call h5_read_dataset_collect_hyperslab_in_group("abs_boundary_normal", abs_boundary_normal_outer_core, & + ! (/0,0,sum(offset_num_abs_boundary_faces(0:myrank-1))/), H5_COL) + + else ! dummy + ! dummy arrays + allocate(abs_boundary_ispec_outer_core(1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ispec') + allocate(abs_boundary_ijk_outer_core(1,1,1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_ijk') + allocate(abs_boundary_jacobian2Dw_outer_core(1,1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_jacobian2Dw') + !allocate(abs_boundary_normal_outer_core(1,1,1),stat=ier) + !if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_normal') + allocate(abs_boundary_npoin_outer_core(1),stat=ier) + if (ier /= 0) call exit_mpi(myrank,'Error allocating array abs_boundary_npoin') + abs_boundary_ispec_outer_core(:) = 0 + abs_boundary_npoin_outer_core(:) = 0 + abs_boundary_ijk_outer_core(:,:,:) = 0 + abs_boundary_jacobian2Dw_outer_core(:,:) = 0.0 + !abs_boundary_normal_outer_core(:,:,:) = 0.0 + + endif ! num_abs_boundary_faces_outer_core > 0 + + ! close group + call h5_close_group() + end if ! NSPEC_OUTER_CORE > 0 + + ! close file + call h5_close_file_p() + + +#else + print* + print*,'ERROR: HDF5 support not enabled' + print*, 'Please recompile with HDF5 support with the --with-hdf5 option' + print* + stop +#endif + +end subroutine read_mesh_databases_stacey_hdf5 + + +subroutine read_attenuation_hdf5(iregion_code, factor_common, scale_factor, tau_s, vnspec, f_c_source) + + use constants_solver + +#ifdef USE_HDF5 + use shared_parameters, only: H5_COL + use specfem_par, only: ATTENUATION_VAL,LOCAL_PATH + use manager_hdf5 +#endif + + implicit none + + integer,intent(in) :: iregion_code + + integer,intent(in) :: vnspec + real(kind=CUSTOM_REAL), dimension(ATT1_VAL,ATT2_VAL,ATT3_VAL,vnspec),intent(inout) :: scale_factor + real(kind=CUSTOM_REAL), dimension(ATT1_VAL,ATT2_VAL,ATT3_VAL,N_SLS,vnspec),intent(inout) :: factor_common + double precision, dimension(N_SLS),intent(inout) :: tau_s + double precision,intent(inout) :: f_c_source + +#ifdef USE_HDF5 + ! offset + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nelems + ! hdf5 variables + character(len=64) :: gname_region + integer :: comm, info + double precision, dimension(0:NPROCTOT_VAL-1) :: tmp_dp_arr + +#endif + + if (.not. ATTENUATION_VAL) return + +#ifdef USE_HDF5 + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize hdf5 + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! get offset_nelems from solver_data.h5 + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/solver_data.h5' + write(gname_region, "('reg',i1)") iregion_code + ! open the hdf5 file + call h5_open_file_p_collect(name_database_hdf5) + ! open the group + call h5_open_group(gname_region) + ! read the offset_nelems + call h5_read_dataset_collect_hyperslab_in_group("offset_nelems", offset_nelems, (/0/), H5_COL) + ! close group and file + call h5_close_group() + call h5_close_file_p() + + ! open attenuation.h5 file + name_database_hdf5 = LOCAL_PATH(1:len_trim(LOCAL_PATH))//'/attenuation.h5' + ! open the hdf5 file + call h5_open_file_p_collect(name_database_hdf5) + ! open the group + call h5_open_group(gname_region) + + ! tau_s + call h5_read_dataset_collect_hyperslab_in_group("tau_s_store", tau_s, (/myrank*N_SLS/), H5_COL) + ! factor_common (tau_e_store) + call h5_read_dataset_collect_hyperslab_in_group("tau_e_store", factor_common, (/0,0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! scale_factor (Qmu_store) + call h5_read_dataset_collect_hyperslab_in_group("Qmu_store", scale_factor, (/0,0,0,sum(offset_nelems(0:myrank-1))/), H5_COL) + ! f_c_source + call h5_read_dataset_collect_hyperslab_in_group("att_f_c_source", tmp_dp_arr, (/0/), H5_COL) + f_c_source = tmp_dp_arr(myrank) + + ! close group + call h5_close_group() + ! close file + call h5_close_file_p() + +#else + print* + print*,'ERROR: HDF5 support not enabled' + print*, 'Please recompile with HDF5 support with the --with-hdf5 option' + print* + stop +#endif + +end subroutine read_attenuation_hdf5 diff --git a/src/specfem3D/read_forward_arrays.F90 b/src/specfem3D/read_forward_arrays.F90 index 763fae92e..4e5d35980 100644 --- a/src/specfem3D/read_forward_arrays.F90 +++ b/src/specfem3D/read_forward_arrays.F90 @@ -57,6 +57,8 @@ subroutine read_forward_arrays_startrun() if (ADIOS_FOR_FORWARD_ARRAYS) then call read_intermediate_forward_arrays_adios() + else if (HDF5_ENABLED) then + call read_intermediate_forward_arrays_hdf5() else write(outputname,"('dump_all_arrays',i6.6)") myrank outputname = trim(LOCAL_TMP_PATH) // '/' // outputname(1:len_trim(outputname)) @@ -166,6 +168,8 @@ subroutine read_forward_arrays() ! reads in file data if (ADIOS_FOR_FORWARD_ARRAYS) then call read_forward_arrays_adios() + else if (HDF5_ENABLED) then + call read_forward_arrays_hdf5() else write(outputname,'(a,i6.6,a)') 'proc',myrank,'_save_forward_arrays.bin' outputname = trim(LOCAL_TMP_PATH) // '/' // outputname(1:len_trim(outputname)) @@ -316,6 +320,8 @@ subroutine read_forward_arrays_undoatt() if (ADIOS_FOR_UNDO_ATTENUATION) then call read_forward_arrays_undoatt_adios(iteration_on_subset_tmp) + else if (HDF5_ENABLED) then + call read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) else ! reads in saved wavefield write(outputname,'(a,i6.6,a,i6.6,a)') 'proc',myrank,'_save_frame_at',iteration_on_subset_tmp,'.bin' diff --git a/src/specfem3D/read_forward_arrays_hdf5.F90 b/src/specfem3D/read_forward_arrays_hdf5.F90 new file mode 100644 index 000000000..1023a50c4 --- /dev/null +++ b/src/specfem3D/read_forward_arrays_hdf5.F90 @@ -0,0 +1,488 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + subroutine read_intermediate_forward_arrays_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! full gravity + integer :: neq_read,neq1_read + + ! MPI variables + integer :: info, comm + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name + + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/dump_all_arrays.h5' + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the file + call h5_open_file_p_collect(file_name) + + ! read the arrays + call h5_read_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/0,sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + + if (ROTATION_VAL) then + call h5_read_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', B_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + endif + + if (ATTENUATION_VAL) then + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + endif + + if (FULL_GRAVITY_VAL) then + call h5_read_dataset_scalar_collect_hyperslab('neq', neq_read, (/myrank/), H5_COL) + call h5_read_dataset_scalar_collect_hyperslab('neq1', neq1_read, (/myrank/), H5_COL) + call h5_read_dataset_collect_hyperslab('pgrav1', pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), H5_COL) + endif + + ! close the file + call h5_close_file_p() + +#else + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') +#endif + + end subroutine read_intermediate_forward_arrays_hdf5 + + + subroutine read_forward_arrays_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! full gravity + integer :: b_neq_read, b_neq1_read + + ! MPI variables + integer :: info, comm + + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name + + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle,2), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core,2), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core,1), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/dump_all_arrays.h5' + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the file + call h5_open_file_p_collect(file_name) + + ! read the arrays + call h5_read_dataset_collect_hyperslab('displ_crust_mantle', b_displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_inner_core', b_displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', b_veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_inner_core', b_veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + + if (ROTATION_VAL) then + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + endif + + if (ATTENUATION_VAL) then + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + endif + + if (FULL_GRAVITY_VAL) then + call h5_read_dataset_scalar_collect_hyperslab('neq', b_neq_read, (/myrank/), H5_COL) + call h5_read_dataset_scalar_collect_hyperslab('neq1', b_neq1_read, (/myrank/), H5_COL) + + ! check if array sizes match + if (b_neq_read /= neq) then + print *,'Error reading forward array for startrun: rank ',myrank,'has read neq =',b_neq_read,' - shoud be ',neq + call exit_MPI(myrank,'Invalid forward array neq for startrun') + endif + if (b_neq1_read /= neq1) then + print *,'Error reading forward array for startrun: rank ',myrank,'has read neq1 =',b_neq1_read,' - shoud be ',neq1 + call exit_MPI(myrank,'Invalid forward array neq1 for startrun') + endif + + call h5_read_dataset_collect_hyperslab('pgrav1', b_pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), H5_COL) + endif + + ! close the file + call h5_close_file_p() + +#else + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') +#endif + + end subroutine read_forward_arrays_hdf5 + + subroutine read_forward_arrays_undoatt_hdf5(iteration_on_subset_tmp) + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + ! input parameter + integer, intent(in) :: iteration_on_subset_tmp + +#ifdef USE_HDF5 + + ! full gravity + integer :: b_neq_read, b_neq1_read + + ! MPI variables + integer :: info, comm + + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name + + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle,2), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core,2), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core,1), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + + write(file_name, '(a,i6.6,a)') 'save_frame_at',iteration_on_subset_tmp,'.h5' + file_name = trim(LOCAL_PATH)//'/'//trim(file_name) + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! open the file + call h5_open_file_p_collect(file_name) + + ! read the arrays + call h5_read_dataset_collect_hyperslab('displ_crust_mantle', b_displ_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_outer_core', b_displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('displ_inner_core', b_displ_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_crust_mantle', b_veloc_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_outer_core', b_veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('veloc_inner_core', b_veloc_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_crust_mantle', b_accel_crust_mantle, (/0,sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_outer_core', b_accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('accel_inner_core', b_accel_inner_core, (/0,sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', b_epsilondev_xx_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', b_epsilondev_yy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', b_epsilondev_xy_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', b_epsilondev_xz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', b_epsilondev_yz_crust_mantle, & + (/0,0,0,sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xx_inner_core', b_epsilondev_xx_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yy_inner_core', b_epsilondev_yy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xy_inner_core', b_epsilondev_xy_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_xz_inner_core', b_epsilondev_xz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('epsilondev_yz_inner_core', b_epsilondev_yz_inner_core, & + (/0,0,0,sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + + if (ROTATION_VAL) then + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_A_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('A_array_rotation', b_B_array_rotation, & + (/0,0,0,sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + endif + + if (ATTENUATION_VAL) then + call h5_read_dataset_collect_hyperslab('R_xx_crust_mantle', b_R_xx_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_crust_mantle', b_R_yy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_crust_mantle', b_R_xy_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_crust_mantle', b_R_xz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_crust_mantle', b_R_yz_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xx_inner_core', b_R_xx_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yy_inner_core', b_R_yy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xy_inner_core', b_R_xy_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_xz_inner_core', b_R_xz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_read_dataset_collect_hyperslab('R_yz_inner_core', b_R_yz_inner_core, & + (/0,0,0,0,sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + endif + + if (FULL_GRAVITY_VAL) then + call h5_read_dataset_scalar_collect_hyperslab('neq', b_neq_read, (/myrank/), H5_COL) + call h5_read_dataset_scalar_collect_hyperslab('neq1', b_neq1_read, (/myrank/), H5_COL) + + ! check if array sizes match + if (b_neq_read /= neq) then + print *,'Error reading forward array for startrun: rank ',myrank,'has read neq =',b_neq_read,' - shoud be ',neq + call exit_MPI(myrank,'Invalid forward array neq for startrun') + endif + if (b_neq1_read /= neq1) then + print *,'Error reading forward array for startrun: rank ',myrank,'has read neq1 =',b_neq1_read,' - shoud be ',neq1 + call exit_MPI(myrank,'Invalid forward array neq1 for startrun') + endif + + call h5_read_dataset_collect_hyperslab('pgrav1', b_pgrav1, (/0,sum(offset_pgrav1(0:myrank-1))/), H5_COL) + endif + + ! close the file + call h5_close_file_p() + +#else + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') +#endif + + end subroutine read_forward_arrays_undoatt_hdf5 diff --git a/src/specfem3D/read_mesh_databases.F90 b/src/specfem3D/read_mesh_databases.F90 index 18010bf77..4e307d1ee 100644 --- a/src/specfem3D/read_mesh_databases.F90 +++ b/src/specfem3D/read_mesh_databases.F90 @@ -387,7 +387,30 @@ subroutine read_mesh_databases_CM() rmassx_crust_mantle,rmassy_crust_mantle,rmassz_crust_mantle, & NGLOB_CRUST_MANTLE_OCEANS,rmass_ocean_load, & b_rmassx_crust_mantle,b_rmassy_crust_mantle) - else + else if (HDF5_ENABLED) then + call read_arrays_solver_hdf5(IREGION_CRUST_MANTLE, & + NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE,NGLOB_XY_CM, & + nspec_iso,nspec_tiso,nspec_ani, & + rho_vp_crust_mantle,rho_vs_crust_mantle, & + xstore_crust_mantle,ystore_crust_mantle,zstore_crust_mantle, & + xix_crust_mantle,xiy_crust_mantle,xiz_crust_mantle, & + etax_crust_mantle,etay_crust_mantle,etaz_crust_mantle, & + gammax_crust_mantle,gammay_crust_mantle,gammaz_crust_mantle, & + rhostore_crust_mantle,kappavstore_crust_mantle,muvstore_crust_mantle, & + kappahstore_crust_mantle,muhstore_crust_mantle,eta_anisostore_crust_mantle, & + c11store_crust_mantle,c12store_crust_mantle,c13store_crust_mantle, & + c14store_crust_mantle,c15store_crust_mantle,c16store_crust_mantle, & + c22store_crust_mantle,c23store_crust_mantle,c24store_crust_mantle, & + c25store_crust_mantle,c26store_crust_mantle,c33store_crust_mantle, & + c34store_crust_mantle,c35store_crust_mantle,c36store_crust_mantle, & + c44store_crust_mantle,c45store_crust_mantle,c46store_crust_mantle, & + c55store_crust_mantle,c56store_crust_mantle,c66store_crust_mantle, & + mu0store_crust_mantle, & + ibool_crust_mantle,dummy_idoubling,ispec_is_tiso_crust_mantle, & + rmassx_crust_mantle,rmassy_crust_mantle,rmassz_crust_mantle, & + NGLOB_CRUST_MANTLE_OCEANS,rmass_ocean_load, & + b_rmassx_crust_mantle,b_rmassy_crust_mantle) + else call read_arrays_solver(IREGION_CRUST_MANTLE, & NSPEC_CRUST_MANTLE,NGLOB_CRUST_MANTLE,NGLOB_XY_CM, & nspec_iso,nspec_tiso,nspec_ani, & @@ -582,6 +605,29 @@ subroutine read_mesh_databases_OC() dummy_rmass,dummy_rmass,rmass_outer_core, & 1,dummy_array, & dummy_rmass,dummy_rmass) + else if (HDF5_ENABLED) then + call read_arrays_solver_hdf5(IREGION_OUTER_CORE, & + NSPEC_OUTER_CORE,NGLOB_OUTER_CORE,NGLOB_XY_dummy, & + nspec_iso,nspec_tiso,nspec_ani, & + vp_outer_core,dummy_array, & + xstore_outer_core,ystore_outer_core,zstore_outer_core, & + xix_outer_core,xiy_outer_core,xiz_outer_core, & + etax_outer_core,etay_outer_core,etaz_outer_core, & + gammax_outer_core,gammay_outer_core,gammaz_outer_core, & + rhostore_outer_core,kappavstore_outer_core,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array, & + ibool_outer_core,dummy_idoubling_outer_core,dummy_ispec_is_tiso, & + dummy_rmass,dummy_rmass,rmass_outer_core, & + 1, dummy_array, & + dummy_rmass,dummy_rmass) else call read_arrays_solver(IREGION_OUTER_CORE, & NSPEC_OUTER_CORE,NGLOB_OUTER_CORE,NGLOB_XY_dummy, & @@ -759,6 +805,29 @@ subroutine read_mesh_databases_IC() rmassx_inner_core,rmassy_inner_core,rmassz_inner_core, & 1,dummy_array, & b_rmassx_inner_core,b_rmassy_inner_core) + else if (HDF5_ENABLED) then + call read_arrays_solver_hdf5(IREGION_INNER_CORE, & + NSPEC_INNER_CORE,NGLOB_INNER_CORE,NGLOB_XY_IC, & + nspec_iso,nspec_tiso,nspec_ani, & + dummy_array,dummy_array, & + xstore_inner_core,ystore_inner_core,zstore_inner_core, & + xix_inner_core,xiy_inner_core,xiz_inner_core, & + etax_inner_core,etay_inner_core,etaz_inner_core, & + gammax_inner_core,gammay_inner_core,gammaz_inner_core, & + rhostore_inner_core,kappavstore_inner_core,muvstore_inner_core, & + dummy_array,dummy_array,dummy_array, & + c11store_inner_core,c12store_inner_core,c13store_inner_core, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,c33store_inner_core, & + dummy_array,dummy_array,dummy_array, & + c44store_inner_core,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array, & + ibool_inner_core,idoubling_inner_core,dummy_ispec_is_tiso, & + rmassx_inner_core,rmassy_inner_core,rmassz_inner_core, & + 1,dummy_array, & + b_rmassx_inner_core,b_rmassy_inner_core) else call read_arrays_solver(IREGION_INNER_CORE, & NSPEC_INNER_CORE,NGLOB_INNER_CORE,NGLOB_XY_IC, & @@ -931,6 +1000,29 @@ subroutine read_mesh_databases_TRINF() dummy_rmass,dummy_rmass,dummy_rmass, & 1,dummy_array, & dummy_rmass,dummy_rmass) + else if (HDF5_ENABLED) then + call read_arrays_solver_hdf5(IREGION_TRINFINITE, & + NSPEC_TRINFINITE,NGLOB_TRINFINITE,NGLOB_XY_dummy, & + nspec_iso,nspec_tiso,nspec_ani, & + dummy_array,dummy_array, & + xstore_trinfinite,ystore_trinfinite,zstore_trinfinite, & + xix_trinfinite,xiy_trinfinite,xiz_trinfinite, & + etax_trinfinite,etay_trinfinite,etaz_trinfinite, & + gammax_trinfinite,gammay_trinfinite,gammaz_trinfinite, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array, & + ibool_trinfinite,dummy_idoubling,dummy_ispec_is_tiso, & + dummy_rmass,dummy_rmass,dummy_rmass, & + 1,dummy_array, & + dummy_rmass,dummy_rmass) else call read_arrays_solver(IREGION_TRINFINITE, & NSPEC_TRINFINITE,NGLOB_TRINFINITE,NGLOB_XY_dummy, & @@ -1061,6 +1153,29 @@ subroutine read_mesh_databases_INF() dummy_rmass,dummy_rmass,dummy_rmass, & 1,dummy_array, & dummy_rmass,dummy_rmass) + else if (HDF5_ENABLED) then + call read_arrays_solver_hdf5(IREGION_INFINITE, & + NSPEC_INFINITE,NGLOB_INFINITE,NGLOB_XY_dummy, & + nspec_iso,nspec_tiso,nspec_ani, & + dummy_array,dummy_array, & + xstore_infinite,ystore_infinite,zstore_infinite, & + xix_infinite,xiy_infinite,xiz_infinite, & + etax_infinite,etay_infinite,etaz_infinite, & + gammax_infinite,gammay_infinite,gammaz_infinite, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array,dummy_array,dummy_array, & + dummy_array, & + ibool_infinite,dummy_idoubling,dummy_ispec_is_tiso, & + dummy_rmass,dummy_rmass,dummy_rmass, & + 1,dummy_array, & + dummy_rmass,dummy_rmass) else call read_arrays_solver(IREGION_INFINITE, & NSPEC_INFINITE,NGLOB_INFINITE,NGLOB_XY_dummy, & @@ -1238,6 +1353,8 @@ subroutine read_mesh_databases_coupling() if (I_should_read_the_database) then if (ADIOS_FOR_ARRAYS_SOLVER) then call read_mesh_databases_coupling_adios() + else if (HDF5_ENABLED) then + call read_mesh_databases_coupling_hdf5() else ! crust and mantle if (NSPEC_CRUST_MANTLE > 0) then @@ -1450,7 +1567,7 @@ subroutine read_mesh_databases_coupling() endif endif - endif ! ADIOS + endif ! ADIOS or HDF5 endif call bcast_mesh_databases_coupling() @@ -1670,6 +1787,8 @@ subroutine read_mesh_databases_MPI() if (I_should_read_the_database .and. NSPEC_CRUST_MANTLE > 0) then if (ADIOS_FOR_MPI_ARRAYS) then call read_mesh_databases_MPI_adios(IREGION_CRUST_MANTLE) + else if (HDF5_ENABLED) then + call read_mesh_databases_MPI_hdf5(IREGION_CRUST_MANTLE) else call read_mesh_databases_MPI_CM() endif @@ -1767,6 +1886,8 @@ subroutine read_mesh_databases_MPI() if (I_should_read_the_database .and. NSPEC_OUTER_CORE > 0) then if (ADIOS_FOR_MPI_ARRAYS) then call read_mesh_databases_MPI_adios(IREGION_OUTER_CORE) + else if (HDF5_ENABLED) then + call read_mesh_databases_MPI_hdf5(IREGION_OUTER_CORE) else call read_mesh_databases_MPI_OC() endif @@ -1861,6 +1982,8 @@ subroutine read_mesh_databases_MPI() if (I_should_read_the_database .and. NSPEC_INNER_CORE > 0) then if (ADIOS_FOR_MPI_ARRAYS) then call read_mesh_databases_MPI_adios(IREGION_INNER_CORE) + else if (HDF5_ENABLED) then + call read_mesh_databases_MPI_hdf5(IREGION_INNER_CORE) else call read_mesh_databases_MPI_IC() endif @@ -1961,6 +2084,8 @@ subroutine read_mesh_databases_MPI() if (I_should_read_the_database .and. NSPEC_TRINFINITE > 0) then if (ADIOS_FOR_MPI_ARRAYS) then call read_mesh_databases_MPI_adios(IREGION_TRINFINITE) + else if (HDF5_ENABLED) then + call read_mesh_databases_MPI_hdf5(IREGION_TRINFINITE) else call read_mesh_databases_MPI_TRINF() endif @@ -2056,6 +2181,8 @@ subroutine read_mesh_databases_MPI() if (I_should_read_the_database .and. NSPEC_INFINITE > 0) then if (ADIOS_FOR_MPI_ARRAYS) then call read_mesh_databases_MPI_adios(IREGION_INFINITE) + else if (HDF5_ENABLED) then + call read_mesh_databases_MPI_hdf5(IREGION_INFINITE) else call read_mesh_databases_MPI_INF() endif @@ -2671,6 +2798,8 @@ subroutine read_mesh_databases_stacey() if (I_should_read_the_database) then if (ADIOS_FOR_ARRAYS_SOLVER) then call read_mesh_databases_stacey_adios() + else if (HDF5_ENABLED) then + call read_mesh_databases_stacey_hdf5() else ! crust and mantle if (NSPEC_CRUST_MANTLE > 0) then @@ -2798,6 +2927,8 @@ subroutine read_mesh_databases_regular_kl() ! checks setup if (ADIOS_FOR_KERNELS) & call exit_mpi(myrank,'saving regular kernels in ADIOS file format is not supported yet') + if (HDF5_ENABLED) & + call exit_mpi(myrank,'saving regular kernels in HDF5 file format is not supported yet') ! assuming 6 chunks full global simulations right now if (NCHUNKS_VAL /= 6 .or. NPROC_XI_VAL /= NPROC_ETA_VAL) & call exit_MPI(myrank, 'Only deal with 6 chunks at this moment') diff --git a/src/specfem3D/rules.mk b/src/specfem3D/rules.mk index e0b41b33d..40461bd97 100644 --- a/src/specfem3D/rules.mk +++ b/src/specfem3D/rules.mk @@ -110,7 +110,9 @@ specfem3D_SOLVER_OBJECTS += \ $O/print_stf_file.solverstatic.o \ $O/read_adjoint_sources.solverstatic.o \ $O/read_arrays_solver.solverstatic.o \ + $O/read_arrays_solver_hdf5.solverstatic.o \ $O/read_forward_arrays.solverstatic.o \ + $O/read_forward_arrays_hdf5.solverstatic.o \ $O/read_mesh_parameters.solverstatic.o \ $O/read_mesh_databases.solverstatic.o \ $O/read_topography_bathymetry.solverstatic.o \ @@ -125,7 +127,9 @@ specfem3D_SOLVER_OBJECTS += \ $O/SIEM_solver_mpi.solverstatic.o \ $O/SIEM_solver_petsc.solverstatic.o \ $O/save_forward_arrays.solverstatic.o \ + $O/save_forward_arrays_hdf5.solverstatic.o \ $O/save_kernels.solverstatic.o \ + $O/save_kernels_hdf5.solverstatic.o \ $O/save_regular_kernels.solverstatic.o \ $O/setup_GLL_points.solverstatic.o \ $O/setup_sources_receivers.solverstatic.o \ @@ -135,9 +139,12 @@ specfem3D_SOLVER_OBJECTS += \ $O/update_displacement_Newmark.solverstatic.o \ $O/write_movie_output.solverstatic.o \ $O/write_movie_volume.solverstatic.o \ + $O/write_movie_volume_hdf5.solverstatic.o \ $O/write_movie_surface.solverstatic.o \ + $O/write_movie_surface_hdf5.solverstatic.o \ $O/write_output_ASCII.solverstatic.o \ $O/write_output_SAC.solverstatic.o \ + $O/write_output_HDF5.solverstatic.o \ $O/write_seismograms.solverstatic.o \ $(EMPTY_MACRO) @@ -186,6 +193,7 @@ specfem3D_SHARED_OBJECTS = \ $O/gll_library.shared.o \ $O/heap_sort.shared.o \ $O/hex_nodes.shared.o \ + $O/hdf5_manager.shared_hdf5_module.o \ $O/init_openmp.shared.o \ $O/intgrl.shared.o \ $O/lagrange_poly.shared.o \ @@ -353,6 +361,17 @@ ifeq ($(PETSC),yes) specfem3D_MODULES += $(FC_MODDIR)/siem_solver_petsc.$(FC_MODEXT) endif + +## +## HDF5 +## + +ifeq ($(HDF5),yes) +specfem3D_MODULES += \ + $(FC_MODDIR)/specfem_par_movie_hdf5.$(FC_MODEXT) \ + $(EMPTY_MACRO) +endif + ####################################### #### @@ -436,6 +455,7 @@ $O/SIEM_solver_mpi.solverstatic.o: $O/SIEM_math_library.shared.o $O/SIEM_solver_petsc.solverstatic.o: $O/SIEM_math_library.shared.o $O/SIEM_compute_seismograms.solverstatic.o: $O/SIEM_math_library.shared.o + ### ### specfem3D - optimized flags and dependence on values from mesher here ### @@ -491,4 +511,4 @@ $O/%.visualc.o: $S/%.c ${SETUP}/config.h ### CEM ### $O/%.checknetcdf.o: $S/%.f90 $O/shared_par.shared_module.o $O/specfem3D_par.solverstatic_module.o - ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(NETCDF_INCLUDE) -c -o $@ $< + ${FCCOMPILE_CHECK} ${FCFLAGS_f90} $(NETCDF_INCLUDE) -c -o $@ $< \ No newline at end of file diff --git a/src/specfem3D/save_forward_arrays.F90 b/src/specfem3D/save_forward_arrays.F90 index 2f44a4f52..637cca9ef 100644 --- a/src/specfem3D/save_forward_arrays.F90 +++ b/src/specfem3D/save_forward_arrays.F90 @@ -66,6 +66,8 @@ subroutine save_forward_arrays() ! saves checkpoint if (ADIOS_FOR_FORWARD_ARRAYS) then call save_intermediate_forward_arrays_adios() + else if (HDF5_ENABLED) then + call save_intermediate_forward_arrays_hdf5() else write(outputname,"('dump_all_arrays',i6.6)") myrank open(unit=IOUT,file=trim(LOCAL_TMP_PATH)//'/'//trim(outputname), & @@ -141,6 +143,8 @@ subroutine save_forward_arrays() if (ADIOS_FOR_FORWARD_ARRAYS) then call save_forward_arrays_adios() + else if (HDF5_ENABLED) then + call save_forward_arrays_hdf5() else write(outputname,'(a,i6.6,a)') 'proc',myrank,'_save_forward_arrays.bin' outputname = trim(LOCAL_TMP_PATH)//'/'//trim(outputname) @@ -246,6 +250,8 @@ subroutine save_forward_arrays_undoatt() if (ADIOS_FOR_UNDO_ATTENUATION) then call save_forward_arrays_undoatt_adios() + else if (HDF5_ENABLED) then + call save_forward_arrays_undoatt_hdf5() else ! current subset iteration iteration_on_subset_tmp = iteration_on_subset diff --git a/src/specfem3D/save_forward_arrays_hdf5.F90 b/src/specfem3D/save_forward_arrays_hdf5.F90 new file mode 100644 index 000000000..8962127fc --- /dev/null +++ b/src/specfem3D/save_forward_arrays_hdf5.F90 @@ -0,0 +1,984 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + +subroutine save_intermediate_forward_arrays_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + + ! MPI variables + integer :: info, comm + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name + + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle,2), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core,2), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core,1), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/dump_all_arrays.h5' + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create file and datasets by myrank==0 + if (myrank == 0) then + call h5_create_file(file_name) + + ! create datasets + call h5_create_dataset_gen('displ_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('accel_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + + if (ROTATION_VAL) then + call h5_create_dataset_gen('A_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('B_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + endif + + if (ATTENUATION_VAL) then + call h5_create_dataset_gen('R_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + + call h5_create_dataset_gen('R_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_create_dataset_gen('neq', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('neq1', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('pgrav1', (/sum(offset_pgrav1)/), 1, CUSTOM_REAL) + endif ! FULL_GRAVITY_VAL + + ! close file + call h5_close_file() + endif ! myrank == 0 + + call synchronize_all() + + ! write data from all ranks + call h5_open_file_p_collect(file_name) + + ! write datasets + call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + + if (ROTATION_VAL) then + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + endif + + if (ATTENUATION_VAL) then + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), H5_COL) + endif ! FULL_GRAVITY_VAL + + ! close file + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + + +end subroutine save_intermediate_forward_arrays_hdf5 + + +subroutine save_forward_arrays_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! MPI variables + integer :: info, comm + + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name + + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle,2), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core,2), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core,1), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/save_forward_arrays.h5' + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create file and datasets by myrank==0 + if (myrank == 0) then + call h5_create_file(file_name) + + call h5_create_dataset_gen('displ_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('accel_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_inner_core', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_inner_core', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_inner_core', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_inner_core', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_inner_core', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + + if (ROTATION_VAL) then + call h5_create_dataset_gen('A_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('B_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + endif + + if (ATTENUATION_VAL) then + call h5_create_dataset_gen('R_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + + call h5_create_dataset_gen('R_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_create_dataset_gen('neq', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('neq1', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('pgrav1', (/sum(offset_pgrav1)/), 1, CUSTOM_REAL) + endif + + ! close file + call h5_close_file() + + endif ! myrank == 0 + + call synchronize_all() + + ! write data from all ranks + call h5_open_file_p_collect(file_name) + + ! write datasets + call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + if (ROTATION_VAL) then + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + endif + if (ATTENUATION_VAL) then + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), H5_COL) + endif ! FULL_GRAVITY_VAL + + ! close file + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + + +end subroutine save_forward_arrays_hdf5 + + +subroutine save_forward_arrays_undoatt_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + use specfem_par_outercore + use specfem_par_full_gravity + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + + ! MPI variables + integer :: info, comm + + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_mc_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nglob_ic_str_or_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_rot + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_att + integer, dimension(0:NPROCTOT_VAL-1) :: offset_pgrav1 + + character(len=MAX_STRING_LEN) :: file_name + + ! gather the offset arrays + call gather_all_all_singlei(size(displ_crust_mantle,2), offset_nglob_cm, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_inner_core,2), offset_nglob_ic, NPROCTOT_VAL) + call gather_all_all_singlei(size(displ_outer_core,1), offset_nglob_oc, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_crust_mantle,4), offset_nglob_mc_str_or_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(epsilondev_xx_inner_core,4), offset_nglob_ic_str_or_att, NPROCTOT_VAL) + if (ROTATION_VAL) then + call gather_all_all_singlei(size(A_array_rotation,4), offset_nspec_oc_rot, NPROCTOT_VAL) + endif + if (ATTENUATION_VAL) then + call gather_all_all_singlei(size(R_xx_crust_mantle,5), offset_nspec_cm_att, NPROCTOT_VAL) + call gather_all_all_singlei(size(R_xx_inner_core,5), offset_nspec_ic_att, NPROCTOT_VAL) + endif + if (FULL_GRAVITY_VAL) then + call gather_all_all_singlei(size(pgrav1), offset_pgrav1, NPROCTOT_VAL) + endif + + write(file_name, '(a,i6.6,a)') 'save_frame_at',iteration_on_subset,'.h5' + file_name = trim(LOCAL_PATH)//'/'//trim(file_name) + + ! get MPI parameters + call world_get_comm(comm) + call world_get_info_null(info) + + ! initialize HDF5 + call h5_initialize() ! called in initialize_mesher() + ! set MPI + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create file and datasets by myrank==0 + if (myrank == 0) then + call h5_create_file(file_name) + + ! create datasets + call h5_create_dataset_gen('displ_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_crust_mantle', (/NDIM, sum(offset_nglob_cm)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('displ_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('accel_outer_core', (/sum(offset_nglob_oc)/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('displ_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('veloc_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('accel_inner_core', (/NDIM, sum(offset_nglob_ic)/), 2, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_crust_mantle', & + (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_mc_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('epsilondev_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, sum(offset_nglob_ic_str_or_att)/), 4, CUSTOM_REAL) + + if (ROTATION_VAL) then + call h5_create_dataset_gen('A_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('B_array_rotation', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_oc_rot)/), 4, CUSTOM_REAL) + endif + + if (ATTENUATION_VAL) then + call h5_create_dataset_gen('R_xx_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_crust_mantle', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_cm_att)/), 5, CUSTOM_REAL) + + call h5_create_dataset_gen('R_xx_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xy_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_xz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + call h5_create_dataset_gen('R_yz_inner_core', (/NGLLX, NGLLY, NGLLZ, N_SLS, sum(offset_nspec_ic_att)/), 5, CUSTOM_REAL) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_create_dataset_gen('neq', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('neq1', (/NPROCTOT_VAL/), 1, CUSTOM_REAL) + call h5_create_dataset_gen('pgrav1', (/sum(offset_pgrav1)/), 1, CUSTOM_REAL) + endif ! FULL_GRAVITY_VAL + + ! close file + call h5_close_file() + endif ! myrank == 0 + + call synchronize_all() + + ! write data from all ranks + call h5_open_file_p_collect(file_name) + + ! write datasets + call h5_write_dataset_collect_hyperslab('displ_crust_mantle', displ_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_crust_mantle', veloc_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_crust_mantle', accel_crust_mantle, (/0, sum(offset_nglob_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_outer_core', displ_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_outer_core', veloc_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_outer_core', accel_outer_core, (/sum(offset_nglob_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('displ_inner_core', displ_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('veloc_inner_core', veloc_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('accel_inner_core', accel_inner_core, (/0, sum(offset_nglob_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_crust_mantle', epsilondev_xx_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_crust_mantle', epsilondev_yy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_crust_mantle', epsilondev_xy_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_crust_mantle', epsilondev_xz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_crust_mantle', epsilondev_yz_crust_mantle, & + (/0, 0, 0, sum(offset_nglob_mc_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xx_inner_core', epsilondev_xx_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yy_inner_core', epsilondev_yy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xy_inner_core', epsilondev_xy_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_xz_inner_core', epsilondev_xz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('epsilondev_yz_inner_core', epsilondev_yz_inner_core, & + (/0, 0, 0, sum(offset_nglob_ic_str_or_att(0:myrank-1))/), H5_COL) + + if (ROTATION_VAL) then + call h5_write_dataset_collect_hyperslab('A_array_rotation', A_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('B_array_rotation', B_array_rotation, & + (/0, 0, 0, sum(offset_nspec_oc_rot(0:myrank-1))/), H5_COL) + endif + + if (ATTENUATION_VAL) then + call h5_write_dataset_collect_hyperslab('R_xx_crust_mantle', R_xx_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_crust_mantle', R_yy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_crust_mantle', R_xy_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_crust_mantle', R_xz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_crust_mantle', R_yz_crust_mantle, & + (/0, 0, 0, 0, sum(offset_nspec_cm_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xx_inner_core', R_xx_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yy_inner_core', R_yy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xy_inner_core', R_xy_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_xz_inner_core', R_xz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('R_yz_inner_core', R_yz_inner_core, & + (/0, 0, 0, 0, sum(offset_nspec_ic_att(0:myrank-1))/), H5_COL) + endif ! ATTENUATION_VAL + + if (FULL_GRAVITY_VAL) then + call h5_write_dataset_collect_hyperslab('neq', (/neq/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('neq1', (/neq1/), (/myrank/), H5_COL) + call h5_write_dataset_collect_hyperslab('pgrav1', pgrav1, (/sum(offset_pgrav1(0:myrank-1))/), H5_COL) + endif ! FULL_GRAVITY_VAL + + ! close file + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine save_forward_arrays_undoatt_hdf5 + + +subroutine save_forward_model_at_shifted_frequency_hdf5(factor_scale_relaxed_crust_mantle,factor_scale_relaxed_inner_core) + + use constants + + use specfem_par_crustmantle + use specfem_par_innercore + +#ifdef USE_HDF5 + use shared_parameters, only: R_PLANET,RHOAV,LOCAL_PATH,TRANSVERSE_ISOTROPY,H5_COL + use manager_hdf5 +#endif + + implicit none + + real(kind=CUSTOM_REAL),dimension(ATT1_VAL,ATT2_VAL,ATT3_VAL,ATT4_VAL) :: factor_scale_relaxed_crust_mantle + real(kind=CUSTOM_REAL),dimension(ATT1_VAL,ATT2_VAL,ATT3_VAL,ATT5_VAL) :: factor_scale_relaxed_inner_core + +#ifdef USE_HDF5 + + ! local parameters + integer :: ier + real(kind=CUSTOM_REAL) :: scaleval1,scale_factor_r + real(kind=CUSTOM_REAL),dimension(:,:,:,:),allocatable :: temp_store + real(kind=CUSTOM_REAL),dimension(:,:,:,:),allocatable :: muv_shifted,muh_shifted + integer :: i,j,k,ispec + + ! debug + logical, parameter :: OUTPUT_RELAXED_MODEL = .false. + + ! TODO HDF5: put offset array creation in a initialization process + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic + character(len=MAX_STRING_LEN) :: file_name, group_name + + ! gather the offset arrays + call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_cm, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_oc, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_ic, NPROCTOT_VAL) + + ! file name + file_name = trim(LOCAL_PATH)//'/'//'model_shifted.h5' + + ! create file and datasets by myrank==0 + if (myrank == 0) then + call h5_create_file(file_name) + + if (NSPEC_CRUST_MANTLE > 0) then + ! safety check + if (ANISOTROPIC_3D_MANTLE_VAL) & + call exit_mpi(myrank,'ANISOTROPIC_3D_MANTLE not supported yet for shifted model file output') + + ! group name + write(group_name, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_create_group(group_name) + call h5_open_group(group_name) + + ! create datasets + call h5_create_dataset_gen_in_group('muv_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('muh_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + + if (TRANSVERSE_ISOTROPY) then + call h5_create_dataset_gen_in_group('vpv_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('vph_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('vsv_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('vsh_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + else ! isotropic + call h5_create_dataset_gen_in_group('vp_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('vs_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + endif + + ! close group + call h5_close_group() + endif ! NSPEC_CRUST_MANTLE > 0 + + if (NSPEC_INNER_CORE > 0) then + if (ANISOTROPIC_INNER_CORE_VAL) then + call exit_mpi(myrank,'ANISOTROPIC_INNER_CORE not supported yet for shifted model file output') + else + ! only isotropic inner core supported + + ! group name + write(group_name, "('reg',i1)") IREGION_INNER_CORE + call h5_create_group(group_name) + call h5_open_group(group_name) + + ! create datasets + call h5_create_dataset_gen_in_group('vp_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_ic)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('vs_shifted', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_ic)/), 4, CUSTOM_REAL) + endif + + ! close group + call h5_close_group() + endif + + ! output relaxed model values + if (OUTPUT_RELAXED_MODEL) then + ! checks + if (.not. TRANSVERSE_ISOTROPY) stop 'Outputting relaxed model requires TRANSVERSE_ISOTROPY' + + if (NSPEC_CRUST_MANTLE > 0) then + ! group name + write(group_name, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_create_group(group_name) + ! open group + call h5_open_group(group_name) + + ! create datasets + call h5_create_dataset_gen_in_group('muv_relaxed', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('muh_relaxed', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('kappav_relaxed', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('kappah_relaxed', (/NGLLX, NGLLY, NGLLZ, sum(offset_nspec_cm)/), 4, CUSTOM_REAL) + endif + + endif ! OUTPUT_RELAXED_MODEL + + ! close file + call h5_close_file() + endif ! myrank == 0 + + call synchronize_all() + + ! + ! write data from all ranks + ! + call h5_open_file_p_collect(file_name) + + ! scaling factors to re-dimensionalize units + scaleval1 = real( sqrt(PI*GRAV*RHOAV)*(R_PLANET/1000.0d0), kind=CUSTOM_REAL) ! velocities + + if (NSPEC_CRUST_MANTLE > 0) then + ! open group + write(group_name, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_open_group(group_name) + + ! uses temporary array + allocate(temp_store(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE), & + muv_shifted(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE), & + muh_shifted(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE), stat=ier) + if (ier /= 0) stop 'Error allocating temp_store array' + temp_store(:,:,:,:) = 0._CUSTOM_REAL + + ! safety check + if (ANISOTROPIC_3D_MANTLE_VAL) & + call exit_mpi(myrank,'ANISOTROPIC_3D_MANTLE not supported yet for shifted model file output') + + ! user output + if (myrank == 0) then + write(IMAIN,*) ' shifted model files in directory: ',trim(LOCAL_PATH) + endif + + ! user output + if (myrank == 0) write(IMAIN,*) ' crust/mantle:' + + ! moduli (muv,muh) are at relaxed values (only Qmu implemented), + ! scales back to have values at center frequency + muv_shifted(:,:,:,:) = muvstore_crust_mantle(:,:,:,:) + muh_shifted(:,:,:,:) = muhstore_crust_mantle(:,:,:,:) + do ispec = 1,NSPEC_CRUST_MANTLE + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + if (ATTENUATION_3D_VAL .or. ATTENUATION_1D_WITH_3D_STORAGE_VAL) then + scale_factor_r = factor_scale_relaxed_crust_mantle(i,j,k,ispec) + else + scale_factor_r = factor_scale_relaxed_crust_mantle(1,1,1,ispec) + endif + ! scaling back from relaxed to values at shifted frequency + ! (see in prepare_attenuation.f90 for how muv,muh are scaled to become relaxed moduli) + ! muv + muv_shifted(i,j,k,ispec) = muv_shifted(i,j,k,ispec) / scale_factor_r + ! muh + if (ispec_is_tiso_crust_mantle(ispec)) then + muh_shifted(i,j,k,ispec) = muh_shifted(i,j,k,ispec) / scale_factor_r + endif + enddo + enddo + enddo + enddo + + if (TRANSVERSE_ISOTROPY) then + ! vpv (at relaxed values) + temp_store(:,:,:,:) = sqrt((kappavstore_crust_mantle(:,:,:,:) & + + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & + * scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vpv_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + ! vph + temp_store(:,:,:,:) = sqrt((kappahstore_crust_mantle(:,:,:,:) & + + FOUR_THIRDS * muh_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & + * scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vph_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + ! vsv + temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vsv_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + ! vsh + temp_store(:,:,:,:) = sqrt( muh_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vsh_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + else ! isotropic + ! vp + temp_store(:,:,:,:) = sqrt((kappavstore_crust_mantle(:,:,:,:) & + + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_crust_mantle(:,:,:,:)) & + * scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + ! vs + temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_crust_mantle(:,:,:,:) )*scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + + endif + + deallocate(temp_store,muv_shifted,muh_shifted) + + ! close group + call h5_close_group() + endif ! NSPEC_CRUST_MANTLE > 0 + + if (NSPEC_INNER_CORE > 0) then + ! open group + write(group_name, "('reg',i1)") IREGION_INNER_CORE + call h5_open_group(group_name) + + ! uses temporary array + allocate(temp_store(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE), & + muv_shifted(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE), stat=ier) + if (ier /= 0) stop 'Error allocating temp_store array' + temp_store(:,:,:,:) = 0._CUSTOM_REAL + + ! user output + if (myrank == 0) write(IMAIN,*) ' inner core:' + + ! moduli (muv,muh) are at relaxed values, scale back to have shifted values at center frequency + muv_shifted(:,:,:,:) = muvstore_inner_core(:,:,:,:) + do ispec = 1,NSPEC_INNER_CORE + do k = 1,NGLLZ + do j = 1,NGLLY + do i = 1,NGLLX + if (ATTENUATION_3D_VAL .or. ATTENUATION_1D_WITH_3D_STORAGE_VAL) then + scale_factor_r = factor_scale_relaxed_inner_core(i,j,k,ispec) + else + scale_factor_r = factor_scale_relaxed_inner_core(1,1,1,ispec) + endif + + ! inverts to scale relaxed back to shifted factor + ! scaling back from relaxed to values at shifted frequency + ! (see in prepare_attenuation.f90 for how muv,muh are scaled to become relaxed moduli) + ! muv + muv_shifted(i,j,k,ispec) = muv_shifted(i,j,k,ispec) / scale_factor_r + enddo + enddo + enddo + enddo + + if (ANISOTROPIC_INNER_CORE_VAL) then + call exit_mpi(myrank,'ANISOTROPIC_INNER_CORE not supported yet for shifted model file output') + else + ! isotropic model + ! vp + temp_store(:,:,:,:) = sqrt((kappavstore_inner_core(:,:,:,:) & + + FOUR_THIRDS * muv_shifted(:,:,:,:))/rhostore_inner_core(:,:,:,:)) & + * scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vp_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), H5_COL) + ! vs + temp_store(:,:,:,:) = sqrt( muv_shifted(:,:,:,:)/rhostore_inner_core(:,:,:,:) )*scaleval1 + call h5_write_dataset_collect_hyperslab_in_group('vs_shifted', temp_store, & + (/0, 0, 0, sum(offset_nspec_ic(0:myrank-1))/), H5_COL) + endif + + deallocate(temp_store,muv_shifted) + + ! close group + call h5_close_group() + + endif ! NSPEC_INNER_CORE > 0 + + if (OUTPUT_RELAXED_MODEL) then + ! user output + if (myrank == 0) then + write(IMAIN,*) ' outputting relaxed model:' + call flush_IMAIN() + endif + ! checks + if (.not. TRANSVERSE_ISOTROPY) stop 'Outputting relaxed model requires TRANSVERSE_ISOTROPY' + + ! scaling factor to re-dimensionalize units + ! the scale of GPa--[g/cm^3][(km/s)^2] + scaleval1 = real( ((sqrt(PI*GRAV*RHOAV)*R_PLANET/1000.d0)**2)*(RHOAV/1000.d0), kind=CUSTOM_REAL) ! moduli GPa + + if (NSPEC_CRUST_MANTLE > 0) then + ! open group + write(group_name, "('reg',i1)") IREGION_CRUST_MANTLE + call h5_open_group(group_name) + + ! muv_relaxed + call h5_write_dataset_collect_hyperslab_in_group('muv_relaxed', muvstore_crust_mantle*scaleval1, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + ! muh_relaxed + call h5_write_dataset_collect_hyperslab_in_group('muh_relaxed', muhstore_crust_mantle*scaleval1, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + ! kappav_relaxed + call h5_write_dataset_collect_hyperslab_in_group('kappav_relaxed', kappavstore_crust_mantle*scaleval1, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + ! kappah_relaxed + call h5_write_dataset_collect_hyperslab_in_group('kappah_relaxed', kappahstore_crust_mantle*scaleval1, & + (/0, 0, 0, sum(offset_nspec_cm(0:myrank-1))/), H5_COL) + + ! close group + call h5_close_group() + + endif ! NSPEC_CRUST_MANTLE > 0 + + endif ! OUTPUT_RELAXED_MODEL + + ! close file + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine save_forward_model_at_shifted_frequency_hdf5 diff --git a/src/specfem3D/save_kernels.F90 b/src/specfem3D/save_kernels.F90 index 7295e8d47..9017ec8c7 100644 --- a/src/specfem3D/save_kernels.F90 +++ b/src/specfem3D/save_kernels.F90 @@ -1298,6 +1298,13 @@ subroutine save_kernels_crust_mantle_ani() bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle, & bulk_betav_kl_crust_mantle,bulk_betah_kl_crust_mantle, & Gc_prime_kl_crust_mantle, Gs_prime_kl_crust_mantle) + else if (HDF5_ENABLED) then + call write_kernels_cm_ani_hdf5(alphav_kl_crust_mantle,alphah_kl_crust_mantle, & + betav_kl_crust_mantle,betah_kl_crust_mantle, & + eta_kl_crust_mantle, & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle, & + bulk_betav_kl_crust_mantle,bulk_betah_kl_crust_mantle, & + Gc_prime_kl_crust_mantle, Gs_prime_kl_crust_mantle) else ! binary file output call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH) @@ -1647,6 +1654,9 @@ subroutine save_kernels_crust_mantle_iso() if (ADIOS_FOR_KERNELS) then call write_kernels_cm_iso_adios(mu_kl_crust_mantle, kappa_kl_crust_mantle, rhonotprime_kl_crust_mantle, & bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle) + else if (HDF5_ENABLED) then + call write_kernels_cm_iso_hdf5(mu_kl_crust_mantle, kappa_kl_crust_mantle, rhonotprime_kl_crust_mantle, & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle) else call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH) @@ -1742,6 +1752,8 @@ subroutine save_kernels_outer_core(rhostore_outer_core,kappavstore_outer_core,rh ! writes out kernels to file if (ADIOS_FOR_KERNELS) then call write_kernels_oc_adios() + else if (HDF5_ENABLED) then + call write_kernels_oc_hdf5() else call create_name_database(prname,myrank,IREGION_OUTER_CORE,LOCAL_TMP_PATH) @@ -1751,7 +1763,6 @@ subroutine save_kernels_outer_core(rhostore_outer_core,kappavstore_outer_core,rh open(unit=IOUT,file=trim(prname)//'alpha_kernel.bin',status='unknown',form='unformatted',action='write') write(IOUT) alpha_kl_outer_core close(IOUT) - endif end subroutine save_kernels_outer_core @@ -1814,6 +1825,8 @@ subroutine save_kernels_inner_core(rhostore_inner_core,muvstore_inner_core,kappa ! writes out kernels to file if (ADIOS_FOR_KERNELS) then call write_kernels_ic_adios() + else if (HDF5_ENABLED) then + call write_kernels_ic_hdf5() else call create_name_database(prname,myrank,IREGION_INNER_CORE,LOCAL_TMP_PATH) @@ -1862,6 +1875,8 @@ subroutine save_kernels_boundary_kl() ! writes out kernels to file if (ADIOS_FOR_KERNELS) then call write_kernels_boundary_kl_adios() + else if (HDF5_ENABLED) then + call write_kernels_boundary_kl_hdf5() else call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH) @@ -1944,6 +1959,9 @@ subroutine save_kernels_source_derivatives() ! writes out kernels to file if (ADIOS_FOR_KERNELS) then call write_kernels_source_derivatives_adios() + !else if (HDF5_ENABLED) then + ! ! TODO ADD HDF5 + ! call write_kernels_source_derivatives_hdf5() else ! kernel file output do irec_local = 1, nrec_local @@ -2001,6 +2019,8 @@ subroutine save_kernels_Hessian() ! writes out kernels to file if (ADIOS_FOR_KERNELS) then call write_kernels_Hessian_adios() + else if (HDF5_ENABLED) then + call write_kernels_Hessian_hdf5() else ! stores into file call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_TMP_PATH) diff --git a/src/specfem3D/save_kernels_hdf5.F90 b/src/specfem3D/save_kernels_hdf5.F90 new file mode 100644 index 000000000..8248ba1ac --- /dev/null +++ b/src/specfem3D/save_kernels_hdf5.F90 @@ -0,0 +1,714 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +subroutine write_kernels_strength_noise_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_noise + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_adj + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: info, comm + + ! gather the number of elements in each region + call gather_all_all_singlei(size(sigma_kl_crust_mantle,4), offset_nspec_cm_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + call h5_create_dataset_gen('sigma_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + call h5_write_dataset_collect_hyperslab('sigma_kernel', sigma_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_strength_noise_hdf5 + + +subroutine write_kernels_cm_ani_hdf5(alphav_kl_crust_mantle,alphah_kl_crust_mantle, & + betav_kl_crust_mantle,betah_kl_crust_mantle, & + eta_kl_crust_mantle, & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle, & + bulk_betav_kl_crust_mantle,bulk_betah_kl_crust_mantle, & + Gc_prime_kl_crust_mantle, Gs_prime_kl_crust_mantle) + + + use specfem_par + use specfem_par_crustmantle + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + ! input Parameters + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: & + alphav_kl_crust_mantle,alphah_kl_crust_mantle, & + betav_kl_crust_mantle,betah_kl_crust_mantle, & + eta_kl_crust_mantle + + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle, & + bulk_betav_kl_crust_mantle,bulk_betah_kl_crust_mantle + + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: & + Gc_prime_kl_crust_mantle, Gs_prime_kl_crust_mantle + +#ifdef USE_HDF5 + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: info, comm + + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_adj + + ! check if anything to do + if (.not. ANISOTROPIC_KL) return + + ! gather the number of elements in each region + !call gather_all_all_singlei(size(alphav_kl_crust_mantle,4), offset_nspec_cm_adj, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_CRUST_MANTLE_ADJOINT, offset_nspec_cm_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! create or open file + call h5_create_or_open_file(file_name) + ! create dataset + if (SAVE_TRANSVERSE_KL_ONLY) then + call h5_create_dataset_gen('alphav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('alphah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('betav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('betah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('eta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('bulk_c_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_betav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_betah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('alpha_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + else if (SAVE_AZIMUTHAL_ANISO_KL_ONLY) then + call h5_create_dataset_gen('alphav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('alphah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('betav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('betah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('bulk_c_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_betav_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_betah_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('eta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('Gc_prime_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Gs_prime_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + ! check isotropic kernel + if (.false.) then + call h5_create_dataset_gen('alpha_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + endif + ! check anisotropic kernels + if (.false.) then + call h5_create_dataset_gen('A_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('C_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('L_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('N_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('F_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Gc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Gs_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Jc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Kc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Mc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Bc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Hc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Ec_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('Dc_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + endif + + else + ! fully anisotropic kernels + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('cijkl_kernel', (/21,NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 5, CUSTOM_REAL) + + endif + + ! close file + call h5_close_file() + + endif ! myrank == 0 + + ! synchronize all + call synchronize_all() + + ! write data from all ranks + call h5_open_file_p_collect(file_name) + + ! write data + if (SAVE_TRANSVERSE_KL_ONLY) then + call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + else if (SAVE_AZIMUTHAL_ANISO_KL_ONLY) then + call h5_write_dataset_collect_hyperslab('alphav_kernel', alphav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alphah_kernel', alphah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betav_kernel', betav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('betah_kernel', betah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betav_kernel', bulk_betav_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_betah_kernel', bulk_betah_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('eta_kernel', eta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('Gc_prime_kernel', Gc_prime_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('Gs_prime_kernel', Gs_prime_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + ! check isotropic kernel + if (.false.) then + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + endif + + ! check anisotropic kernels + !if (.false.) then + ! call h5_write_dataset_collect_hyperslab('A_kernel', A_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('C_kernel', C_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('L_kernel', L_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('N_kernel', N_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('F_kernel', F_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Gc_kernel', Gc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Gs_kernel', Gs_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Jc_kernel', Jc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Kc_kernel', Kc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Mc_kernel', Mc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Bc_kernel', Bc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Hc_kernel', Hc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Ec_kernel', Ec_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + ! call h5_write_dataset_collect_hyperslab('Dc_kernel', Dc_kl_crust_mantle, (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + !endif + + else + + ! fully anisotropic kernels + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('cijkl_kernel', cijkl_kl_crust_mantle, & + (/0,0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + endif + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_cm_ani_hdf5 + + +subroutine write_kernels_cm_iso_hdf5(mu_kl_crust_mantle, kappa_kl_crust_mantle, rhonotprime_kl_crust_mantle, & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle) + + use specfem_par + use specfem_par_crustmantle + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + + ! Parameters + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_ADJOINT) :: & + mu_kl_crust_mantle, kappa_kl_crust_mantle, rhonotprime_kl_crust_mantle, & + bulk_c_kl_crust_mantle,bulk_beta_kl_crust_mantle + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_adj + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: info, comm + + ! checks if anything to do + if (ANISOTROPIC_KL) return + + ! gather the number of elements in each region + call gather_all_all_singlei(NSPEC_CRUST_MANTLE_ADJOINT, offset_nspec_cm_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + call h5_create_dataset_gen('rhonotprime_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('kappa_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('mu_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('alpha_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + call h5_create_dataset_gen('bulk_c_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('bulk_beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + call h5_write_dataset_collect_hyperslab('rhonotprime_kernel', rhonotprime_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('kappa_kernel', kappa_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('mu_kernel', mu_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + call h5_write_dataset_collect_hyperslab('bulk_c_kernel', bulk_c_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('bulk_beta_kernel', bulk_beta_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_cm_iso_hdf5 + + +subroutine write_kernels_oc_hdf5() + + use specfem_par + use specfem_par_outercore + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_oc_adj + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: info, comm + + ! gather the number of elements in each region + call gather_all_all_singlei(NSPEC_OUTER_CORE_ADJOINT, offset_nspec_oc_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_oc_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('alpha_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_oc_adj)/), 4, CUSTOM_REAL) + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_outer_core, & + (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_outer_core, & + (/0,0,0,sum(offset_nspec_oc_adj(0:myrank-1))/), H5_COL) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_oc_hdf5 + + +subroutine write_kernels_ic_hdf5() + + use specfem_par + use specfem_par_innercore + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_ic_adj + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: info, comm + + ! gather the number of elements in each region + call gather_all_all_singlei(NSPEC_INNER_CORE_ADJOINT, offset_nspec_ic_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + call h5_create_dataset_gen('rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_ic_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('alpha_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_ic_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('beta_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_ic_adj)/), 4, CUSTOM_REAL) + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + call h5_write_dataset_collect_hyperslab('rho_kernel', rho_kl_inner_core, & + (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('alpha_kernel', alpha_kl_inner_core, & + (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('beta_kernel', beta_kl_inner_core, & + (/0,0,0,sum(offset_nspec_ic_adj(0:myrank-1))/), H5_COL) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_ic_hdf5 + + +subroutine write_kernels_boundary_kl_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_innercore + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2d_moho + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2d_400 + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2d_670 + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2d_cmb + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec2d_icb + + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: info, comm + + if (.not. SAVE_KERNELS_BOUNDARY) return + + ! gather the number of elements in each region + call gather_all_all_singlei(NSPEC2D_MOHO, offset_nspec2d_moho, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC2D_400, offset_nspec2d_400, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC2D_670, offset_nspec2d_670, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC2D_CMB, offset_nspec2d_cmb, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC2D_ICB, offset_nspec2d_icb, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + if (.not. SUPPRESS_CRUSTAL_MESH .and. HONOR_1D_SPHERICAL_MOHO) then + call h5_create_dataset_gen('moho_kernel', (/0,0,sum(offset_nspec2d_moho(0:myrank-1))/), 3, CUSTOM_REAL) + endif + call h5_create_dataset_gen('d400_kernel', (/0,0,sum(offset_nspec2d_400(0:myrank-1))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen('d670_kernel', (/0,0,sum(offset_nspec2d_670(0:myrank-1))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen('CMB_kernel', (/0,0,sum(offset_nspec2d_cmb(0:myrank-1))/), 3, CUSTOM_REAL) + call h5_create_dataset_gen('ICB_kernel', (/0,0,sum(offset_nspec2d_icb(0:myrank-1))/), 3, CUSTOM_REAL) + + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + if (.not. SUPPRESS_CRUSTAL_MESH .and. HONOR_1D_SPHERICAL_MOHO) then + call h5_write_dataset_collect_hyperslab('moho_kernel', moho_kl, (/0,0,sum(offset_nspec2d_moho(0:myrank-1))/), H5_COL) + endif + call h5_write_dataset_collect_hyperslab('d400_kernel', d400_kl, (/0,0,sum(offset_nspec2d_400(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('d670_kernel', d670_kl, (/0,0,sum(offset_nspec2d_670(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('CMB_kernel', CMB_kl, (/0,0,sum(offset_nspec2d_CMB(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('ICB_kernel', ICB_kl, (/0,0,sum(offset_nspec2d_ICB(0:myrank-1))/), H5_COL) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_boundary_kl_hdf5 + + +subroutine write_kernels_Hessian_hdf5() + + use specfem_par + use specfem_par_crustmantle + +#ifdef USE_HDF5 + use manager_hdf5 +#endif + + implicit none + +#ifdef USE_HDF5 + ! offset array + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_cm_adj + + ! local parameters + character(len=MAX_STRING_LEN) :: file_name + integer :: info, comm + + ! gather the number of elements in each region + call gather_all_all_singlei(NSPEC_CRUST_MANTLE_ADJOINT, offset_nspec_cm_adj, NPROCTOT_VAL) + + ! initialize hdf5 + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() ! called in initialize_mesher() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = LOCAL_TMP_PATH(1:len_trim(LOCAL_TMP_PATH))//'/kernels.h5' + + if (myrank == 0) then + ! check if file exists + call h5_create_or_open_file(file_name) + ! create dataset + call h5_create_dataset_gen('hess_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('hess_rho_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('hess_kappa_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + call h5_create_dataset_gen('hess_mu_kernel', (/NGLLX,NGLLY,NGLLZ,sum(offset_nspec_cm_adj)/), 4, CUSTOM_REAL) + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! open hdf5 + call h5_open_file_p_collect(file_name) + + ! write data + call h5_write_dataset_collect_hyperslab('hess_kernel', hess_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('hess_rho_kernel', hess_rho_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('hess_kappa_kernel', hess_kappa_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab('hess_mu_kernel', hess_mu_kl_crust_mantle, & + (/0,0,0,sum(offset_nspec_cm_adj(0:myrank-1))/), H5_COL) + + ! close hdf5 + call h5_close_file_p() + +#else + + print *,'Error: HDF5 not enabled in this version of the code' + print *, 'Please recompile with the HDF5 option enabled with the configure flag --with-hdf5' + call exit_mpi(myrank,'Error: HDF5 not enabled in this version of the code') + +#endif + +end subroutine write_kernels_Hessian_hdf5 diff --git a/src/specfem3D/save_regular_kernels.f90 b/src/specfem3D/save_regular_kernels.f90 index d88b4039e..e37981260 100644 --- a/src/specfem3D/save_regular_kernels.f90 +++ b/src/specfem3D/save_regular_kernels.f90 @@ -390,6 +390,9 @@ subroutine save_regular_kernels_cm() if (ADIOS_FOR_KERNELS) then ! check implementation call exit_mpi(myrank,'saving regular kernels in ADIOS file format is not supported yet') + else if (HDF5_ENABLED) then + ! TODO ADD HDF5 + call exit_mpi(myrank,'saving regular kernels in HDF5 file format is not supported yet') else ! sets up database name call create_name_database(prname,myrank,IREGION_CRUST_MANTLE,LOCAL_PATH) diff --git a/src/specfem3D/setup_sources_receivers.f90 b/src/specfem3D/setup_sources_receivers.f90 index eff03f7c3..9fe5b0b5e 100644 --- a/src/specfem3D/setup_sources_receivers.f90 +++ b/src/specfem3D/setup_sources_receivers.f90 @@ -1895,6 +1895,8 @@ subroutine setup_receivers_precompute_intp() endif endif + ! TODO: add HDF5 if any + end subroutine setup_receivers_precompute_intp ! diff --git a/src/specfem3D/specfem3D_par.F90 b/src/specfem3D/specfem3D_par.F90 index 77cadf84c..762bcb75d 100644 --- a/src/specfem3D/specfem3D_par.F90 +++ b/src/specfem3D/specfem3D_par.F90 @@ -403,6 +403,9 @@ module specfem_par ! process/partition name character(len=MAX_STRING_LEN) :: prname + ! hdf5 file name + character(len=MAX_STRING_LEN) :: hdf5_seismo_fname + !----------------------------------------------------------------- ! MPI partitions !----------------------------------------------------------------- @@ -1521,4 +1524,88 @@ end module my_libxsmm #endif +!===================================================================== + +#ifdef USE_HDF5 + + module specfem_par_movie_hdf5 + + use constants_solver, only: NPROCTOT_VAL,MAX_STRING_LEN,CUSTOM_REAL + use manager_hdf5 + + + integer :: info, comm + character(len=MAX_STRING_LEN) :: file_name, group_name + + ! surface movie + integer :: npoints_surf_mov_all_proc + integer, dimension(:), allocatable :: offset_poin + + ! volume movie + ! output parameters + logical, parameter :: MOVIE_OUTPUT_DIV = .true. ! divergence + logical, parameter :: MOVIE_OUTPUT_CURL = .true. ! curl + logical, parameter :: MOVIE_OUTPUT_CURLNORM = .true. ! Frobenius norm of curl + logical, parameter :: OUTPUT_CRUST_MANTLE = .true. + logical, parameter :: OUTPUT_OUTER_CORE = .true. + logical, parameter :: OUTPUT_INNER_CORE = .true. + + ! flags for check which region is output for movie + logical :: output_sv = .false. ! strain or vector output + logical :: output_cm = .false. + logical :: output_oc = .false. + logical :: output_ic = .false. + + integer :: npoints_vol_mov_all_proc + integer :: npoints_vol_mov_all_proc_cm + integer :: npoints_vol_mov_all_proc_oc + integer :: npoints_vol_mov_all_proc_ic + + integer :: nspec_vol_mov_all_proc + integer :: nspec_vol_mov_all_proc_cm + integer :: nspec_vol_mov_all_proc_oc + integer :: nspec_vol_mov_all_proc_ic + + ! number of elements for visualization (nspec * (NGLLX-1) * (NGLLY-1) * (NGLLZ-1)) + integer :: nspec_vol_mov_all_proc_cm_conn, nspec_vol_mov_all_proc_oc_conn, nspec_vol_mov_all_proc_ic_conn + integer, dimension(:), allocatable :: offset_poin_vol, offset_nspec_vol + integer, dimension(:), allocatable :: offset_poin_vol_oc, offset_nspec_vol_oc + integer, dimension(:), allocatable :: offset_poin_vol_ic, offset_nspec_vol_ic + integer, dimension(:), allocatable :: offset_poin_vol_cm, offset_nspec_vol_cm + + ! xdmf + integer :: xdmf_surf = 30000 + integer :: xdmf_vol = 30001 + integer :: surf_xdmf_pos = 0 + integer :: vol_xdmf_pos = 0 + +contains + + !------------------------------------------- + + function r2c(k) result(str) + + ! "Convert an real to string." + + implicit none + real(kind=CUSTOM_REAL), intent(in) :: k + character(len=20) str + write (str, *) k + str = adjustl(str) + end function r2c + + !function i2c(k) result(str) + !! "Convert an integer to string." + ! implicit none + ! integer, intent(in) :: k + ! character(len=20) str + ! write (str, "(i20)") k + ! str = adjustl(str) + !end function i2c + +! + + end module specfem_par_movie_hdf5 + +#endif diff --git a/src/specfem3D/write_movie_output.f90 b/src/specfem3D/write_movie_output.f90 index 6d2eb005b..e712dca9a 100644 --- a/src/specfem3D/write_movie_output.f90 +++ b/src/specfem3D/write_movie_output.f90 @@ -30,7 +30,8 @@ subroutine write_movie_output() use specfem_par, only: deltat,it,myrank,Mesh_pointer, & GPU_MODE,NTSTEP_BETWEEN_FRAMES, & MOVIE_START,MOVIE_STOP,MOVIE_SURFACE,MOVIE_VOLUME,MOVIE_VOLUME_TYPE, & - scale_displ,scale_veloc + scale_displ,scale_veloc, & + HDF5_ENABLED use specfem_par_crustmantle, only: displ_crust_mantle,veloc_crust_mantle,accel_crust_mantle, & eps_trace_over_3_crust_mantle,epsilondev_xx_crust_mantle,epsilondev_xy_crust_mantle,epsilondev_xz_crust_mantle, & @@ -75,8 +76,13 @@ subroutine write_movie_output() endif endif + ! TODO ADD IO_SERVER ! save velocity here to avoid static offset on displacement for movies - call write_movie_surface() + if (HDF5_ENABLED) then + call write_movie_surface_hdf5() + else + call write_movie_surface() + endif ! executes an external script on the node if (RUN_EXTERNAL_MOVIE_SCRIPT) then @@ -109,14 +115,14 @@ subroutine write_movie_output() ! integrates strain call movie_volume_integrate_strain(deltat,NSPEC_CRUST_MANTLE_3DMOVIE, & - eps_trace_over_3_crust_mantle, & - epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle, & - epsilondev_xy_crust_mantle,epsilondev_xz_crust_mantle, & - epsilondev_yz_crust_mantle, & - Ieps_trace_over_3_crust_mantle, & - Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle, & - Iepsilondev_xy_crust_mantle,Iepsilondev_xz_crust_mantle, & - Iepsilondev_yz_crust_mantle) + eps_trace_over_3_crust_mantle, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle, & + epsilondev_xy_crust_mantle,epsilondev_xz_crust_mantle, & + epsilondev_yz_crust_mantle, & + Ieps_trace_over_3_crust_mantle, & + Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle, & + Iepsilondev_xy_crust_mantle,Iepsilondev_xz_crust_mantle, & + Iepsilondev_yz_crust_mantle) endif ! file output @@ -134,19 +140,38 @@ subroutine write_movie_output() epsilondev_xy_crust_mantle,epsilondev_xz_crust_mantle, & epsilondev_yz_crust_mantle) endif - call write_movie_volume_strains(NSPEC_CRUST_MANTLE_STRAIN_ONLY, & - eps_trace_over_3_crust_mantle, & - NSPEC_CRUST_MANTLE_STR_OR_ATT, & - epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & - epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle) + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_strains_hdf5(NSPEC_CRUST_MANTLE_STRAIN_ONLY, & + eps_trace_over_3_crust_mantle, & + NSPEC_CRUST_MANTLE_STR_OR_ATT, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle) + + else + call write_movie_volume_strains(NSPEC_CRUST_MANTLE_STRAIN_ONLY, & + eps_trace_over_3_crust_mantle, & + NSPEC_CRUST_MANTLE_STR_OR_ATT, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle) + endif case (2, 3) + ! TODO ADD IO_SERVER ! output the Time Integral of Strain, or \mu*TIS - call write_movie_volume_strains(NSPEC_CRUST_MANTLE_3DMOVIE, & - Ieps_trace_over_3_crust_mantle, & - NSPEC_CRUST_MANTLE_3DMOVIE, & - Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle,Iepsilondev_xy_crust_mantle, & - Iepsilondev_xz_crust_mantle,Iepsilondev_yz_crust_mantle) + if (HDF5_ENABLED) then + call write_movie_volume_strains_hdf5(NSPEC_CRUST_MANTLE_3DMOVIE, & + Ieps_trace_over_3_crust_mantle, & + NSPEC_CRUST_MANTLE_3DMOVIE, & + Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle,Iepsilondev_xy_crust_mantle, & + Iepsilondev_xz_crust_mantle,Iepsilondev_yz_crust_mantle) + else + call write_movie_volume_strains(NSPEC_CRUST_MANTLE_3DMOVIE, & + Ieps_trace_over_3_crust_mantle, & + NSPEC_CRUST_MANTLE_3DMOVIE, & + Iepsilondev_xx_crust_mantle,Iepsilondev_yy_crust_mantle,Iepsilondev_xy_crust_mantle, & + Iepsilondev_xz_crust_mantle,Iepsilondev_yz_crust_mantle) + endif case (4) ! output divergence and curl in whole volume @@ -167,38 +192,66 @@ subroutine write_movie_output() call transfer_fields_oc_from_device(NGLOB_OUTER_CORE, & displ_outer_core,veloc_outer_core,accel_outer_core,Mesh_pointer) endif - call write_movie_volume_divcurl(NSPEC_CRUST_MANTLE_STRAIN_ONLY,eps_trace_over_3_crust_mantle, & - div_displ_outer_core, & - accel_outer_core,kappavstore_outer_core,rhostore_outer_core,ibool_outer_core, & - NSPEC_INNER_CORE_STRAIN_ONLY,eps_trace_over_3_inner_core, & - NSPEC_CRUST_MANTLE_STR_OR_ATT, & - epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & - epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, & - NSPEC_INNER_CORE_STR_OR_ATT, & - epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, & - epsilondev_xz_inner_core,epsilondev_yz_inner_core) - + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_divcurl_hdf5(NSPEC_CRUST_MANTLE_STRAIN_ONLY,eps_trace_over_3_crust_mantle, & + div_displ_outer_core, & + accel_outer_core,kappavstore_outer_core,rhostore_outer_core,ibool_outer_core, & + NSPEC_INNER_CORE_STRAIN_ONLY, eps_trace_over_3_inner_core, & + NSPEC_CRUST_MANTLE_STR_OR_ATT, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, & + NSPEC_INNER_CORE_STR_OR_ATT, & + epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, & + epsilondev_xz_inner_core,epsilondev_yz_inner_core) + else + call write_movie_volume_divcurl(NSPEC_CRUST_MANTLE_STRAIN_ONLY,eps_trace_over_3_crust_mantle, & + div_displ_outer_core, & + accel_outer_core,kappavstore_outer_core,rhostore_outer_core,ibool_outer_core, & + NSPEC_INNER_CORE_STRAIN_ONLY,eps_trace_over_3_inner_core, & + NSPEC_CRUST_MANTLE_STR_OR_ATT, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, & + NSPEC_INNER_CORE_STR_OR_ATT, & + epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, & + epsilondev_xz_inner_core,epsilondev_yz_inner_core) + endif case (5) ! output displacement if (GPU_MODE) then call transfer_displ_cm_from_device(NDIM*NGLOB_CRUST_MANTLE,displ_crust_mantle,Mesh_pointer) endif scalingval = scale_displ - call write_movie_volume_vector(npoints_3dmovie, & - ibool_crust_mantle, & - displ_crust_mantle, & - scalingval,mask_3dmovie,nu_3dmovie) - + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_vector_hdf5(npoints_3dmovie, & + ibool_crust_mantle, & + displ_crust_mantle, & + scalingval,mask_3dmovie,nu_3dmovie) + else + call write_movie_volume_vector(npoints_3dmovie, & + ibool_crust_mantle, & + displ_crust_mantle, & + scalingval,mask_3dmovie,nu_3dmovie) + endif case (6) ! output velocity if (GPU_MODE) then call transfer_veloc_cm_from_device(NDIM*NGLOB_CRUST_MANTLE,veloc_crust_mantle,Mesh_pointer) endif scalingval = scale_veloc - call write_movie_volume_vector(npoints_3dmovie, & - ibool_crust_mantle, & - veloc_crust_mantle, & - scalingval,mask_3dmovie,nu_3dmovie) + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_vector_hdf5(npoints_3dmovie, & + ibool_crust_mantle, & + veloc_crust_mantle, & + scalingval,mask_3dmovie,nu_3dmovie) + else + call write_movie_volume_vector(npoints_3dmovie, & + ibool_crust_mantle, & + veloc_crust_mantle, & + scalingval,mask_3dmovie,nu_3dmovie) + endif case (7) ! output norm of displacement @@ -209,8 +262,14 @@ subroutine write_movie_output() call transfer_displ_ic_from_device(NDIM*NGLOB_INNER_CORE,displ_inner_core,Mesh_pointer) call transfer_displ_oc_from_device(NGLOB_OUTER_CORE,displ_outer_core,Mesh_pointer) endif - call write_movie_volume_displnorm(displ_crust_mantle,displ_inner_core,displ_outer_core, & - ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core,displ_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + else + call write_movie_volume_displnorm(displ_crust_mantle,displ_inner_core,displ_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + endif case (8) ! output norm of velocity @@ -221,8 +280,14 @@ subroutine write_movie_output() call transfer_veloc_ic_from_device(NDIM*NGLOB_INNER_CORE,veloc_inner_core,Mesh_pointer) call transfer_veloc_oc_from_device(NGLOB_OUTER_CORE,veloc_outer_core,Mesh_pointer) endif - call write_movie_volume_velnorm(veloc_crust_mantle,veloc_inner_core,veloc_outer_core, & - ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,veloc_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + else + call write_movie_volume_velnorm(veloc_crust_mantle,veloc_inner_core,veloc_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + endif case (9) ! output norm of acceleration @@ -233,8 +298,14 @@ subroutine write_movie_output() call transfer_accel_ic_from_device(NDIM*NGLOB_INNER_CORE,accel_inner_core,Mesh_pointer) call transfer_accel_oc_from_device(NGLOB_OUTER_CORE,accel_outer_core,Mesh_pointer) endif - call write_movie_volume_accelnorm(accel_crust_mantle,accel_inner_core,accel_outer_core, & - ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + ! TODO ADD IO_SERVER + if (HDF5_ENABLED) then + call write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core,accel_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + else + call write_movie_volume_accelnorm(accel_crust_mantle,accel_inner_core,accel_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + endif case default call exit_MPI(myrank, 'MOVIE_VOLUME_TYPE has to be in range from 1 to 9') diff --git a/src/specfem3D/write_movie_surface.f90 b/src/specfem3D/write_movie_surface.f90 index 8e70ca87c..29b3c24ea 100644 --- a/src/specfem3D/write_movie_surface.f90 +++ b/src/specfem3D/write_movie_surface.f90 @@ -37,6 +37,8 @@ subroutine movie_surface_count_points() ! local parameters integer :: ispec2D,i,j,npoin + if (HDF5_ENABLED) return + ! gets number of points on surface mesh npoin = 0 do ispec2D = 1, NSPEC_TOP ! NSPEC2D_TOP(IREGION_CRUST_MANTLE) diff --git a/src/specfem3D/write_movie_surface_hdf5.F90 b/src/specfem3D/write_movie_surface_hdf5.F90 new file mode 100644 index 000000000..28074e83a --- /dev/null +++ b/src/specfem3D/write_movie_surface_hdf5.F90 @@ -0,0 +1,598 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + +subroutine movie_surface_init_hdf5() +#ifdef USE_HDF5 + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + integer :: ier + + allocate(offset_poin(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_poin array') + + npoints_surf_mov_all_proc = 0 + +#else + + write(*,*) 'Error: HDF5 is not enabled in this version of Specfem3D_Globe.' + write(*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop +#endif +end subroutine movie_surface_init_hdf5 + + +subroutine movie_surface_finalize_hdf5() +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 + + implicit none + + deallocate(offset_poin) + +#else + + write(*,*) 'Error: HDF5 is not enabled in this version of Specfem3D_Globe.' + write(*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop +#endif +end subroutine movie_surface_finalize_hdf5 + +subroutine write_movie_surface_mesh_hdf5() + + use specfem_par + use specfem_par_crustmantle + use specfem_par_movie + +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 + + implicit none + + ! local parameters + real(kind=CUSTOM_REAL), dimension(:), allocatable :: store_val_x,store_val_y,store_val_z + integer :: ipoin,ispec2D,ispec,i,j,k,ier,iglob1,iglob2,iglob3,iglob4,npoin + real(kind=CUSTOM_REAL) :: rval,thetaval,phival,xval,yval,zval + + call movie_surface_init_hdf5() + + ! gather npoints on each process + call gather_all_all_singlei(nmovie_points,offset_poin,NPROCTOT_VAL) + ! total number of points on all processes + npoints_surf_mov_all_proc = sum(offset_poin) + + ! allocates movie surface arrays + allocate(store_val_x(nmovie_points), & + store_val_y(nmovie_points), & + store_val_z(nmovie_points),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating movie surface location arrays') + + ! initialize h5 file for surface movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create file and dataset + file_name = trim(OUTPUT_FILES)//"/movie_surface.h5" + + if (myrank == 0) then + call h5_create_file(file_name) + ! create group surf_coord + call h5_create_group("surf_coord") + call h5_open_group("surf_coord") + ! create datasets x, y, z + call h5_create_dataset_gen_in_group("x", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("y", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("z", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + + ! close group surf_coord + call h5_close_group() + + ! close file + call h5_close_file() + endif + + ! gets coordinates of surface mesh + ipoin = 0 + do ispec2D = 1, NSPEC_TOP ! NSPEC2D_TOP(IREGION_CRUST_MANTLE) + ispec = ibelm_top_crust_mantle(ispec2D) + ! in case of global, NCHUNKS_VAL == 6 simulations, be aware that for + ! the cubed sphere, the mapping changes for different chunks, + ! i.e. e.g. x(1,1) and x(5,5) flip left and right sides of the elements in geographical coordinates. + ! for future consideration, like in create_movie_GMT_global.f90 ... + k = NGLLZ + ! loop on all the points inside the element + if (.not. MOVIE_COARSE) then + do j = 1, NGLLY-1, 1 + do i = 1, NGLLX-1, 1 + ! stores values + iglob1 = ibool_crust_mantle(i,j,k,ispec) + iglob2 = ibool_crust_mantle(i+1,j,k,ispec) + iglob3 = ibool_crust_mantle(i+1,j+1,k,ispec) + iglob4 = ibool_crust_mantle(i,j+1,k,ispec) + ! iglob1 + ipoin = ipoin + 1 + rval = rstore_crust_mantle(1,iglob1) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob1) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob1) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval + ! iglob2 + ipoin = ipoin + 1 + rval = rstore_crust_mantle(1,iglob2) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob2) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob2) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval + ! iglob3 + ipoin = ipoin + 1 + rval = rstore_crust_mantle(1,iglob3) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob3) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob3) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval + ! iglob4 + ipoin = ipoin + 1 + rval = rstore_crust_mantle(1,iglob4) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob4) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob4) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval + enddo + enddo + else ! MOVIE_COARSE + iglob1 = ibool_crust_mantle(1,1,k,ispec) + iglob2 = ibool_crust_mantle(NGLLX,1,k,ispec) + iglob3 = ibool_crust_mantle(NGLLX,NGLLY,k,ispec) + iglob4 = ibool_crust_mantle(1,NGLLY,k,ispec) + + ipoin = ipoin + 1 + rval = rstore_crust_mantle(1,iglob1) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob1) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob1) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval + + ipoin = ipoin + 1 + rval = rstore_crust_mantle(1,iglob2) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob2) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob2) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval + + ipoin = ipoin + 1 + rval = rstore_crust_mantle(1,iglob3) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob3) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob3) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval + + ipoin = ipoin + 1 + rval = rstore_crust_mantle(1,iglob4) ! radius r (normalized) + thetaval = rstore_crust_mantle(2,iglob4) ! colatitude theta (in radian) + phival = rstore_crust_mantle(3,iglob4) ! longitude phi (in radian) + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + store_val_x(ipoin) = xval + store_val_y(ipoin) = yval + store_val_z(ipoin) = zval + + endif + enddo + npoin = ipoin + if (npoin /= nmovie_points ) call exit_mpi(myrank,'Error number of movie points not equal to nmovie_points') + + call synchronize_all() + + ! write data to h5 file + call h5_open_file_p_collect(file_name) + call h5_open_group("surf_coord") + + ! write x, y, z + call h5_write_dataset_collect_hyperslab_in_group("x", store_val_x, (/sum(offset_poin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("y", store_val_y, (/sum(offset_poin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("z", store_val_z, (/sum(offset_poin(0:myrank-1))/), H5_COL) + + ! close group and file + call h5_close_group() + call h5_close_file_p() + + deallocate(store_val_x,store_val_y,store_val_z) + + if (myrank == 0) then + ! write xdmf header + call write_xdmf_surface_header(npoints_surf_mov_all_proc) + endif + +#else + + write(*,*) 'Error: HDF5 is not enabled in this version of Specfem3D_Globe.' + write(*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + +end subroutine write_movie_surface_mesh_hdf5 + + +subroutine write_movie_surface_hdf5() + +#ifdef USE_HDF5 + use specfem_par + use specfem_par_crustmantle + use specfem_par_movie + use specfem_par_movie_hdf5 + + implicit none + + ! local parameters + integer :: ipoin,ispec2D,ispec,i,j,k,iglob1,iglob2,iglob3,iglob4 + + ! by default: save velocity here to avoid static offset on displacement for movies + + ! gets coordinates of surface mesh and surface displacement + ipoin = 0 + do ispec2D = 1, NSPEC_TOP ! NSPEC2D_TOP(IREGION_CRUST_MANTLE) + ispec = ibelm_top_crust_mantle(ispec2D) + + ! in case of global, NCHUNKS_VAL == 6 simulations, be aware that for + ! the cubed sphere, the mapping changes for different chunks, + ! i.e. e.g. x(1,1) and x(5,5) flip left and right sides of the elements in geographical coordinates. + ! for future consideration, like in create_movie_GMT_global.f90 ... + k = NGLLZ + + ! loop on all the points inside the element + if (.not. MOVIE_COARSE) then + do j = 1, NGLLY-1, 1 + do i = 1, NGLLX-1, 1 + ! stores values + iglob1 = ibool_crust_mantle(i,j,k,ispec) + iglob2 = ibool_crust_mantle(i+1,j,k,ispec) + iglob3 = ibool_crust_mantle(i+1,j+1,k,ispec) + iglob4 = ibool_crust_mantle(i,j+1,k,ispec) + + if (MOVIE_VOLUME_TYPE == 5) then + ! stores displacement + ! iglob1 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob2 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob3 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob4 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + else + ! stores velocity + ! iglob1 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob2 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob3 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob4 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + endif + enddo + enddo + else ! MOVIE_COARSE + iglob1 = ibool_crust_mantle(1,1,k,ispec) + iglob2 = ibool_crust_mantle(NGLLX-1,1,k,ispec) + iglob3 = ibool_crust_mantle(NGLLX-1,NGLLY-1,k,ispec) + iglob4 = ibool_crust_mantle(1,NGLLY-1,k,ispec) + + if (MOVIE_VOLUME_TYPE == 5) then + ! stores displacement + ! iglob1 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob1) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob2 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob2) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob3 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob3) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob4 + ipoin = ipoin + 1 + store_val_ux(ipoin) = displ_crust_mantle(1,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = displ_crust_mantle(2,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = displ_crust_mantle(3,iglob4) * real(scale_displ,kind=CUSTOM_REAL) ! longitude phi (in radian) + else + ! stores velocity + ! iglob1 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob1) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob2 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob2) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob3 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob3) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + ! iglob4 + ipoin = ipoin + 1 + store_val_ux(ipoin) = veloc_crust_mantle(1,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! radius r (normalized) + store_val_uy(ipoin) = veloc_crust_mantle(2,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! colatitude theta (in radian) + store_val_uz(ipoin) = veloc_crust_mantle(3,iglob4) * real(scale_veloc,kind=CUSTOM_REAL) ! longitude phi (in radian) + endif + endif + enddo + ! TODO ADD IOSERVER + + ! initialize h5 file for surface movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create file and dataset + file_name = trim(OUTPUT_FILES)//"/movie_surface.h5" + group_name = "it_"//trim(i2c(it)) + + ! create dataset + if (myrank == 0) then + call h5_open_file(file_name) + call h5_create_group(group_name) + call h5_open_group(group_name) + + ! create datasets ux, uy, uz + call h5_create_dataset_gen_in_group("ux", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("uy", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group("uz", (/npoints_surf_mov_all_proc/), 1, CUSTOM_REAL) + + ! close group + call h5_close_group() + ! close file + call h5_close_file() + endif + + call synchronize_all() + + ! write data to h5 file + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + ! write ux, uy, uz + call h5_write_dataset_collect_hyperslab_in_group("ux", store_val_ux, (/sum(offset_poin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("uy", store_val_uy, (/sum(offset_poin(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group("uz", store_val_uz, (/sum(offset_poin(0:myrank-1))/), H5_COL) + + ! close group and file + call h5_close_group() + call h5_close_file_p() + + ! write xdmf body + call write_xdmf_surface_body(it, npoints_surf_mov_all_proc) + +#else + + write(*,*) 'Error: HDF5 is not enabled in this version of Specfem3D_Globe.' + write(*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + +end subroutine write_movie_surface_hdf5 + + + +! +! xdmf output routines +! +#ifdef USE_HDF5 + + subroutine write_xdmf_surface_header(num_nodes) + + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + integer, intent(in) :: num_nodes + ! local parameters + integer :: num_elm + character(len=MAX_STRING_LEN) :: fname_xdmf_surf + character(len=MAX_STRING_LEN) :: fname_h5_data_surf_xdmf + + ! checks if anything do, only main process writes out xdmf file + if (myrank /= 0) return + + ! writeout xdmf file for surface movie + fname_xdmf_surf = trim(OUTPUT_FILES) // "/movie_surface.xmf" + fname_h5_data_surf_xdmf = "./movie_surface.h5" ! relative to movie_surface.xmf file + ! note: this seems not to work and point to a wrong directory: + ! fname_h5_data_surf_xdmf = trim(OUTPUT_FILES) // "/movie_surface.h5" + + num_elm = num_nodes / 4 + + open(unit=xdmf_surf, file=trim(fname_xdmf_surf), recl=256) + + write(xdmf_surf,'(a)') '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) ' '//trim(fname_h5_data_surf_xdmf)//':/surf_coord/x' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) ' '//trim(fname_h5_data_surf_xdmf)//':/surf_coord/y' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) ' '//trim(fname_h5_data_surf_xdmf)//':/surf_coord/z' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + + write(xdmf_surf,*) '' + ! 17 lines + + ! file finish + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + ! 20 lines + + ! position where the additional data will be inserted + surf_xdmf_pos = 17 + + close(xdmf_surf) + + end subroutine write_xdmf_surface_header + +#endif + + +#ifdef USE_HDF5 + + subroutine write_xdmf_surface_body(it_io, num_nodes) + + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + integer, intent(in) :: it_io + integer, intent(in) :: num_nodes + ! local parameters + integer :: i + character(len=20) :: it_str + character(len=MAX_STRING_LEN) :: fname_xdmf_surf + character(len=MAX_STRING_LEN) :: fname_h5_data_surf_xdmf + + ! checks if anything do, only main process writes out xdmf file + if (myrank /= 0) return + + ! append data section to xdmf file for surface movie + fname_xdmf_surf = trim(OUTPUT_FILES)//"/movie_surface.xmf" + fname_h5_data_surf_xdmf = "./movie_surface.h5" ! relative to movie_surface.xmf file + ! this seems to point to a wrong directory: + ! fname_h5_data_surf_xdmf = trim(OUTPUT_FILES) // "/movie_surface.h5" + + ! open xdmf file + open(unit=xdmf_surf, file=trim(fname_xdmf_surf), status='old', recl=256) + + ! skip lines till the position where we want to write new information + do i = 1, surf_xdmf_pos + read(xdmf_surf, *) + enddo + + !write(it_str, "(i6.6)") it_io + it_str = i2c(it_io) + + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + ! 20 lines + + ! file finish + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + write(xdmf_surf,*) '' + + close(xdmf_surf) + + ! updates file record position + surf_xdmf_pos = surf_xdmf_pos + 20 + + end subroutine write_xdmf_surface_body + +#endif + diff --git a/src/specfem3D/write_movie_volume_hdf5.F90 b/src/specfem3D/write_movie_volume_hdf5.F90 new file mode 100644 index 000000000..e60f0242e --- /dev/null +++ b/src/specfem3D/write_movie_volume_hdf5.F90 @@ -0,0 +1,2132 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + subroutine movie_volume_init_hdf5() +#ifdef USE_HDF5 + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + integer :: ier + + ! nglobs + allocate(offset_poin_vol(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_poin_vol') + allocate(offset_poin_vol_cm(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_poin_vol_cm') + allocate(offset_poin_vol_oc(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_poin_vol_oc') + allocate(offset_poin_vol_ic(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_poin_vol_ic') + allocate(offset_nspec_vol(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_nspec_vol') + allocate(offset_nspec_vol_cm(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_nspec_vol_cm') + allocate(offset_nspec_vol_oc(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_nspec_vol_oc') + allocate(offset_nspec_vol_ic(0:NPROCTOT_VAL-1),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating offset_nspec_vol_ic') + + npoints_vol_mov_all_proc = 0 + npoints_vol_mov_all_proc_cm = 0 + npoints_vol_mov_all_proc_oc = 0 + npoints_vol_mov_all_proc_ic = 0 + + nspec_vol_mov_all_proc = 0 + nspec_vol_mov_all_proc_cm = 0 + nspec_vol_mov_all_proc_oc = 0 + nspec_vol_mov_all_proc_ic = 0 + + nspec_vol_mov_all_proc_cm_conn = 0 + nspec_vol_mov_all_proc_oc_conn = 0 + nspec_vol_mov_all_proc_ic_conn = 0 + + ! if strain or vector output is requested + if ((MOVIE_VOLUME_TYPE == 1 .or. MOVIE_VOLUME_TYPE == 2 .or. MOVIE_VOLUME_TYPE == 3 & + .or. MOVIE_VOLUME_TYPE == 5 .or. MOVIE_VOLUME_TYPE == 6 )) then + output_sv = .true. + endif + ! check if crust mantle region is used for movie + if (MOVIE_VOLUME_TYPE == 4 .and. OUTPUT_CRUST_MANTLE) then + output_cm = .true. + endif + ! check if outer core region is used for movie + if ( (MOVIE_VOLUME_TYPE == 4 .or. MOVIE_VOLUME_TYPE == 7 .or. MOVIE_VOLUME_TYPE == 8 .or. MOVIE_VOLUME_TYPE == 9) & + .and. OUTPUT_OUTER_CORE) then + output_oc = .true. + endif + ! check if inner core region is used for movie + if ( (MOVIE_VOLUME_TYPE == 4 .or. MOVIE_VOLUME_TYPE == 7 .or. MOVIE_VOLUME_TYPE == 8 .or. MOVIE_VOLUME_TYPE == 9) & + .and. OUTPUT_INNER_CORE) then + output_ic = .true. + endif + + ! force false if number of elements is zero + if (NSPEC_CRUST_MANTLE == 0) output_cm = .false. + if (NSPEC_OUTER_CORE == 0) output_oc = .false. + if (NSPEC_INNER_CORE == 0) output_ic = .false. + + ! print + !print*, 'output_sv = ',output_sv + !print*, 'output_cm = ',output_cm + !print*, 'output_oc = ',output_oc + !print*, 'output_ic = ',output_ic + !print*, 'NSPEC_CRUST_MANTLE = ',NSPEC_CRUST_MANTLE + !print*, 'NSPEC_CRUST_MANTLE_STRAIN_ONLY = ',NSPEC_CRUST_MANTLE_STRAIN_ONLY + !print*, 'NSPEC_CRUST_MANTLE_STR_OR_ATT = ',NSPEC_CRUST_MANTLE_STR_OR_ATT + !print*, 'NSPEC_OUTER_CORE = ',NSPEC_OUTER_CORE + !print*, 'NSPEC_OUTER_CORE_3DMOVIE = ',NSPEC_OUTER_CORE_3DMOVIE + !print*, 'NSPEC_INNER_CORE = ',NSPEC_INNER_CORE + +#else + write (*,*) 'Error: HDF5 is not enabled in this version of the code.' + write (*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop +#endif + end subroutine movie_volume_init_hdf5 + + subroutine movie_volume_finalize_hdf5() +#ifdef USE_HDF5 + use specfem_par_movie_hdf5 + + deallocate(offset_poin_vol) + deallocate(offset_poin_vol_cm) + deallocate(offset_poin_vol_oc) + deallocate(offset_poin_vol_ic) + deallocate(offset_nspec_vol) + deallocate(offset_nspec_vol_cm) + deallocate(offset_nspec_vol_oc) + deallocate(offset_nspec_vol_ic) + +#else + write (*,*) 'Error: HDF5 is not enabled in this version of the code.' + write (*,*) 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop +#endif + end subroutine movie_volume_finalize_hdf5 + + subroutine write_movie_volume_mesh_hdf5(nu_3dmovie,num_ibool_3dmovie,mask_3dmovie,mask_ibool_3dmovie, & + muvstore_crust_mantle_3dmovie,npoints_3dmovie,nelems_3dmovie_in) + + use specfem_par + +#ifdef USE_HDF5 + use specfem_par_crustmantle, only: ibool_crust_mantle,rstore_crust_mantle + use specfem_par_outercore, only: ibool_outer_core,rstore_outer_core + use specfem_par_innercore, only: ibool_inner_core,rstore_inner_core + use specfem_par_movie_hdf5 +#endif + + implicit none + + integer,intent(in) :: npoints_3dmovie,nelems_3dmovie_in + integer, dimension(NGLOB_CRUST_MANTLE_3DMOVIE),intent(in) :: num_ibool_3dmovie + + real(kind=CUSTOM_REAL), dimension(3,3,npoints_3dmovie),intent(inout) :: nu_3dmovie + + logical, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_3DMOVIE),intent(in) :: mask_3dmovie + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_3DMOVIE),intent(in) :: muvstore_crust_mantle_3dmovie + logical, dimension(NGLOB_CRUST_MANTLE_3DMOVIE),intent(in) :: mask_ibool_3dmovie + +#ifdef USE_HDF5 + ! local parameters + integer :: ipoints_3dmovie,ispec,i,j,k,iNIT,nelems_3dmovie + integer :: iglob,iglob_center + real(kind=CUSTOM_REAL) :: rval,thetaval,phival,xval,yval,zval,st,ct,sp,cp + real(kind=CUSTOM_REAL), dimension(npoints_3dmovie) :: store_val3D_x, store_val3D_y, store_val3D_z + real(kind=CUSTOM_REAL), dimension(NGLOB_CRUST_MANTLE) :: store_val3D_x_cm, store_val3D_y_cm, store_val3D_z_cm + real(kind=CUSTOM_REAL), dimension(NGLOB_OUTER_CORE) :: store_val3D_x_oc, store_val3D_y_oc, store_val3D_z_oc + real(kind=CUSTOM_REAL), dimension(NGLOB_INNER_CORE) :: store_val3D_x_ic, store_val3D_y_ic, store_val3D_z_ic + real(kind=CUSTOM_REAL), dimension(npoints_3dmovie) :: store_val3D_mu + ! dummy num_ibool_3dmovie for cm oc ic + integer, dimension(NGLOB_CRUST_MANTLE) :: num_ibool_3dmovie_cm + integer, dimension(NGLOB_OUTER_CORE) :: num_ibool_3dmovie_oc + integer, dimension(NGLOB_INNER_CORE) :: num_ibool_3dmovie_ic + ! dummy mask_ibool_3dmovie for cm oc ic + logical, dimension(NGLOB_CRUST_MANTLE) :: mask_ibool_3dmovie_cm + logical, dimension(NGLOB_OUTER_CORE) :: mask_ibool_3dmovie_oc + logical, dimension(NGLOB_INNER_CORE) :: mask_ibool_3dmovie_ic + + integer, dimension(:,:), allocatable :: elm_conn, elm_conn_cm, elm_conn_oc, elm_conn_ic + + integer, dimension(0:NPROCTOT_VAL-1) :: offset_nspec_vol_cm_conn, offset_nspec_vol_oc_conn, offset_nspec_vol_ic_conn + + integer :: nelems_3dmovie_cm, nelems_3dmovie_oc, nelems_3dmovie_ic + + ! initialize arrays for hdf5 volume movie output + call movie_volume_init_hdf5() + + ! safety check + if (NDIM /= 3) stop 'movie volume output requires NDIM = 3' + + ! output resolution + if (MOVIE_COARSE) then + iNIT = NGLLX-1 + nelems_3dmovie = nelems_3dmovie_in + else + iNIT = 1 + nelems_3dmovie = nelems_3dmovie_in * (NGLLX-1) * (NGLLY-1) * (NGLLZ-1) + endif + + ! outer core and inner core is always fine mesh + nelems_3dmovie_cm = NSPEC_CRUST_MANTLE * (NGLLX-1) * (NGLLY-1) * (NGLLZ-1) + nelems_3dmovie_oc = NSPEC_OUTER_CORE * (NGLLX-1) * (NGLLY-1) * (NGLLZ-1) + nelems_3dmovie_ic = NSPEC_INNER_CORE * (NGLLX-1) * (NGLLY-1) * (NGLLZ-1) + + ! allocate elm_conn + allocate(elm_conn(9,nelems_3dmovie)) + allocate(elm_conn_cm(9,nelems_3dmovie_cm)) + allocate(elm_conn_oc(9,nelems_3dmovie_oc)) + allocate(elm_conn_ic(9,nelems_3dmovie_ic)) + + ! prepare offset array + call gather_all_all_singlei(npoints_3dmovie, offset_poin_vol, NPROCTOT_VAL) + call gather_all_all_singlei(NGLOB_CRUST_MANTLE, offset_poin_vol_cm, NPROCTOT_VAL) + call gather_all_all_singlei(NGLOB_OUTER_CORE, offset_poin_vol_oc, NPROCTOT_VAL) + call gather_all_all_singlei(NGLOB_INNER_CORE, offset_poin_vol_ic, NPROCTOT_VAL) + + call gather_all_all_singlei(nelems_3dmovie, offset_nspec_vol, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_CRUST_MANTLE, offset_nspec_vol_cm, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_OUTER_CORE, offset_nspec_vol_oc, NPROCTOT_VAL) + call gather_all_all_singlei(NSPEC_INNER_CORE, offset_nspec_vol_ic, NPROCTOT_VAL) + + ! offset arrays for element connectivity + call gather_all_all_singlei(nelems_3dmovie_cm, offset_nspec_vol_cm_conn, NPROCTOT_VAL) + call gather_all_all_singlei(nelems_3dmovie_oc, offset_nspec_vol_oc_conn, NPROCTOT_VAL) + call gather_all_all_singlei(nelems_3dmovie_ic, offset_nspec_vol_ic_conn, NPROCTOT_VAL) + + npoints_vol_mov_all_proc = sum(offset_poin_vol) + npoints_vol_mov_all_proc_cm = sum(offset_poin_vol_cm) + npoints_vol_mov_all_proc_oc = sum(offset_poin_vol_oc) + npoints_vol_mov_all_proc_ic = sum(offset_poin_vol_ic) + + nspec_vol_mov_all_proc = sum(offset_nspec_vol) + nspec_vol_mov_all_proc_cm = sum(offset_nspec_vol_cm) + nspec_vol_mov_all_proc_oc = sum(offset_nspec_vol_oc) + nspec_vol_mov_all_proc_ic = sum(offset_nspec_vol_ic) + + nspec_vol_mov_all_proc_cm_conn = sum(offset_nspec_vol_cm_conn) + nspec_vol_mov_all_proc_oc_conn = sum(offset_nspec_vol_oc_conn) + nspec_vol_mov_all_proc_ic_conn = sum(offset_nspec_vol_ic_conn) + + !if (myrank == 0) then + ! print*, 'npoints_vol_mov_all_proc = ', npoints_vol_mov_all_proc + ! print*, 'npoints_vol_mov_all_proc_cm = ', npoints_vol_mov_all_proc_cm + ! print*, 'npoints_vol_mov_all_proc_oc = ', npoints_vol_mov_all_proc_oc + ! print*, 'npoints_vol_mov_all_proc_ic = ', npoints_vol_mov_all_proc_ic + + ! print*, 'nspec_vol_mov_all_proc = ', nspec_vol_mov_all_proc + ! print*, 'nspec_vol_mov_all_proc_cm = ', nspec_vol_mov_all_proc_cm + ! print*, 'nspec_vol_mov_all_proc_oc = ', nspec_vol_mov_all_proc_oc + ! print*, 'nspec_vol_mov_all_proc_ic = ', nspec_vol_mov_all_proc_ic + + ! print*, 'nspec_vol_mov_all_proc_cm_conn = ', nspec_vol_mov_all_proc_cm_conn + ! print*, 'nspec_vol_mov_all_proc_oc_conn = ', nspec_vol_mov_all_proc_oc_conn + ! print*, 'nspec_vol_mov_all_proc_ic_conn = ', nspec_vol_mov_all_proc_ic_conn + !endif + + ! + ! create the xyz arrays for crust and mantle and strain and vector output + ! + if (output_sv) then + ! loops over all elements + ipoints_3dmovie = 0 + do ispec = 1,NSPEC_CRUST_MANTLE + + ! checks center of element for movie flag + iglob_center = ibool_crust_mantle((NGLLX+1)/2,(NGLLY+1)/2,(NGLLZ+1)/2,ispec) + + ! checks if movie element + if (mask_ibool_3dmovie(iglob_center)) then + + ! stores element coordinates + do k = 1,NGLLZ,iNIT + do j = 1,NGLLY,iNIT + do i = 1,NGLLX,iNIT + ! only store points once + if (mask_3dmovie(i,j,k,ispec)) then + ! point increment + ipoints_3dmovie = ipoints_3dmovie + 1 + + ! gets point position + iglob = ibool_crust_mantle(i,j,k,ispec) + + rval = rstore_crust_mantle(1,iglob) + thetaval = rstore_crust_mantle(2,iglob) + phival = rstore_crust_mantle(3,iglob) + + !x,y,z store have been converted to r theta phi already, need to revert back for xyz output + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + + store_val3D_x(ipoints_3dmovie) = xval + store_val3D_y(ipoints_3dmovie) = yval + store_val3D_z(ipoints_3dmovie) = zval + store_val3D_mu(ipoints_3dmovie) = muvstore_crust_mantle_3dmovie(i,j,k,ispec) + + st = sin(thetaval) + ct = cos(thetaval) + sp = sin(phival) + cp = cos(phival) + + nu_3dmovie(1,1,ipoints_3dmovie) = -ct*cp + nu_3dmovie(1,2,ipoints_3dmovie) = -ct*sp + nu_3dmovie(1,3,ipoints_3dmovie) = st + nu_3dmovie(2,1,ipoints_3dmovie) = -sp + nu_3dmovie(2,2,ipoints_3dmovie) = cp + nu_3dmovie(2,3,ipoints_3dmovie) = 0.d0 + nu_3dmovie(3,1,ipoints_3dmovie) = st*cp + nu_3dmovie(3,2,ipoints_3dmovie) = st*sp + nu_3dmovie(3,3,ipoints_3dmovie) = ct + endif !mask_3dmovie + enddo !i + enddo !j + enddo !k + endif + + enddo !ispec + + ! check if counters are correct + if (ipoints_3dmovie /= npoints_3dmovie) then + print*, 'Error: did not find the right number of points for 3D movie' + print*, 'ipoints_3dmovie = ',ipoints_3dmovie,' npoints_3dmovie = ',npoints_3dmovie + stop + endif + + endif + + ! + ! create the xyz arrays for crust and mantle (not strain or vector output) + ! + if (output_cm) then + + do ispec = 1,NSPEC_CRUST_MANTLE + do k = 1,NGLLZ,1 + do j = 1,NGLLY,1 + do i = 1,NGLLX,1 + iglob = ibool_crust_mantle(i,j,k,ispec) + rval = rstore_crust_mantle(1,iglob) + thetaval = rstore_crust_mantle(2,iglob) + phival = rstore_crust_mantle(3,iglob) + + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + + store_val3D_x_cm(iglob) = xval + store_val3D_y_cm(iglob) = yval + store_val3D_z_cm(iglob) = zval + + ! dummy num_ibool_3dmovie for cm + num_ibool_3dmovie_cm(iglob) = iglob + ! all the mask_ibool_3dmovie_cm are true + mask_ibool_3dmovie_cm(iglob) = .true. + enddo + enddo + enddo + enddo + + endif + + ! + ! create the xyz arrays for outer core + ! + if (output_oc) then + + do ispec = 1, NSPEC_OUTER_CORE + do k = 1,NGLLZ,1 + do j = 1,NGLLY,1 + do i = 1,NGLLX,1 + iglob = ibool_outer_core(i,j,k,ispec) + rval = rstore_outer_core(1,iglob) + thetaval = rstore_outer_core(2,iglob) + phival = rstore_outer_core(3,iglob) + + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + + store_val3D_x_oc(iglob) = xval + store_val3D_y_oc(iglob) = yval + store_val3D_z_oc(iglob) = zval + + ! dummy num_ibool_3dmovie for oc + num_ibool_3dmovie_oc(iglob) = iglob + ! all the mask_ibool_3dmovie_oc are true + mask_ibool_3dmovie_oc(iglob) = .true. + enddo + enddo + enddo + enddo + + endif + + ! + ! create the xyz arrays for inner core + ! + if (output_ic) then + + do ispec = 1, NSPEC_INNER_CORE + do k = 1,NGLLZ,1 + do j = 1,NGLLY,1 + do i = 1,NGLLX,1 + iglob = ibool_inner_core(i,j,k,ispec) + rval = rstore_inner_core(1,iglob) + thetaval = rstore_inner_core(2,iglob) + phival = rstore_inner_core(3,iglob) + + call rthetaphi_2_xyz(xval,yval,zval,rval,thetaval,phival) + + store_val3D_x_ic(iglob) = xval + store_val3D_y_ic(iglob) = yval + store_val3D_z_ic(iglob) = zval + + ! dummy num_ibool_3dmovie for ic + num_ibool_3dmovie_ic(iglob) = iglob + ! all the mask_ibool_3dmovie_ic are true + mask_ibool_3dmovie_ic(iglob) = .true. + enddo + enddo + enddo + enddo + + endif + + ! create elm_conn for movie + ! for crust and mantle (strain and vector output) + if (output_sv) call get_conn_for_movie(elm_conn, sum(offset_poin_vol(0:myrank-1)), iNIT, nelems_3dmovie, & + npoints_3dmovie, NSPEC_CRUST_MANTLE, num_ibool_3dmovie, mask_ibool_3dmovie, ibool_crust_mantle) + ! for crust and mantle (not strain or vector output) + if (output_cm) call get_conn_for_movie(elm_conn_cm, sum(offset_poin_vol_cm(0:myrank-1)), 1, nelems_3dmovie_cm, & + NGLOB_CRUST_MANTLE, NSPEC_CRUST_MANTLE, num_ibool_3dmovie_cm, mask_ibool_3dmovie_cm, ibool_crust_mantle) + ! for outer core + if (output_oc) call get_conn_for_movie(elm_conn_oc, sum(offset_poin_vol_oc(0:myrank-1)), 1, nelems_3dmovie_oc, & + NGLOB_OUTER_CORE, NSPEC_OUTER_CORE, num_ibool_3dmovie_oc, mask_ibool_3dmovie_oc, ibool_outer_core) + ! for inner core + if (output_ic) call get_conn_for_movie(elm_conn_ic, sum(offset_poin_vol_ic(0:myrank-1)), 1, nelems_3dmovie_ic, & + NGLOB_INNER_CORE, NSPEC_INNER_CORE, num_ibool_3dmovie_ic, mask_ibool_3dmovie_ic, ibool_inner_core) + + ! TODO ADD IOSERVER + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! file name and group name + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'mesh' + + ! create the file, group and dataset + if (myrank == 0) then + call h5_create_file(file_name) + call h5_open_or_create_group(group_name) + + ! create the dataset + ! for crust and mantle (strain and vector output) + if (output_sv) then + call h5_create_dataset_gen_in_group('elm_conn', (/9, nspec_vol_mov_all_proc/), 2, 1) + call h5_create_dataset_gen_in_group('x', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('y', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('z', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + endif + ! for crust and mantle (not strain or vector output) + if (output_cm) then + call h5_create_dataset_gen_in_group('elm_conn_cm', (/9, nspec_vol_mov_all_proc_cm_conn/), 2, 1) + call h5_create_dataset_gen_in_group('x_cm', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('y_cm', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('z_cm', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + endif + ! for outer core + if (output_oc) then + call h5_create_dataset_gen_in_group('elm_conn_oc', (/9, nspec_vol_mov_all_proc_oc_conn/), 2, 1) + call h5_create_dataset_gen_in_group('x_oc', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('y_oc', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('z_oc', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) + endif + ! for inner core + if (output_ic) then + call h5_create_dataset_gen_in_group('elm_conn_ic', (/9, nspec_vol_mov_all_proc_ic_conn/), 2, 1) + call h5_create_dataset_gen_in_group('x_ic', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('y_ic', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('z_ic', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + endif + + ! close the group + call h5_close_group() + ! close the file + call h5_close_file() + + endif + + ! synchronize + call synchronize_all() + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + ! write the data + ! for crust and mantle (strain and vector output) + if (output_sv) then + call h5_write_dataset_collect_hyperslab_in_group('elm_conn', elm_conn, (/0, sum(offset_nspec_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('x', store_val3D_x, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('y', store_val3D_y, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('z', store_val3D_z, (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + endif + ! for crust and mantle (not strain or vector output) + if (output_cm) then + call h5_write_dataset_collect_hyperslab_in_group('elm_conn_cm', elm_conn_cm, & + (/0, sum(offset_nspec_vol_cm_conn(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('x_cm', store_val3D_x_cm, (/sum(offset_poin_vol_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('y_cm', store_val3D_y_cm, (/sum(offset_poin_vol_cm(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('z_cm', store_val3D_z_cm, (/sum(offset_poin_vol_cm(0:myrank-1))/), H5_COL) + endif + ! for outer core + if (output_oc) then + call h5_write_dataset_collect_hyperslab_in_group('elm_conn_oc', elm_conn_oc, & + (/0, sum(offset_nspec_vol_oc_conn(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('x_oc', store_val3D_x_oc, (/sum(offset_poin_vol_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('y_oc', store_val3D_y_oc, (/sum(offset_poin_vol_oc(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('z_oc', store_val3D_z_oc, (/sum(offset_poin_vol_oc(0:myrank-1))/), H5_COL) + endif + ! for inner core + if (output_ic) then + call h5_write_dataset_collect_hyperslab_in_group('elm_conn_ic', elm_conn_ic, & + (/0, sum(offset_nspec_vol_ic_conn(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('x_ic', store_val3D_x_ic, (/sum(offset_poin_vol_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('y_ic', store_val3D_y_ic, (/sum(offset_poin_vol_ic(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group('z_ic', store_val3D_z_ic, (/sum(offset_poin_vol_ic(0:myrank-1))/), H5_COL) + endif + + ! close the group + call h5_close_group() + ! close the file + call h5_close_file_p() + + ! deallocate + deallocate(elm_conn) + deallocate(elm_conn_cm) + deallocate(elm_conn_oc) + deallocate(elm_conn_ic) + + ! write xdmf for all timesteps + call write_xdmf_vol_hdf5(npoints_vol_mov_all_proc, nspec_vol_mov_all_proc, & + npoints_vol_mov_all_proc_cm, nspec_vol_mov_all_proc_cm, & + npoints_vol_mov_all_proc_oc, nspec_vol_mov_all_proc_oc, & + npoints_vol_mov_all_proc_ic, nspec_vol_mov_all_proc_ic) + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_mesh_hdf5 + + + subroutine write_movie_volume_strains_hdf5(vnspec_eps_cm, & + eps_trace_over_3_crust_mantle, & + vnspec_cm, & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle) + + use constants_solver +#ifdef USE_HDF5 + use shared_parameters, only: OUTPUT_FILES,MOVIE_VOLUME_TYPE,MOVIE_COARSE,H5_COL + use specfem_par, only: it + use specfem_par_movie, only: npoints_3dmovie,muvstore_crust_mantle_3dmovie,mask_3dmovie,nu_3dmovie + use specfem_par_movie_hdf5 +#endif + + implicit none + + ! input + integer,intent(in) :: vnspec_eps_cm + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_eps_cm),intent(in) :: eps_trace_over_3_crust_mantle + + integer,intent(in) :: vnspec_cm + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_cm),intent(in) :: & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle + +#ifdef USE_HDF5 + ! variables + real(kind=CUSTOM_REAL) :: muv_3dmovie + real(kind=CUSTOM_REAL),dimension(3,3) :: eps_loc,eps_loc_new + real(kind=CUSTOM_REAL),dimension(:),allocatable :: store_val3d_NN,store_val3d_EE,store_val3d_ZZ, & + store_val3d_NE,store_val3d_NZ,store_val3d_EZ + integer :: ipoints_3dmovie,i,j,k,ispec,iNIT,ier + character(len=1) :: movie_prefix + + ! check + if (NDIM /= 3) call exit_MPI(myrank, 'write_movie_volume_strains() requires NDIM = 3') + if (vnspec_cm /= NSPEC_CRUST_MANTLE) call exit_MPI(myrank,'Invalid vnspec_cm value for write_movie_volume_strains() routine') + + ! allocates arrays + allocate(store_val3d_NN(npoints_3dmovie), & + store_val3d_EE(npoints_3dmovie), & + store_val3d_ZZ(npoints_3dmovie), & + store_val3d_NE(npoints_3dmovie), & + store_val3d_NZ(npoints_3dmovie), & + store_val3d_EZ(npoints_3dmovie), & + stat=ier) + if (ier /= 0 ) call exit_mpi(myrank,'Error allocating store_val3d_ .. arrays') + + if (MOVIE_VOLUME_TYPE == 1) then + movie_prefix='E' ! strain + else if (MOVIE_VOLUME_TYPE == 2) then + movie_prefix='S' ! time integral of strain + else if (MOVIE_VOLUME_TYPE == 3) then + movie_prefix='P' ! potency, or integral of strain x \mu + endif + + ! stepping + if (MOVIE_COARSE) then + iNIT = NGLLX-1 + else + iNIT = 1 + endif + + ipoints_3dmovie = 0 + do ispec = 1,NSPEC_CRUST_MANTLE + do k = 1,NGLLZ,iNIT + do j = 1,NGLLY,iNIT + do i = 1,NGLLX,iNIT + if (mask_3dmovie(i,j,k,ispec)) then + ipoints_3dmovie = ipoints_3dmovie + 1 + muv_3dmovie = muvstore_crust_mantle_3dmovie(i,j,k,ispec) + + eps_loc(1,1) = eps_trace_over_3_crust_mantle(i,j,k,ispec) + epsilondev_xx_crust_mantle(i,j,k,ispec) + eps_loc(2,2) = eps_trace_over_3_crust_mantle(i,j,k,ispec) + epsilondev_yy_crust_mantle(i,j,k,ispec) + eps_loc(3,3) = eps_trace_over_3_crust_mantle(i,j,k,ispec) & + - epsilondev_xx_crust_mantle(i,j,k,ispec) & + - epsilondev_yy_crust_mantle(i,j,k,ispec) + + eps_loc(1,2) = epsilondev_xy_crust_mantle(i,j,k,ispec) + eps_loc(1,3) = epsilondev_xz_crust_mantle(i,j,k,ispec) + eps_loc(2,3) = epsilondev_yz_crust_mantle(i,j,k,ispec) + + eps_loc(2,1) = eps_loc(1,2) + eps_loc(3,1) = eps_loc(1,3) + eps_loc(3,2) = eps_loc(2,3) + + ! rotate eps_loc to spherical coordinates + eps_loc_new(:,:) = matmul(matmul(nu_3dmovie(:,:,ipoints_3dmovie),eps_loc(:,:)), & + transpose(nu_3dmovie(:,:,ipoints_3dmovie))) + if (MOVIE_VOLUME_TYPE == 3) eps_loc_new(:,:) = eps_loc(:,:)*muv_3dmovie + + store_val3d_NN(ipoints_3dmovie) = eps_loc_new(1,1) + store_val3d_EE(ipoints_3dmovie) = eps_loc_new(2,2) + store_val3d_ZZ(ipoints_3dmovie) = eps_loc_new(3,3) + store_val3d_NE(ipoints_3dmovie) = eps_loc_new(1,2) + store_val3d_NZ(ipoints_3dmovie) = eps_loc_new(1,3) + store_val3d_EZ(ipoints_3dmovie) = eps_loc_new(2,3) + endif + enddo + enddo + enddo + enddo + if (ipoints_3dmovie /= npoints_3dmovie) stop 'did not find the right number of points for 3D movie' + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + ! create group and datasets + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + if (myrank == 0) then + + ! create the file, group and dataset + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + ! create the dataset + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'NN', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'EE', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'ZZ', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'NE', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'NZ', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'EZ', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + + ! close the group + call h5_close_group() + ! close the file + call h5_close_file() + + endif + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NN', store_val3d_NN, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EE', store_val3d_EE, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'ZZ', store_val3d_ZZ, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NE', store_val3d_NE, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'NZ', store_val3d_NZ, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'EZ', store_val3d_EZ, & + (/sum(offset_poin_vol(0:myrank-1))/), H5_COL) + + call h5_close_group() + call h5_close_file_p() + + deallocate(store_val3d_NN,store_val3d_EE,store_val3d_ZZ, & + store_val3d_NE,store_val3d_NZ,store_val3d_EZ) + + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_strains_hdf5 + + + + +subroutine write_movie_volume_divcurl_hdf5(vnspec_eps_cm,eps_trace_over_3_crust_mantle, & + div_displ_outer_core, & + accel_outer_core,kappavstore_outer_core,rhostore_outer_core,ibool_outer_core, & + vnspec_eps_ic,eps_trace_over_3_inner_core, & + vnspec_cm,epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle, & + vnspec_ic,epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, & + epsilondev_xz_inner_core,epsilondev_yz_inner_core) + +! outputs divergence and curl: MOVIE_VOLUME_TYPE == 4 + + use constants_solver + +#ifdef USE_HDF5 + use shared_parameters, only: OUTPUT_FILES,H5_COL + use specfem_par, only: it + use specfem_par_crustmantle, only: ibool_crust_mantle + use specfem_par_innercore, only: ibool_inner_core + use specfem_par_movie_hdf5 +#endif + + implicit none + + integer,intent(in) :: vnspec_eps_cm + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_eps_cm) :: eps_trace_over_3_crust_mantle + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE_3DMOVIE) :: div_displ_outer_core + + real(kind=CUSTOM_REAL), dimension(NGLOB_OUTER_CORE) :: accel_outer_core + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE) :: rhostore_outer_core,kappavstore_outer_core + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE) :: ibool_outer_core + + integer,intent(in) :: vnspec_eps_ic + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_eps_ic) :: eps_trace_over_3_inner_core + + integer,intent(in) :: vnspec_cm + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_cm) :: & + epsilondev_xx_crust_mantle,epsilondev_yy_crust_mantle,epsilondev_xy_crust_mantle, & + epsilondev_xz_crust_mantle,epsilondev_yz_crust_mantle + + integer,intent(in) :: vnspec_ic + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,vnspec_ic) :: & + epsilondev_xx_inner_core,epsilondev_yy_inner_core,epsilondev_xy_inner_core, & + epsilondev_xz_inner_core,epsilondev_yz_inner_core + +#ifdef USE_HDF5 + + ! local parameters + real(kind=CUSTOM_REAL) :: rhol,kappal + real(kind=CUSTOM_REAL), dimension(:,:,:,:), allocatable :: div_s_outer_core + integer :: ispec,iglob,i,j,k,ier + real(kind=CUSTOM_REAL), dimension(:), allocatable :: tmp_data + + ! checks + if (vnspec_cm /= NSPEC_CRUST_MANTLE) call exit_MPI(myrank,'Invalid vnspec_cm value for write_movie_volume_divcurl() routine') + + ! create group and datasets + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + if (myrank == 0) then + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + if (MOVIE_OUTPUT_DIV) then + call h5_create_dataset_gen_in_group('reg1_div_displ', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg2_div_displ', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg3_div_displ', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + endif + + if (MOVIE_OUTPUT_CURL) then + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xx', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_yy', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xy', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_xz', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('crust_mantle_epsdev_displ_yz', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xx', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_yy', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xy', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_xz', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('inner_core_epsdev_displ_yz', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + endif + + if (MOVIE_OUTPUT_CURLNORM) then + call h5_create_dataset_gen_in_group('reg1_epsdev_displ_norm', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group('reg3_epsdev_displ_norm', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + endif + + call h5_close_group() + call h5_close_file() + endif + + call synchronize_all() + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + if (MOVIE_OUTPUT_DIV) then + call write_array3dspec_as_1d_hdf5('reg1_div_displ', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + eps_trace_over_3_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + + if (NSPEC_OUTER_CORE_3DMOVIE > 1) then + call write_array3dspec_as_1d_hdf5('reg2_div_displ', offset_nspec_vol_oc(myrank), offset_poin_vol_oc(myrank), & + div_displ_outer_core, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) + else + allocate(div_s_outer_core(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array div_s_outer_core') + do ispec = 1, NSPEC_OUTER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_outer_core(i,j,k,ispec) + rhol = rhostore_outer_core(i,j,k,ispec) + kappal = kappavstore_outer_core(i,j,k,ispec) + div_s_outer_core(i,j,k,ispec) = rhol * accel_outer_core(iglob) / kappal + enddo + enddo + enddo + enddo + call write_array3dspec_as_1d_hdf5('reg2_div_displ', offset_nspec_vol_oc(myrank), offset_poin_vol_oc(myrank), & + div_s_outer_core, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) + deallocate(div_s_outer_core) + endif + + call write_array3dspec_as_1d_hdf5('reg3_div_displ', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + eps_trace_over_3_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + endif + + if (MOVIE_OUTPUT_CURL) then + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xx', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + epsilondev_xx_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_yy', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + epsilondev_yy_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xy', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + epsilondev_xy_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_xz', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + epsilondev_xz_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + call write_array3dspec_as_1d_hdf5('crust_mantle_epsdev_displ_yz', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + epsilondev_yz_crust_mantle, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xx', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + epsilondev_xx_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_yy', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + epsilondev_yy_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xy', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + epsilondev_xy_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_xz', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + epsilondev_xz_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + call write_array3dspec_as_1d_hdf5('inner_core_epsdev_displ_yz', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + epsilondev_yz_inner_core, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + endif + + if (MOVIE_OUTPUT_CURLNORM) then + allocate(tmp_data(NGLOB_CRUST_MANTLE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + ! Frobenius norm + do ispec = 1, NSPEC_CRUST_MANTLE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_crust_mantle(i,j,k,ispec) + tmp_data(iglob) = sqrt( epsilondev_xx_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_yy_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_xy_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_xz_crust_mantle(i,j,k,ispec)**2 & + + epsilondev_yz_crust_mantle(i,j,k,ispec)**2) + enddo + enddo + enddo + enddo + call write_array3dspec_as_1d_hdf5('reg1_epsdev_displ_norm', offset_nspec_vol_cm(myrank), offset_poin_vol_cm(myrank), & + tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + deallocate(tmp_data) + + ! Frobenius norm + allocate(tmp_data(NGLOB_INNER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + do ispec = 1, NSPEC_INNER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + tmp_data(iglob) = sqrt( epsilondev_xx_inner_core(i,j,k,ispec)**2 & + + epsilondev_yy_inner_core(i,j,k,ispec)**2 & + + epsilondev_xy_inner_core(i,j,k,ispec)**2 & + + epsilondev_xz_inner_core(i,j,k,ispec)**2 & + + epsilondev_yz_inner_core(i,j,k,ispec)**2) + enddo + enddo + enddo + enddo + call write_array3dspec_as_1d_hdf5('reg3_epsdev_displ_norm', offset_nspec_vol_ic(myrank), offset_poin_vol_ic(myrank), & + tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + deallocate(tmp_data) + endif + + call h5_close_group() + call h5_close_file() + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_divcurl_hdf5 + + + + subroutine write_movie_volume_vector_hdf5(npoints_3dmovie, & + ibool_crust_mantle,vector_crust_mantle, & + scalingval,mask_3dmovie,nu_3dmovie) + +! outputs displacement/velocity: MOVIE_VOLUME_TYPE == 5 / 6 + + use constants_solver +#ifdef USE_HDF5 + use shared_parameters, only: OUTPUT_FILES,MOVIE_VOLUME_TYPE,MOVIE_COARSE,H5_COL + use specfem_par, only: it + use specfem_par_movie_hdf5 +#endif + + implicit none + + ! input + integer :: npoints_3dmovie + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle + + ! displacement or velocity array + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CRUST_MANTLE) :: vector_crust_mantle + logical, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE_3DMOVIE) :: mask_3dmovie + + double precision :: scalingval + real(kind=CUSTOM_REAL), dimension(NDIM,NDIM,npoints_3dmovie) :: nu_3dmovie + +#ifdef USE_HDF5 + + ! local variables + real(kind=CUSTOM_REAL), dimension(:), allocatable :: store_val3d_N,store_val3d_E,store_val3d_Z + real(kind=CUSTOM_REAL), dimension(NDIM) :: vector_local,vector_local_new + + integer :: ipoints_3dmovie,i,j,k,ispec,iNIT,iglob,ier + character(len=2) :: movie_prefix + + ! check + if (NDIM /= 3) call exit_MPI(myrank,'write_movie_volume requires NDIM = 3') + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + ! allocates arrays + allocate(store_val3d_N(npoints_3dmovie), & + store_val3d_E(npoints_3dmovie), & + store_val3d_Z(npoints_3dmovie), & + stat=ier) + if (ier /= 0 ) call exit_mpi(myrank,'Error allocating store_val3d_N,.. movie arrays') + + if (MOVIE_VOLUME_TYPE == 5) then + movie_prefix='DI' ! displacement + else if (MOVIE_VOLUME_TYPE == 6) then + movie_prefix='VE' ! velocity + endif + + if (MOVIE_COARSE) then + iNIT = NGLLX-1 + else + iNIT = 1 + endif + + ipoints_3dmovie = 0 + + ! stores field in crust/mantle region + do ispec = 1,NSPEC_CRUST_MANTLE + do k = 1,NGLLZ,iNIT + do j = 1,NGLLY,iNIT + do i = 1,NGLLX,iNIT + if (mask_3dmovie(i,j,k,ispec)) then + ipoints_3dmovie = ipoints_3dmovie + 1 + iglob = ibool_crust_mantle(i,j,k,ispec) + + ! dimensionalizes field by scaling + vector_local(:) = vector_crust_mantle(:,iglob)*real(scalingval,kind=CUSTOM_REAL) + + ! rotate eps_loc to spherical coordinates + vector_local_new(:) = matmul(nu_3dmovie(:,:,ipoints_3dmovie), vector_local(:)) + + ! stores field + store_val3d_N(ipoints_3dmovie) = vector_local_new(1) + store_val3d_E(ipoints_3dmovie) = vector_local_new(2) + store_val3d_Z(ipoints_3dmovie) = vector_local_new(3) + endif + enddo + enddo + enddo + enddo + close(IOUT) + + ! checks number of processed points + if (ipoints_3dmovie /= npoints_3dmovie) stop 'did not find the right number of points for 3D movie' + + ! create group and datasets + if (myrank == 0) then + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'N', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'E', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + call h5_create_dataset_gen_in_group(trim(movie_prefix)//'Z', (/npoints_vol_mov_all_proc/), 1, CUSTOM_REAL) + + call h5_close_group() + call h5_close_file() + endif + + call synchronize_all() + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'N', store_val3d_N(1:npoints_3dmovie), & + (/offset_poin_vol(0:myrank-1)/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'E', store_val3d_E(1:npoints_3dmovie), & + (/offset_poin_vol(0:myrank-1)/), H5_COL) + call h5_write_dataset_collect_hyperslab_in_group(trim(movie_prefix)//'Z', store_val3d_Z(1:npoints_3dmovie), & + (/offset_poin_vol(0:myrank-1)/), H5_COL) + + call h5_close_group() + call h5_close_file() + + deallocate(store_val3d_N,store_val3d_E,store_val3d_Z) + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + + end subroutine write_movie_volume_vector_hdf5 + + + subroutine write_movie_volume_displnorm_hdf5(displ_crust_mantle,displ_inner_core,displ_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + +! outputs norm of displacement: MOVIE_VOLUME_TYPE == 7 + + use constants_solver + +#ifdef USE_HDF5 + use shared_parameters, only: OUTPUT_FILES,H5_COL + use specfem_par, only: it, scale_displ + use specfem_par_movie_hdf5 +#endif + + implicit none + + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CRUST_MANTLE) :: displ_crust_mantle + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_INNER_CORE) :: displ_inner_core + real(kind=CUSTOM_REAL), dimension(NGLOB_OUTER_CORE) :: displ_outer_core + + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE) :: ibool_inner_core + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE) :: ibool_outer_core + +#ifdef USE_HDF5 + + ! local parameters + integer :: ispec,iglob,i,j,k,ier + real(kind=CUSTOM_REAL), dimension(:), allocatable :: tmp_data + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + ! create group and datasets + if (myrank == 0) then + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + if (OUTPUT_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group('reg1_displ', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + endif + if (OUTPUT_OUTER_CORE) then + call h5_create_dataset_gen_in_group('reg2_displ', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) + endif + if (OUTPUT_INNER_CORE) then + call h5_create_dataset_gen_in_group('reg3_displ', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + endif + + call h5_close_group() + call h5_close_file() + endif + + call synchronize_all() + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + ! outputs norm of displacement + if (OUTPUT_CRUST_MANTLE) then + ! crust mantle + ! these binary arrays can be converted into mesh format using the utility ./bin/xcombine_vol_data + allocate(tmp_data(NGLOB_CRUST_MANTLE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_CRUST_MANTLE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_crust_mantle(i,j,k,ispec) + ! norm + tmp_data(iglob) = real(scale_displ,kind=CUSTOM_REAL) * sqrt( displ_crust_mantle(1,iglob)**2 & + + displ_crust_mantle(2,iglob)**2 & + + displ_crust_mantle(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call write_array3dspec_as_1d_hdf5('reg1_displ', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), & + tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + + deallocate(tmp_data) + endif + + if (OUTPUT_OUTER_CORE) then + ! outer core + allocate(tmp_data(NGLOB_OUTER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_OUTER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_outer_core(i,j,k,ispec) + ! norm + ! note: disp_outer_core is potential, this just outputs the potential, + ! not the actual displacement u = grad(rho * Chi) / rho + tmp_data(iglob) = abs(displ_outer_core(iglob)) + enddo + enddo + enddo + enddo + + call write_array3dspec_as_1d_hdf5('reg2_displ', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), & + tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) + + deallocate(tmp_data) + endif + + if (OUTPUT_INNER_CORE) then + ! inner core + allocate(tmp_data(NGLOB_INNER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_INNER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_inner_core(i,j,k,ispec) + ! norm + tmp_data(iglob) = real(scale_displ,kind=CUSTOM_REAL) * sqrt( displ_inner_core(1,iglob)**2 & + + displ_inner_core(2,iglob)**2 & + + displ_inner_core(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call write_array3dspec_as_1d_hdf5('reg3_displ', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), & + tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + + deallocate(tmp_data) + endif + + call h5_close_group() + call h5_close_file_p() + + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_displnorm_hdf5 + + +subroutine write_movie_volume_velnorm_hdf5(veloc_crust_mantle,veloc_inner_core,veloc_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + +! outputs norm of velocity: MOVIE_VOLUME_TYPE == 8 + + use constants_solver + +#ifdef USE_HDF5 + use shared_parameters, only: OUTPUT_FILES,H5_COL + use specfem_par, only: it, scale_veloc + use specfem_par_movie_hdf5 +#endif + + implicit none + + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CRUST_MANTLE) :: veloc_crust_mantle + real(kind=CUSTOM_REAL), dimension(NGLOB_OUTER_CORE) :: veloc_outer_core + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_INNER_CORE) :: veloc_inner_core + + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE) :: ibool_inner_core + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE) :: ibool_outer_core + +#ifdef USE_HDF5 + + ! local parameters + integer :: ispec,iglob,i,j,k,ier + real(kind=CUSTOM_REAL), dimension(:), allocatable :: tmp_data + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + ! create group and datasets + if (myrank == 0) then + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + if (OUTPUT_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group('reg1_veloc', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + endif + if (OUTPUT_OUTER_CORE) then + call h5_create_dataset_gen_in_group('reg2_veloc', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) + endif + if (OUTPUT_INNER_CORE) then + call h5_create_dataset_gen_in_group('reg3_veloc', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + endif + + call h5_close_group() + call h5_close_file() + endif + + call synchronize_all() + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + ! outputs norm of velocity + if (OUTPUT_CRUST_MANTLE) then + ! crust mantle + ! these binary arrays can be converted into mesh format using the utility ./bin/xcombine_vol_data + allocate(tmp_data(NGLOB_CRUST_MANTLE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_CRUST_MANTLE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_crust_mantle(i,j,k,ispec) + ! norm of velocity + tmp_data(iglob) = real(scale_veloc,kind=CUSTOM_REAL) * sqrt( veloc_crust_mantle(1,iglob)**2 & + + veloc_crust_mantle(2,iglob)**2 & + + veloc_crust_mantle(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call write_array3dspec_as_1d_hdf5('reg1_veloc', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), & + tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + + deallocate(tmp_data) + endif + + if (OUTPUT_OUTER_CORE) then + ! outer core + allocate(tmp_data(NGLOB_OUTER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_OUTER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_outer_core(i,j,k,ispec) + ! norm of velocity + ! note: this outputs only the first time derivative of the potential, + ! not the actual velocity v = grad(Chi_dot) + tmp_data(iglob) = abs(veloc_outer_core(iglob)) + enddo + enddo + enddo + enddo + + call write_array3dspec_as_1d_hdf5('reg2_veloc', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), & + tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) + + deallocate(tmp_data) + endif + + if (OUTPUT_INNER_CORE) then + ! inner core + allocate(tmp_data(NGLOB_INNER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_INNER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_inner_core(i,j,k,ispec) + ! norm of velocity + tmp_data(iglob) = real(scale_veloc,kind=CUSTOM_REAL) * sqrt( veloc_inner_core(1,iglob)**2 & + + veloc_inner_core(2,iglob)**2 & + + veloc_inner_core(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call write_array3dspec_as_1d_hdf5('reg3_veloc', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), & + tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + + deallocate(tmp_data) + endif + + call h5_close_group() + call h5_close_file_p() + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_velnorm_hdf5 + + + + subroutine write_movie_volume_accelnorm_hdf5(accel_crust_mantle,accel_inner_core,accel_outer_core, & + ibool_crust_mantle,ibool_inner_core,ibool_outer_core) + +! outputs norm of acceleration: MOVIE_VOLUME_TYPE == 1 + + use constants_solver + +#ifdef USE_HDF5 + use shared_parameters, only: OUTPUT_FILES,H5_COL + use specfem_par, only: it, scale_t_inv,scale_veloc + use specfem_par_movie_hdf5 +#endif + + implicit none + + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_CRUST_MANTLE) :: accel_crust_mantle + real(kind=CUSTOM_REAL), dimension(NDIM,NGLOB_INNER_CORE) :: accel_inner_core + real(kind=CUSTOM_REAL), dimension(NGLOB_OUTER_CORE) :: accel_outer_core + + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_CRUST_MANTLE) :: ibool_crust_mantle + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_INNER_CORE) :: ibool_inner_core + integer, dimension(NGLLX,NGLLY,NGLLZ,NSPEC_OUTER_CORE) :: ibool_outer_core + +#ifdef USE_HDF5 + ! local parameters + integer :: ispec,iglob,i,j,k,ier + real(kind=CUSTOM_REAL), dimension(:), allocatable :: tmp_data + real(kind=CUSTOM_REAL) :: scale_accel + + ! dimensionalized scaling + scale_accel = real(scale_veloc * scale_t_inv,kind=CUSTOM_REAL) + + ! initialize h5 file for volume movie + call world_get_comm(comm) + call world_get_info_null(info) + call h5_initialize() + call h5_set_mpi_info(comm, info, myrank, NPROCTOT_VAL) + + file_name = trim(OUTPUT_FILES) // '/movie_volume.h5' + group_name = 'it_' // trim(i2c(it)) + + ! create group and datasets + if (myrank == 0) then + call h5_open_file(file_name) + call h5_open_or_create_group(group_name) + + if (OUTPUT_CRUST_MANTLE) then + call h5_create_dataset_gen_in_group('reg1_accel', (/npoints_vol_mov_all_proc_cm/), 1, CUSTOM_REAL) + endif + if (OUTPUT_OUTER_CORE) then + call h5_create_dataset_gen_in_group('reg2_accel', (/npoints_vol_mov_all_proc_oc/), 1, CUSTOM_REAL) + endif + if (OUTPUT_INNER_CORE) then + call h5_create_dataset_gen_in_group('reg3_accel', (/npoints_vol_mov_all_proc_ic/), 1, CUSTOM_REAL) + endif + + call h5_close_group() + call h5_close_file() + endif + + call synchronize_all() + + ! write the data + call h5_open_file_p_collect(file_name) + call h5_open_group(group_name) + + ! outputs norm of acceleration + if (OUTPUT_CRUST_MANTLE) then + ! acceleration + ! these binary arrays can be converted into mesh format using the utility ./bin/xcombine_vol_data + allocate(tmp_data(NGLOB_CRUST_MANTLE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_CRUST_MANTLE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_crust_mantle(i,j,k,ispec) + ! norm + tmp_data(iglob) = scale_accel * sqrt( accel_crust_mantle(1,iglob)**2 & + + accel_crust_mantle(2,iglob)**2 & + + accel_crust_mantle(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call write_array3dspec_as_1d_hdf5('reg1_accel', offset_nspec_vol_cm(myrank-1), offset_poin_vol_cm(myrank-1), & + tmp_data, sum(offset_poin_vol_cm(0:myrank-1)), ibool_crust_mantle) + + deallocate(tmp_data) + endif + + if (OUTPUT_OUTER_CORE) then + ! outer core acceleration + allocate(tmp_data(NGLOB_OUTER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_OUTER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_outer_core(i,j,k,ispec) + ! norm + ! note: this outputs only the second time derivative of the potential, + ! not the actual acceleration or pressure p = - rho * Chi_dot_dot + tmp_data(iglob) = abs(accel_outer_core(iglob)) + enddo + enddo + enddo + enddo + + call write_array3dspec_as_1d_hdf5('reg2_accel', offset_nspec_vol_oc(myrank-1), offset_poin_vol_oc(myrank-1), & + tmp_data, sum(offset_poin_vol_oc(0:myrank-1)), ibool_outer_core) + + deallocate(tmp_data) + endif + + if (OUTPUT_INNER_CORE) then + ! inner core + allocate(tmp_data(NGLOB_INNER_CORE),stat=ier) + if (ier /= 0 ) call exit_MPI(myrank,'Error allocating temporary array tmp_data') + + do ispec = 1, NSPEC_INNER_CORE + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_inner_core(i,j,k,ispec) + ! norm of acceleration + tmp_data(iglob) = scale_accel * sqrt( accel_inner_core(1,iglob)**2 & + + accel_inner_core(2,iglob)**2 & + + accel_inner_core(3,iglob)**2 ) + enddo + enddo + enddo + enddo + + call write_array3dspec_as_1d_hdf5('reg3_accel', offset_nspec_vol_ic(myrank-1), offset_poin_vol_ic(myrank-1), & + tmp_data, sum(offset_poin_vol_ic(0:myrank-1)), ibool_inner_core) + + deallocate(tmp_data) + endif + +#else + + print*, 'Error: HDF5 is not enabled in this version of the code.' + print*, 'Please recompile with the HDF5 option enabled with --with-hdf5' + stop + +#endif + + end subroutine write_movie_volume_accelnorm_hdf5 + +! +!------------------------------------------------------------------------------------------------- +! + +#ifdef USE_HDF5 + subroutine get_conn_for_movie(elm_conn,offset,iNIT,nelems_3dmovie,npoints_3dmovie, & + nelems_in_this_region, & + num_ibool_3dmovie,mask_ibool_3dmovie,ibool_of_the_section) + + use specfem_par + !use specfem_par_crustmantle, only: ibool_crust_mantle + use constants_solver + + + implicit none + + integer, intent(in) :: offset ! node id offset (starting global element id of each proc) + integer, intent(in) :: iNIT + integer, intent(in) :: nelems_3dmovie, npoints_3dmovie, nelems_in_this_region + integer, dimension(npoints_3dmovie),intent(in) :: num_ibool_3dmovie + logical, dimension(npoints_3dmovie),intent(in) :: mask_ibool_3dmovie + integer, dimension(NGLLX,NGLLY,NGLLZ,nelems_in_this_region),intent(in) :: ibool_of_the_section + integer, dimension(9,nelems_3dmovie), intent(out) :: elm_conn + ! local parameters + integer :: ispec,ii + integer,parameter :: cell_type = 9 + integer :: iglob1,iglob2,iglob3,iglob4,iglob5,iglob6,iglob7,iglob8 + integer :: n1,n2,n3,n4,n5,n6,n7,n8 + integer :: i,j,k + integer :: ispecele, iglob_center + + ispecele = 0 + do ispec = 1, nelems_in_this_region + + ! checks center of element for movie flag + iglob_center = ibool_of_the_section((NGLLX+1)/2,(NGLLY+1)/2,(NGLLZ+1)/2,ispec) + + ! checks if movie element + if (mask_ibool_3dmovie(iglob_center)) then + + do k = 1,NGLLZ-1,iNIT + do j = 1,NGLLY-1,iNIT + do i = 1,NGLLX-1,iNIT + ! this element is in the movie region + ispecele = ispecele+1 + + ! defines corners of a vtk element + iglob1 = ibool_of_the_section(i,j,k,ispec) + iglob2 = ibool_of_the_section(i+iNIT,j,k,ispec) + iglob3 = ibool_of_the_section(i+iNIT,j+iNIT,k,ispec) + iglob4 = ibool_of_the_section(i,j+iNIT,k,ispec) + iglob5 = ibool_of_the_section(i,j,k+iNIT,ispec) + iglob6 = ibool_of_the_section(i+iNIT,j,k+iNIT,ispec) + iglob7 = ibool_of_the_section(i+iNIT,j+iNIT,k+iNIT,ispec) + iglob8 = ibool_of_the_section(i,j+iNIT,k+iNIT,ispec) + + ! vtk indexing starts at 0 -> adds minus 1 + n1 = num_ibool_3dmovie(iglob1)-1 + n2 = num_ibool_3dmovie(iglob2)-1 + n3 = num_ibool_3dmovie(iglob3)-1 + n4 = num_ibool_3dmovie(iglob4)-1 + n5 = num_ibool_3dmovie(iglob5)-1 + n6 = num_ibool_3dmovie(iglob6)-1 + n7 = num_ibool_3dmovie(iglob7)-1 + n8 = num_ibool_3dmovie(iglob8)-1 + + elm_conn(1, ispecele) = cell_type + elm_conn(2, ispecele) = n1 + offset ! node id starts 0 in xdmf rule + elm_conn(3, ispecele) = n2 + offset + elm_conn(4, ispecele) = n3 + offset + elm_conn(5, ispecele) = n4 + offset + elm_conn(6, ispecele) = n5 + offset + elm_conn(7, ispecele) = n6 + offset + elm_conn(8, ispecele) = n7 + offset + elm_conn(9, ispecele) = n8 + offset + + ! checks indices + if (n1 < 0 .or. n2 < 0 .or. n3 < 0 .or. n4 < 0 .or. n5 < 0 .or. n6 < 0 .or. n7 < 0 .or. n8 < 0) then + print *,'Error: movie element ',ispec,ispecele,'has invalid node index:',n1,n2,n3,n4,n5,n6,n7,n8 + call exit_mpi(myrank,'Error invalid movie element node index') + endif + + enddo !i + enddo !j + enddo !k + endif + + enddo !ispec + + ! check if ispecele is consistent with nelems_3dmovie + if (ispecele /= nelems_3dmovie) then + print *,'Error: number of movie elements is not consistent with nelems_3dmovie' + print *,'ispecele = ',ispecele,'nelems_3dmovie = ',nelems_3dmovie + call exit_mpi(myrank,'Error number of movie elements is not consistent with nelems_3dmovie') + endif + + end subroutine get_conn_for_movie + +#endif + +! +!------------------------------------------------------------------------------------------------- +! +#ifdef USE_HDF5 + + subroutine elm2node_base(array_3dspec, array_1dmovie, nelms, npoints, ibool_of_the_section) + + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + integer, intent(in) :: nelms, npoints + integer, dimension(NGLLX,NGLLY,NGLLZ,nelms), intent(in) :: ibool_of_the_section + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nelms), intent(in) :: array_3dspec + real(kind=CUSTOM_REAL), dimension(npoints), intent(inout) :: array_1dmovie + + ! local parameters + integer :: i,j,k,ispec,iglob + + ! convert 3d array to 1d array + do ispec = 1, nelms + + do k = 1, NGLLZ + do j = 1, NGLLY + do i = 1, NGLLX + iglob = ibool_of_the_section(i,j,k,ispec) + array_1dmovie(iglob) = array_3dspec(i,j,k,ispec) + enddo + enddo + enddo + enddo + + end subroutine elm2node_base + +#endif + + +#ifdef USE_HDF5 + + subroutine write_array3dspec_as_1d_hdf5(dset_name, nelms, npoints, array_3dspec, offset1d, ibool_of_the_section) + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + character(len=*), intent(in) :: dset_name + integer, intent(in) :: nelms, npoints + real(kind=CUSTOM_REAL), dimension(NGLLX,NGLLY,NGLLZ,nelms), intent(in) :: array_3dspec + integer, intent(in) :: offset1d + integer, dimension(NGLLX,NGLLY,NGLLZ,nelms), intent(in) :: ibool_of_the_section + + ! local parameters + integer :: i,j,k,ii + real(kind=CUSTOM_REAL), dimension(npoints) :: array_1dmovie + + ! convert 3d array to 1d array + call elm2node_base(array_3dspec, array_1dmovie, nelms, npoints, ibool_of_the_section) + + ! write 1d array to hdf5 + call h5_write_dataset_collect_hyperslab_in_group(dset_name, array_1dmovie, (/offset1d/), H5_COL) + + end subroutine write_array3dspec_as_1d_hdf5 + +#endif + +! +!------------------------------------------------------------------------------------------------- +! +#ifdef USE_HDF5 + + subroutine write_xdmf_vol_hdf5_one_data(fname_h5, attr_name, dset_name, len_data, target_unit, it_str, value_on_node) + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + character(len=*), intent(in) :: fname_h5 + character(len=*), intent(in) :: attr_name + character(len=*), intent(in) :: dset_name + integer, intent(in) :: len_data + integer, intent(in) :: target_unit + character(len=*), intent(in) :: it_str + logical, intent(in) :: value_on_node ! values are defined on nodes or elements + + character(len=20) :: center_str + + if (value_on_node) then + center_str = 'Node' ! len data should be the number of nodes + else + center_str = 'Cell' ! len data should be the number of elements + endif + + ! write a header for single data + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) ' '//trim(fname_h5)//':/it_'//trim(it_str)//'/'//trim(dset_name) + write(target_unit,*) '' + write(target_unit,*) '' + + end subroutine write_xdmf_vol_hdf5_one_data + + + subroutine write_xdmf_vol_hdf5_header(nelems, nglobs, fname_h5_data_vol_xdmf, target_unit, region_flag) + + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + integer, intent(in) :: nelems, nglobs + character(len=*), intent(in) :: fname_h5_data_vol_xdmf + integer, intent(in) :: target_unit + integer, intent(in) :: region_flag ! 1: crust mantle, 2: outer core, 3: inner core + + character(len=64) :: nelm_str, nglo_str, elemconn_str, x_str, y_str, z_str + + if (region_flag == 1) then + elemconn_str = 'elm_conn' + x_str = 'x' + y_str = 'y' + z_str = 'z' + else if (region_flag == 2) then + elemconn_str = 'elm_conn_cm' + x_str = 'x_cm' + y_str = 'y_cm' + z_str = 'z_cm' + else if (region_flag == 3) then + elemconn_str = 'elm_conn_oc' + x_str = 'x_oc' + y_str = 'y_oc' + z_str = 'z_oc' + else if (region_flag == 4) then + elemconn_str = 'elm_conn_ic' + x_str = 'x_ic' + y_str = 'y_ic' + z_str = 'z_ic' + else + print *,'Error: invalid region_flag in write_xdmf_vol_hdf5_header' + call exit_mpi(myrank,'Error invalid region_flag in write_xdmf_vol_hdf5_header') + endif + + ! convert integer to string + nelm_str = i2c(nelems) + nglo_str = i2c(nglobs) + + ! definition of topology and geometry + ! refer only control nodes (8 or 27) as a coarse output + ! data array need to be extracted from full data array on GLL points + write(target_unit,'(a)') '' + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) '' + + ! loop for writing information of mesh partitions + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/'//trim(elemconn_str) + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/'//trim(x_str) + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/'//trim(y_str) + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) ' '//trim(fname_h5_data_vol_xdmf)//':/mesh/'//trim(z_str) + write(target_unit,*) '' + write(target_unit,*) '' + + write(target_unit,*) '' + + end subroutine write_xdmf_vol_hdf5_header + + subroutine write_xdmf_vol_hdf5_footer(target_unit) + + implicit none + + integer, intent(in) :: target_unit + + ! file finish + write(target_unit,*) '' + write(target_unit,*) '' + write(target_unit,*) '' + + end subroutine write_xdmf_vol_hdf5_footer + + + subroutine write_xdmf_vol_hdf5(npoints_3dmovie, nelems_3dmovie, & + npoints_3dmovie_cm, nelems_3dmovie_cm, & + npoints_3dmovie_oc, nelems_3dmovie_oc, & + npoints_3dmovie_ic, nelems_3dmovie_ic) + + use specfem_par + use specfem_par_movie_hdf5 + + implicit none + + integer, intent(in) :: npoints_3dmovie, nelems_3dmovie + integer, intent(in) :: npoints_3dmovie_cm, nelems_3dmovie_cm + integer, intent(in) :: npoints_3dmovie_oc, nelems_3dmovie_oc + integer, intent(in) :: npoints_3dmovie_ic, nelems_3dmovie_ic + + ! local parameters + integer :: i, ii + character(len=20) :: it_str, movie_prefix + character(len=20) :: nelm_str, nglo_str + character(len=20) :: nelm_str_cm, nglo_str_cm + character(len=20) :: nelm_str_oc, nglo_str_oc + character(len=20) :: nelm_str_ic, nglo_str_ic + character(len=MAX_STRING_LEN) :: fname_xdmf_vol, fname_xdmf_vol_oc, fname_xdmf_vol_ic + character(len=MAX_STRING_LEN) :: fname_h5_data_vol_xdmf + + ! checks if anything do, only main process writes out xdmf file + if (myrank /= 0) return + + ! + ! write out the crust mantle xdmf file (for strain and vector) + ! + if (output_sv) then + + fname_xdmf_vol = trim(OUTPUT_FILES) // "/movie_volume.xmf" + fname_h5_data_vol_xdmf = "./movie_volume.h5" ! relative to movie_volume.xmf file + + ! open xdmf file + open(unit=xdmf_vol, file=trim(fname_xdmf_vol), recl=256) + + call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc, npoints_vol_mov_all_proc, fname_h5_data_vol_xdmf, xdmf_vol, 1) + + + do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) + + ii = i*NTSTEP_BETWEEN_FRAMES + !write(it_str, "(i6.6)") ii + it_str = i2c(ii) + + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + + enddo + + call write_xdmf_vol_hdf5_footer(xdmf_vol) + + ! close xdmf file + close(xdmf_vol) + + endif ! output_sv + + ! + ! write out the crust and mantle xdmf file (for strain and vector) + ! + if (output_cm) then + + fname_xdmf_vol = trim(OUTPUT_FILES) // "/movie_volume_cm.xmf" + fname_h5_data_vol_xdmf = "./movie_volume.h5" ! relative to movie_volume_cm.xmf file + + ! open xdmf file + open(unit=xdmf_vol, file=trim(fname_xdmf_vol), recl=256) + + call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_cm_conn, npoints_vol_mov_all_proc_cm, & + fname_h5_data_vol_xdmf, xdmf_vol, 2) + + do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) + + ii = i*NTSTEP_BETWEEN_FRAMES + !write(it_str, "(i6.6)") ii + it_str = i2c(ii) + + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + + enddo + + call write_xdmf_vol_hdf5_footer(xdmf_vol) + + ! close xdmf file + close(xdmf_vol) + + endif ! output_cm + + ! + ! write out the outer core xdmf file + ! + if (output_oc) then + + fname_xdmf_vol_oc = trim(OUTPUT_FILES) // "/movie_volume_oc.xmf" + fname_h5_data_vol_xdmf = "./movie_volume.h5" ! relative to movie_volume_oc.xmf file + + ! open xdmf file + open(unit=xdmf_vol, file=trim(fname_xdmf_vol_oc), recl=256) + + call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_oc_conn, npoints_vol_mov_all_proc_oc, & + fname_h5_data_vol_xdmf, xdmf_vol, 3) + + do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) + + ii = i*NTSTEP_BETWEEN_FRAMES + !write(it_str, "(i6.6)") ii + it_str = i2c(ii) + + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + + enddo + + call write_xdmf_vol_hdf5_footer(xdmf_vol) + + ! close xdmf file + close(xdmf_vol) + + endif ! output_oc + + ! + ! write out the inner core xdmf file + ! + if (output_ic) then + + fname_xdmf_vol_ic = trim(OUTPUT_FILES) // "/movie_volume_ic.xmf" + fname_h5_data_vol_xdmf = "./movie_volume.h5" ! relative to movie_volume_ic.xmf file + + ! open xdmf file + open(unit=xdmf_vol, file=trim(fname_xdmf_vol_ic), recl=256) + + call write_xdmf_vol_hdf5_header(nspec_vol_mov_all_proc_ic_conn, npoints_vol_mov_all_proc_ic, & + fname_h5_data_vol_xdmf, xdmf_vol, 4) + + do i = 1, int(NSTEP/NTSTEP_BETWEEN_FRAMES) + + ii = i*NTSTEP_BETWEEN_FRAMES + !write(it_str, "(i6.6)") ii + it_str = i2c(ii) + + write(xdmf_vol,*) '' + write(xdmf_vol,*) '' + + enddo + + call write_xdmf_vol_hdf5_footer(xdmf_vol) + + ! close xdmf file + close(xdmf_vol) + + endif ! output_ic + + end subroutine write_xdmf_vol_hdf5 + +#endif diff --git a/src/specfem3D/write_output_HDF5.F90 b/src/specfem3D/write_output_HDF5.F90 new file mode 100644 index 000000000..c7ac9311c --- /dev/null +++ b/src/specfem3D/write_output_HDF5.F90 @@ -0,0 +1,258 @@ +!===================================================================== +! +! S p e c f e m 3 D G l o b e +! ---------------------------- +! +! Main historical authors: Dimitri Komatitsch and Jeroen Tromp +! Princeton University, USA +! and CNRS / University of Marseille, France +! (there are currently many more authors!) +! (c) Princeton University and CNRS / University of Marseille, April 2014 +! +! This program is free software; you can redistribute it and/or modify +! it under the terms of the GNU General Public License as published by +! the Free Software Foundation; either version 3 of the License, or +! (at your option) any later version. +! +! This program is distributed in the hope that it will be useful, +! but WITHOUT ANY WARRANTY; without even the implied warranty of +! MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +! GNU General Public License for more details. +! +! You should have received a copy of the GNU General Public License along +! with this program; if not, write to the Free Software Foundation, Inc., +! 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +! +!===================================================================== + + + subroutine split_string(input_string, delimiter, part1, part2) + implicit none + character(len=*), intent(in) :: input_string + character(len=*), intent(in) :: delimiter + character(len=*), intent(out) :: part1 + character(len=*), intent(out) :: part2 + integer :: delim_pos + + ! Find the position of the delimiter + delim_pos = index(input_string, delimiter) + + ! Split the string at the delimiter + if (delim_pos > 0) then + part1 = input_string(1:delim_pos-1) + part2 = input_string(delim_pos+1:) + else + part1 = input_string + part2 = '' + end if + end subroutine split_string + +#ifdef USE_HDF5 + + subroutine write_hdf5_seismogram_init(nlength_total_seismogram) + + use specfem_par, only: CUSTOM_REAL, MAX_LENGTH_NETWORK_NAME, MAX_LENGTH_STATION_NAME, & + MAX_STRING_LEN, t0, DT, IMAIN, IOUT, myrank, & + NSTEP, nrec, hdf5_seismo_fname, & + OUTPUT_FILES, SIMULATION_TYPE, WRITE_SEISMOGRAMS_BY_MAIN + + use shared_parameters, only: IO_compute_task, NTSTEP_BETWEEN_OUTPUT_SAMPLE + + use manager_hdf5 + + implicit none + + ! local parameters + integer :: i,irec,ier + integer, intent(in) :: nlength_total_seismogram + integer :: nrec_total + real(kind=CUSTOM_REAL), dimension(:), allocatable :: time_array + real(kind=CUSTOM_REAL), dimension(:), allocatable :: rec_dists ! store the distances + character(len=MAX_LENGTH_STATION_NAME), dimension(:), allocatable :: stations + character(len=MAX_LENGTH_NETWORK_NAME), dimension(:), allocatable :: networks + integer, parameter :: n_header_lines = 3 + character(len=MAX_STRING_LEN) :: tmpstr, dummy, line + + ! only main process writes out + if (myrank /= 0) return + + ! create the file and all the datasets for the seismograms + ! set the name of the file + hdf5_seismo_fname = trim(OUTPUT_FILES)//'/seismograms.h5' + + ! user output + if (myrank == 0 .and. IO_compute_task) then + write(IMAIN,*) 'Creating seismograms in HDF5 file format' + if (WRITE_SEISMOGRAMS_BY_MAIN) then + write(IMAIN,*) ' writing waveforms by main...' + else + write(IMAIN,*) ' writing waveforms in parallel...' + endif + write(IMAIN,*) ' seismogram file: ',trim(hdf5_seismo_fname) + call flush_IMAIN() + endif + + ! initialize the HDF5 manager + call h5_initialize() + + ! create the file + call h5_create_file(hdf5_seismo_fname) + + ! create time dataset it = 1 ~ NTSTEP + allocate(time_array(nlength_total_seismogram),stat=ier) + if (ier /= 0) stop 'Error: write_hdf5_seismogram_init: time_array allocation failed' + time_array(:) = 0.0_CUSTOM_REAL + + do i = 1,nlength_total_seismogram + if (SIMULATION_TYPE == 1) then + ! forward simulation + time_array(i) = real( dble((i-1)*NTSTEP_BETWEEN_OUTPUT_SAMPLE) * DT - t0, kind=CUSTOM_REAL) + else + ! adjoint simulation: backward/reconstructed wavefields + time_array(i) = real( dble((NSTEP-i)*NTSTEP_BETWEEN_OUTPUT_SAMPLE) * DT - t0, kind=CUSTOM_REAL) + end if + end do + + ! write the time dataset + call h5_write_dataset_no_group('time',time_array) + ! free the memory + deallocate(time_array) + + ! read output_list_stations.txt generated at locate_receivers.f90:613 here to write in the h5 file. + allocate(stations(nrec), & + networks(nrec), & + rec_dists(nrec),stat=ier) + if (ier /= 0) stop 'Error: write_hdf5_seismogram_init: stations allocation failed' + rec_dists(:) = 0.0_CUSTOM_REAL + + open(unit=IOUT,file=trim(OUTPUT_FILES)//'/output_list_stations.txt', & + status='unknown',action='read',iostat=ier) + if (ier /= 0) then + stop 'Error: write_hdf5_seismogram_init: error opening output_list_stations.txt' + end if + + ! skip the header (3 lines) + do i = 1,n_header_lines + read(IOUT,*) + end do + + ! read the station information + ! each line includes: network.station dummy dummy dist dummy + do irec = 1,nrec + ! read network.station and separate them by '.' + !read(IOUT) tmpstr, dummy, dummy, rec_dists(irec), dummy + !read(IOUT, '(A, 1X, A, 1X, A, 3X, F9.7, 6X, A)') tmpstr, dummy, dummy, rec_dists(irec), dummy + read(IOUT, '(A)', iostat=ier) line + if (ier /= 0) stop 'Error: write_hdf5_seismogram_init: error reading output_list_stations.txt' + read(line, *) tmpstr, dummy, dummy, rec_dists(irec), dummy + call split_string(trim(tmpstr), '.', networks(irec), stations(irec)) + end do + + close(IOUT) + + ! write the station information + call h5_write_dataset_no_group('stations',stations) + call h5_write_dataset_no_group('networks',networks) + call h5_write_dataset_no_group('dists',rec_dists) + + ! free the memory + deallocate(stations,networks,rec_dists) + + ! close the file + call h5_close_file() + + end subroutine write_hdf5_seismogram_init + + +#endif + + subroutine write_output_hdf5(seismogram_tmp_in, irec_local, irec, chn, iorientation) + + use specfem_par, only: & + nlength_seismogram, & + CUSTOM_REAL + +#ifdef USE_HDF5 + use specfem_par, only: & + myrank, seismo_current, nrec, & + ROTATE_SEISMOGRAMS_RT, NTSTEP_BETWEEN_OUTPUT_SAMPLE, & + WRITE_SEISMOGRAMS_BY_MAIN, hdf5_seismo_fname + use shared_parameters, only: & + NSTEP, OUTPUT_SEISMOS_HDF5 + use manager_hdf5 +#endif + + implicit none + + ! input/output variables + character(len=4),intent(in) :: chn + integer,intent(in) :: irec_local, irec, iorientation + real(kind=CUSTOM_REAL),dimension(5,nlength_seismogram),intent(in) :: seismogram_tmp_in + +#ifdef USE_HDF5 + + real(kind=CUSTOM_REAL), dimension(nlength_seismogram,1) :: seismogram_tmp + logical, save :: is_initialized = .false. + integer :: i, ier, nlength_total_seismogram + logical :: if_dataset_exists + + + ! check if anything to do + if (.not. OUTPUT_SEISMOS_HDF5) return + + ! safety check + if (.not. WRITE_SEISMOGRAMS_BY_MAIN) & + stop 'Error: WRITE_SEIMSMOGRAMS_BY_MAIN must be true to use HDF5 seismogram output' + + ! total length of the seismogram + nlength_total_seismogram = NSTEP / NTSTEP_BETWEEN_OUTPUT_SAMPLE + print*, 'nlength_total_seismogram = ', nlength_total_seismogram + print*, 'NSTEP = ', NSTEP + print*, 'NTSTEP_BETWEEN_OUTPUT_SAMPLE = ', NTSTEP_BETWEEN_OUTPUT_SAMPLE + print*, 'nrec = ', nrec + print*, 'irec = ', irec + print*, 'seismo_current = ', seismo_current + print*, 'shape(seismogram_tmp) = ', shape(seismogram_tmp) + print*, 'shape(seismogram_tmp(iorientation,1:seismo_current)) = ', shape(seismogram_tmp(iorientation,1:seismo_current)) + ! convert array with shape (seimo_current) to (nlength_total_seismogram, 1) + do i = 1, seismo_current + seismogram_tmp(i,1) = seismogram_tmp_in(iorientation,i) + end do + + + + ! initialize + if (.not. is_initialized) then + call write_hdf5_seismogram_init(nlength_total_seismogram) + is_initialized = .true. + endif + + ! only main process writes out + if (myrank /= 0) return + + ! write the seismograms + call h5_open_file(hdf5_seismo_fname) + + ! check if the target dataset components are already created + call h5_check_dataset_exists(trim(chn),if_dataset_exists) + + ! if the dataset does not exist, create it + if (.not. if_dataset_exists) then + call h5_create_dataset_gen(trim(chn),(/nlength_total_seismogram, nrec/), 2, CUSTOM_REAL) + endif + + ! write the seismogram + call h5_write_dataset_collect_hyperslab(trim(chn), seismogram_tmp(1:seismo_current, :), (/0,irec-1/), .false.) + + ! close the file + call h5_close_file() + +#else + + write(*,*) 'Error: HDF5 support not enabled in this version of Specfem3D_Globe' + write(*,*) 'Please recompile with the --with-hdf5 option' + stop + +#endif + + end subroutine write_output_hdf5 \ No newline at end of file diff --git a/src/specfem3D/write_seismograms.f90 b/src/specfem3D/write_seismograms.f90 index 1472388b0..bd2542eb0 100644 --- a/src/specfem3D/write_seismograms.f90 +++ b/src/specfem3D/write_seismograms.f90 @@ -51,6 +51,9 @@ subroutine write_seismograms() implicit none + ! HDF5 is not implemented because ASDF is already available + ! TODO ADD IO_SERVER for seismograms (low priority because it's not I/O intensive) + ! local parameters ! timing double precision, external :: wtime @@ -249,6 +252,7 @@ subroutine write_seismograms_to_file(istore) OUTPUT_SEISMOS_SAC_ALPHANUM,OUTPUT_SEISMOS_SAC_BINARY, & OUTPUT_SEISMOS_ASDF, & OUTPUT_SEISMOS_3D_ARRAY, & + OUTPUT_SEISMOS_HDF5, & SAVE_ALL_SEISMOS_IN_ONE_FILE,USE_BINARY_FOR_LARGE_FILE, & OUTPUT_FILES, & WRITE_SEISMOGRAMS_BY_MAIN, & @@ -318,7 +322,7 @@ subroutine write_seismograms_to_file(istore) ! write this seismogram ! note: ASDF data structure is given in module ! stores all traces into ASDF container in case - call write_one_seismogram(one_seismogram,irec,irec_local,.true.,component,istore) + call write_one_seismogram(one_seismogram,irec,irec_local,1,component,istore) enddo ! writes ASDF file output @@ -387,12 +391,18 @@ subroutine write_seismograms_to_file(istore) endif endif - ! ASCII / SAC format - if (OUTPUT_SEISMOS_ASCII_TEXT .or. OUTPUT_SEISMOS_SAC_ALPHANUM .or. OUTPUT_SEISMOS_SAC_BINARY) then + ! ASCII / SAC / HDF5 format + if ( OUTPUT_SEISMOS_ASCII_TEXT & + .or. OUTPUT_SEISMOS_SAC_ALPHANUM & + .or. OUTPUT_SEISMOS_SAC_BINARY & + .or. OUTPUT_SEISMOS_HDF5) then ! write out seismograms: all processes write their local seismograms themselves if (.not. WRITE_SEISMOGRAMS_BY_MAIN) then + ! HDF5 is not supported + if (OUTPUT_SEISMOS_HDF5) call exit_MPI(myrank,'WRITE_SEISMOGRAMS_BY_MAIN must be true for HDF5 format') + ! all the processes write their local seismograms themselves if (SAVE_ALL_SEISMOS_IN_ONE_FILE .and. OUTPUT_SEISMOS_ASCII_TEXT) then write(sisname,'(A,I5.5)') '/all_seismograms_'//trim(component)//'_node_',myrank @@ -425,7 +435,7 @@ subroutine write_seismograms_to_file(istore) ! write this seismogram ! note: ASDF data structure is given in module ! stores all traces into ASDF container in case - call write_one_seismogram(one_seismogram,irec,irec_local,.false.,component,istore) + call write_one_seismogram(one_seismogram,irec,irec_local,0,component,istore) enddo ! create one large file instead of one small file per station to avoid file system overload @@ -503,7 +513,13 @@ subroutine write_seismograms_to_file(istore) endif ! write this seismogram - call write_one_seismogram(one_seismogram,irec,irec_local,.false.,component,istore) + if (.not. OUTPUT_SEISMOS_HDF5) then + ! ASCII or SAC format + call write_one_seismogram(one_seismogram,irec,irec_local,0,component,istore) + else + ! HDF5 format + call write_one_seismogram(one_seismogram,irec,irec_local,2,component,istore) + endif ! counts seismos written total_seismos = total_seismos + 1 @@ -629,7 +645,7 @@ end subroutine write_seismograms_to_file !------------------------------------------------------------------------------------------------- ! - subroutine write_one_seismogram(one_seismogram,irec,irec_local,is_for_asdf,component,istore) + subroutine write_one_seismogram(one_seismogram,irec,irec_local,ftype_flag,component,istore) use constants_solver, only: MAX_STRING_LEN,CUSTOM_REAL,NDIM,DEGREES_TO_RADIANS, & MAX_LENGTH_STATION_NAME,MAX_LENGTH_NETWORK_NAME @@ -649,7 +665,8 @@ subroutine write_one_seismogram(one_seismogram,irec,irec_local,is_for_asdf,compo implicit none integer,intent(in) :: irec,irec_local - logical,intent(in) :: is_for_asdf + integer,intent(in) :: ftype_flag + logical :: is_for_asdf,is_for_hdf5 real(kind=CUSTOM_REAL), dimension(NDIM,nlength_seismogram),intent(in) :: one_seismogram integer,intent(in) :: istore @@ -674,6 +691,23 @@ subroutine write_one_seismogram(one_seismogram,irec,irec_local,is_for_asdf,compo ! initializes seismogram_tmp(:,:) = 0.0_CUSTOM_REAL + ! check file type + if (ftype_flag == 0) then + ! ascii + is_for_asdf = .false. + is_for_hdf5 = .false. + else if (ftype_flag == 1) then + ! asdf + is_for_asdf = .true. + is_for_hdf5 = .false. + else if (ftype_flag == 2) then + ! hdf5 + is_for_asdf = .false. + is_for_hdf5 = .true. + else + call exit_MPI(myrank,'wrong file type flag') + endif + ! get band code call band_instrument_code(DT,bic) @@ -801,6 +835,9 @@ subroutine write_one_seismogram(one_seismogram,irec,irec_local,is_for_asdf,compo if (OUTPUT_SEISMOS_ASDF) then call store_asdf_data(seismogram_tmp,irec_local,irec,chn,iorientation) endif + else if (is_for_hdf5) then + ! TODO: add HDF5 format + call write_output_hdf5(seismogram_tmp,irec_local,irec,chn,iorientation) else ! SAC output format if (OUTPUT_SEISMOS_SAC_ALPHANUM .or. OUTPUT_SEISMOS_SAC_BINARY ) then