-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathfunctions_download_dwd.sh
110 lines (110 loc) · 5.39 KB
/
functions_download_dwd.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
#Given a variable name and year-month-day-run as environmental variables download and merges the variable
################################################
listurls() {
filename="$1"
url="$2"
wget -qO- $url | grep -Eoi '<a [^>]+>' | \
grep -Eo 'href="[^\"]+"' | \
grep -Eo $filename | \
xargs -I {} echo "$url"{}
}
export -f listurls
#
get_and_extract_one() {
url="$1"
file=`basename $url | sed 's/\.bz2//g'`
if [ ! -f "$file" ]; then
wget -t 2 -q -O - "$url" | bzip2 -dc > "$file"
fi
}
export -f get_and_extract_one
##############################################
download_merge_2d_variable_icon_globe()
{
filename="icon_global_icosahedral_single-level_${year}${month}${day}${run}_*_${1}.grib2"
filename_grep="icon_global_icosahedral_single-level_${year}${month}${day}${run}_(${hours_download})_${1}.grib2.bz2"
url="https://opendata.dwd.de/weather/nwp/icon/grib/${run}/${1,,}/"
echo "folder: ${url}"
echo "files: ${filename}"
if [ ! -f "${1}_${year}${month}${day}${run}_global.nc" ]; then
listurls $filename_grep $url | parallel -j 10 get_and_extract_one {}
find ${filename} -empty -type f -delete # Remove empty files
cdo -f nc copy -mergetime ${filename} ${1}_${year}${month}${day}${run}_global.nc
rm ${filename}
fi
if [ "$REMAP" = true ]; then
mkdir -p ${MODEL_DATA_FOLDER}remap
echo "Remapping the output as requested"
cdo -P 8 remap,"${HOME_FOLDER}/grids/target_grid_world_0150.txt","${HOME_FOLDER}/grids/weights_icogl2world_0150.nc" \
-setgrid,"${HOME_FOLDER}/grids/icon_grid_0026_R03B07_G.nc" "${1}_${year}${month}${day}${run}_global.nc" "remap/${1}_${year}${month}${day}${run}_global.nc"
rm "${1}_${year}${month}${day}${run}_global.nc"
fi
}
export -f download_merge_2d_variable_icon_globe
################################################
download_merge_3d_variable_icon_globe()
{
filename="icon_global_icosahedral_pressure-level_${year}${month}${day}${run}_*_${1}.grib2"
filename_grep="icon_global_icosahedral_pressure-level_${year}${month}${day}${run}_(${hours_download})_(1000|850|700|500|250|50)_${1}.grib2.bz2"
url="https://opendata.dwd.de/weather/nwp/icon/grib/${run}/${1,,}/"
if [ ! -f "${1}_${year}${month}${day}${run}_global.nc" ]; then
listurls $filename_grep $url | parallel -j 10 get_and_extract_one {}
find ${filename} -empty -type f -delete # Remove empty files
cdo merge ${filename} ${1}_${year}${month}${day}${run}_global.grib2
rm ${filename}
cdo -f nc copy ${1}_${year}${month}${day}${run}_global.grib2 ${1}_${year}${month}${day}${run}_global.nc
rm ${1}_${year}${month}${day}${run}_global.grib2
fi
if [ "$REMAP" = true ]; then
mkdir -p ${MODEL_DATA_FOLDER}remap
echo "Remapping the output as requested"
cdo -P 8 remap,"${HOME_FOLDER}/grids/target_grid_world_0150.txt","${HOME_FOLDER}/grids/weights_icogl2world_0150.nc" \
-setgrid,"${HOME_FOLDER}/grids/icon_grid_0026_R03B07_G.nc" "${1}_${year}${month}${day}${run}_global.nc" "remap/${1}_${year}${month}${day}${run}_global.nc"
rm "${1}_${year}${month}${day}${run}_global.nc"
fi
}
export -f download_merge_3d_variable_icon_globe
################################################
# Download grid and topography, the grid we need it for ICON global
download_invariant_icon_globe()
{
for invar_variable in "HSURF" "CLAT" "CLON" ; do
filename="icon_global_icosahedral_time-invariant_${year}${month}${day}${run}_${invar_variable}.grib2"
wget -r -nH -np -nv -nd --reject "index.html*" --cut-dirs=3 -A "${filename}.bz2" "https://opendata.dwd.de/weather/nwp/icon/grib/${run}/${invar_variable,,}/"
bzip2 -d ${filename}.bz2
cdo -f nc copy ${filename} invariant_${invar_variable}_${year}${month}${day}${run}_global.nc
rm ${filename}
done
files_to_merge=`ls -v invariant_*_${year}${month}${day}${run}_global.nc`
cdo merge ${files_to_merge} invariant_${year}${month}${day}${run}_global.nc
rm ${files_to_merge}
if [ "$REMAP" = true ]; then
mkdir -p ${MODEL_DATA_FOLDER}remap
echo "Remapping the output as requested"
cdo -P 8 remap,"${HOME_FOLDER}/grids/target_grid_world_0150.txt","${HOME_FOLDER}/grids/weights_icogl2world_0150.nc" \
-setgrid,"${HOME_FOLDER}/grids/icon_grid_0026_R03B07_G.nc" "invariant_${year}${month}${day}${run}_global.nc" "remap/invariant_${year}${month}${day}${run}_global.nc"
rm "invariant_${year}${month}${day}${run}_global.nc"
fi
}
export -f download_invariant_icon_globe
################################################
download_merge_soil_variable_icon_globe()
{
filename="icon_global_icosahedral_soil-level_${year}${month}${day}${run}_*_3_${1}.grib2"
filename_grep="icon_global_icosahedral_soil-level_${year}${month}${day}${run}_(${hours_download})_3_${1}.grib2.bz2"
url="https://opendata.dwd.de/weather/nwp/icon/grib/${run}/${1,,}/"
if [ ! -f "${1}_${year}${month}${day}${run}_global.nc" ]; then
listurls $filename_grep $url | parallel -j 10 get_and_extract_one {}
find ${filename} -empty -type f -delete # Remove empty files
cdo -f nc copy -mergetime ${filename} ${1}_${year}${month}${day}${run}_global.nc
rm ${filename}
fi
if [ "$REMAP" = true ]; then
mkdir -p ${MODEL_DATA_FOLDER}remap
echo "Remapping the output as requested"
cdo -P 8 remap,"${HOME_FOLDER}/grids/target_grid_world_0150.txt","${HOME_FOLDER}/grids/weights_icogl2world_0150.nc" \
-setgrid,"${HOME_FOLDER}/grids/icon_grid_0026_R03B07_G.nc" "${1}_${year}${month}${day}${run}_global.nc" "remap/${1}_${year}${month}${day}${run}_global.nc"
rm "${1}_${year}${month}${day}${run}_global.nc"
fi
}
export -f download_merge_soil_variable_icon_globe