PR to test openacc in main

.github/workflows/fesom2.1.yml

@@ -12,7 +12,7 @@ jobs:
     # Containers must run in Linux based operating systems
     runs-on: ubuntu-latest
     # Docker Hub image that `container-job` executes in
-    container: koldunovn/fesom2_test:refactoring2
+    container: ghcr.io/fesom/fesom2_docker:fesom2_test_refactoring-master
 
     # Service containers to run with `gfortran_ubuntu`
     steps:
@@ -69,6 +69,12 @@ jobs:
       run: |
         cd work_pi
         fcheck .
+    - name: Check UGRID Compliance
+      run: |
+        pip install ugrid-checks xarray
+        # Needed to combine the mesh diag and output
+        python3 -c "import xarray as xr; ds1 = xr.open_dataset('./test/output_pi/sst.fesom.1948.nc'); ds2 = xr.open_dataset('./test/output_pi/fesom.mesh.diag.nc'); xr.merge([ds1, ds2]).to_netcdf('./test/output_pi/merged_ugrid_check.nc')"
+        ugrid-checker ./test/output_pi/merged_ugrid_check.nc
 
     - name: Check restarts
       run: |

.github/workflows/fesom2_openmp.yml

@@ -12,7 +12,7 @@ jobs:
     # Containers must run in Linux based operating systems
     runs-on: ubuntu-latest
     # Docker Hub image that `container-job` executes in
-    container: koldunovn/fesom2_test:refactoring2
+    container: ghcr.io/fesom/fesom2_docker:fesom2_test_refactoring-nightly
 
     # Service containers to run with `gfortran_ubuntu`
     steps:

.gitignore

@@ -1,5 +1,7 @@
 /build
 /build.lib
+/work
+/lib64
 *.o
 *.mod
 *.x
@@ -12,3 +14,4 @@ lib/*.so
 /work_*
 Makefile.in
 mesh_part/build
+test/output_pi 

CMakeLists.txt

@@ -15,6 +15,8 @@ set(OIFS_COUPLED OFF CACHE BOOL "compile fesom coupled to OpenIFS. (Also needs F
 set(CRAY OFF CACHE BOOL "compile with cray ftn")
 set(USE_ICEPACK OFF CACHE BOOL "compile fesom with the Iceapck modules for sea ice column physics.")
 set(OPENMP_REPRODUCIBLE OFF CACHE BOOL "serialize OpenMP loops that are critical for reproducible results")
+set(RECOM_COUPLED OFF CACHE BOOL "compile fesom including biogeochemistry, REcoM3")
+set(CISO_COUPLED OFF CACHE BOOL "compile ciso coupled to REcoM3. RECOM_COUPLED has to be active")
 set(USE_MULTIO OFF CACHE BOOL "Use MULTIO for IO, either grib or binary for now. This also means path to MULTIO installation has to provided using env MULTIO_INSTALL_PATH='..' and multio configuration yamls must be present to run the model with MULTIO")
 set(OASIS_WITH_YAC OFF CACHE BOOL "Useing a version of OASIS compiled with YAC instead of SCRIP for interpolation?")
 set(ASYNC_ICEBERGS ON CACHE BOOL "compile fesom with or without support for asynchronous iceberg computations")

README.md

@@ -29,12 +29,3 @@ References
 * **[Version with ICEPACK sea ice thermodynamics]** Zampieri, Lorenzo, Frank Kauker, Jörg Fröhle, Hiroshi Sumata, Elizabeth C. Hunke, and Helge Goessling. Impact of Sea-Ice Model Complexity on the Performance of an Unstructured-Mesh Sea-ice/ocean Model Under Different Atmospheric Forcings. Washington: American Geophysical Union, 2020. https://dx.doi.org/10.1002/essoar.10505308.1.
 
 * **[Version coupled with OpenIFS atmosphere]** Streffing, J., Sidorenko, D., Semmler, T., Zampieri, L., Scholz, P., Andrés-Martínez, M., et al ( 2022). AWI-CM3 coupled climate model: description and evaluation experiments for a prototype post-CMIP6 model. Geoscientific Model Development, 15. https://doi.org/10.5194/gmd-15-6399-2022
-
-Documentation for FESOM2
-************************
-
-Here lives FESOM2 documentation
-
-Rendered version: https://fesom2.readthedocs.io/en/latest/
-
-

config/bin_2p1z1d/job_albedo

@@ -0,0 +1,66 @@
+#!/bin/bash
+#SBATCH --account=nwg_hauck.maresys # edit your account
+#SBATCH --job-name=recom
+#SBATCH --partition=mpp
+#SBATCH --time=01:30:00
+####SBATCH --constraint="[rack1|rack3]"
+#SBATCH --qos=12h
+#SBATCH --nodes=9            # Number of tasks (MPI) tasks to be launched
+#SBATCH --tasks-per-node 96 #8 #4  # using all 128CPus we exceed the maxload limit (>130) of many used nodes when us parallel I/O
+#SBATCH --cpus-per-task 1
+
+#SBATCH -o fesom2o.out
+#SBATCH -e fesom2e.out
+
+# disable hyperthreading
+#SBATCH --hint=nomultithread
+
+module purge 
+source ../env/albedo/shell
+export OMP_NUM_THREADS=1
+ulimit -s unlimited
+
+# determine JOBID
+JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1`
+
+ln -s ../bin/fesom.x .           # cp -n ../bin/fesom.x
+cp -n ../config/namelist.config  .
+cp -n ../config/namelist.forcing .
+cp -n ../config/namelist.oce     .
+cp -n ../config/namelist.dyn     .
+cp -n ../config/namelist.tra     .
+cp -n ../config/namelist.ice     .
+cp -n ../config/namelist.io      .
+cp -n ../config/namelist.icepack .
+
+#___DETERMINE SLURM JOBID+OUTPUTFILE____________________________________________
+jobid=$(echo $SLURM_JOB_ID | cut -d"." -f1)
+fname="fesom2_${SLURM_JOB_NAME}_${jobid}.out"
+
+#___PUT JOB IN QUEUE____________________________________________________________
+date
+####srun --mpi=pmi2 ./fesom.x >> ${fname}
+srun --mpi=pmi2 ./fesom.x > fesom2.0.out
+date
+
+#qstat -f $PBS_JOBID
+#export EXITSTATUS=$?
+#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then
+#sbatch job_ollie
+#fi
+
+#Resultpath='/albedo/scratch/user/ogurses/ocean_only/'
+Resultpath='/albedo/scratch/user/ogurses/fesom2.5_recom_st1_vsink_vben_diags/'
+test -d $Resultpath/fesom.2019.oce.restart && exit
+
+IsInFile=$( tail -3 fesom2.0.out | grep -c timesteps)
+if (( IsInFile > 0 )); then
+# submit next #job                                                                                                     \
+ echo "submitting next job"
+ cp fesom2.0.out fesom.out.done
+ sbatch job_albedo
+else
+ echo "something is wrong, last line of fesom.out reads"
+ echo $( tail -1 fesom2.0.out)
+ echo "abnormal termination of job script"
+fi

config/bin_2p1z1d/namelist.config

@@ -0,0 +1,67 @@
+! This is the namelist file for model general configuration
+
+&modelname
+runid='fesom'
+/
+
+&timestep
+step_per_day=32 !96 !96 !72 !72 !45 !72 !96
+run_length=1 !62 !62 !62 !28
+run_length_unit='y'		! y, m, d, s
+/
+
+&clockinit              ! the model starts at
+timenew=0.0
+daynew=1
+yearnew=1958
+/
+
+&paths
+MeshPath='/albedo/work/projects/p_recompdaf/frbunsen/FESOM2/meshes/core2/'
+ClimateDataPath='/albedo/work/projects/MarESys/ogurses/input/corrected_input/' 
+ResultPath='/albedo/scratch/user/ogurses/fesom2.5_recom_st1_vsink_vben_diags/'
+/
+
+&restart_log
+restart_length=1            ! --> do netcdf restart ( only required for d,h,s cases, y, m take 1)
+restart_length_unit='y'     !output period: y, d, h, s, off
+raw_restart_length=1        ! --> do core dump restart
+raw_restart_length_unit='off' ! e.g. y, d, h, s, off
+bin_restart_length=1        ! --> do derived type binary restart 
+bin_restart_length_unit='off' ! e.g. y, d, h, s, off
+logfile_outfreq=960     !in logfile info. output frequency, # steps
+/
+
+&ale_def
+which_ALE='zstar'       ! 'linfs','zlevel', 'zstar'
+use_partial_cell=.true.
+/
+
+&geometry
+cartesian=.false.
+fplane=.false.
+cyclic_length=360       ![degree]
+rotated_grid=.true.     !option only valid for coupled model case now
+force_rotation=.true.
+alphaEuler=50.          ![degree] Euler angles, convention:
+betaEuler=15.           ![degree] first around z, then around new x,
+gammaEuler=-90.         ![degree] then around new z.
+/
+
+&calendar
+include_fleapyear=.false.
+/
+
+&run_config
+use_ice=.true.                  ! ocean+ice
+use_cavity=.false.              !
+use_cavity_partial_cell=.false. 
+use_floatice = .false.
+use_sw_pene=.true.
+flag_debug=.false.
+/
+
+&machine
+n_levels=2
+n_part= 12, 36          ! 432 number of partitions on each hierarchy level
+/

config/bin_2p1z1d/namelist.dyn

@@ -0,0 +1,24 @@
+&dynamics_visc
+visc_gamma0  = 0.003 ! [m/s],   backgroung viscosity= gamma0*len, it should be as small a s possible (keep it < 0.01 m/s).
+visc_gamma1  = 0.1   ! [nodim], for computation of the flow aware viscosity
+visc_gamma2  = 0.285 ! [s/m],   is only used in easy backscatter option
+visc_easybsreturn= 1.5
+
+opt_visc     = 5          
+! 5=Kinematic (easy) Backscatter
+! 6=Biharmonic flow aware (viscosity depends on velocity Laplacian)
+! 7=Biharmonic flow aware (viscosity depends on velocity differences)
+! 8=Dynamic Backscatter
+
+use_ivertvisc= .true.  
+/
+
+&dynamics_general
+momadv_opt   = 2       ! option for momentum advection in moment only =2
+use_freeslip = .false. ! Switch on free slip
+use_wsplit   = .false. ! Switch for implicite/explicte splitting of vert. velocity
+wsplit_maxcfl= 1.0     ! maximum allowed CFL criteria in vertical (0.5 < w_max_cfl < 1.) 
+                       ! in older FESOM it used to be w_exp_max=1.e-3
+ldiag_KE=.false.       ! activates energy diagnostics
+/
+