-
Notifications
You must be signed in to change notification settings - Fork 5
Building GridPACK on Perlmutter
Sumathi Lakshmiranganatha edited this page Nov 30, 2023
·
22 revisions
First, copy the gpbuild folder into the home directory:
rsync -rl --info=progress2 /global/common/software/m3363/gpbuildpm ~
cd ~/gpbuildpm
module load gpu
module load cmake
module load python
tar xvzf boost*.tar.gz
cp mpi.jam boost_1_80_0/tools/build/src/tools/mpi.jam
cd boost_1_80_0
./bootstrap.sh --with-libraries=mpi --prefix=$HOME/gpbuildpm
cp ../project-config.jam project-config.jam
./b2 --debug-configuration --debug-building -d+2 2>&1 | tee output.txt
cd libs/mpi/build
../../../b2 --debug-building -d+2 --debug-configuration 2>&1 | tee output.txt
cd ../../../
cd libs/random/build
../../../b2 --debug-building -d+2 --debug-configuration 2>&1 | tee output.txt
cd ../../../
cd libs/filesystem/build
../../../b2 --debug-building -d+2 --debug-configuration 2>&1 | tee output.txt
cd ../../../
cd libs/system/build
../../../b2 --debug-building -d+2 --debug-configuration 2>&1 | tee output.txt
cd ../../../
./b2 install | tee output.txt
cd ~/gpbuildpm
tar xvzf ga-5.8.tar.gz
cd ga-5.8
mkdir build_ts_so
cp ../build_csh_ga build_ts_so/build_csh
cd build_ts_so
# Edit build_csh to reflect your home directory
source build_csh 2>&1 | tee output.txt
make 2>&1 | tee output-make.txt
make install 2>&1 | tee output-make.txt
cd ~/gpbuildpm
module load gpu
module load cmake
module load python
tar xvzf petsc-3.18.1.tar.gz
cp build_csh_petsc petsc-3.18.1/build_csh
cd petsc*/
chmod +x build_csh
./build_csh
make PETSC_DIR=/global/u2/t/tflynn/gpbuildpm/petsc-3.18.1 PETSC_ARCH=pm all
cd ~/gpbuildpm
module load gpu
module load cmake
module load python
mkdir GridPACK/src/build
cp build_csh_gridpack GridPACK/src/build/build_csh
cp ToolChain.cmake GridPACK/src/build/
cd GridPACK/src/build
# Update paths in build_csh
make clean
./build_csh
make
make install
cd ~/gpbuildpm/GridPACK/python
GRIDPACK_DIR=$HOME/gpbuildpm/GridPACK/src/gridpack-install
export GRIDPACK_DIR
unset RHEL_OPENMPI_HACK
rm -rf build/*
rm -rf *.so
rm -rf gridpack_hadrec.egg-info/
python setup.py build (it gives an error for boost library not found)
Workaround to overcome this error for now is to add the following to the file build/temp.linux-x86_64-cpython-39/CMakeCache.txt under //Dependencies for the target
-L/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/lib;general;-lboost_mpi;-lboost_serialization;-lboost_random;-lboost_filesystem;-lboost_system;
NOTE: Change the lib path to your installation path
python setup.py build
mkdir $GRIDPACK_DIR/lib/python
PYTHONPATH="${GRIDPACK_DIR}/lib/python:${PYTHONPATH}"
export PYTHONPATH
python setup.py install --home="$GRIDPACK_DIR"
cd ~/gpbuildpm/boost_mpi_test
salloc -C gpu --qos debug -N 1 -t 10 -c 4 -A m3363
module load gpu
module load cmake
module load python
CC -I../include -L../lib -lboost_mpi -lboost_serialization test.cpp -o test
LD_LIBRARY_PATH=$HOME/gpbuildpm/lib:$LD_LIBRARY_PATH srun -n 4 ./test
I am process 1 of 4.
I am process 2 of 4.
I am process 0 of 4.
I am process 3 of 4.
salloc -C gpu --qos debug -N 1 -t 10 -c 4 -A m3363
module load gpu
module load cmake
module load python
export PETSC_DIR=$HOME/gpbuildpm/petsc-3.18.1/
export PETSC_ARCH=pm
cd ~/gpbuildpm/petsc-3.18.1/src/snes/tutorials
make ex3
srun -n 3 ./ex3 -nox -pc_type asm -mat_type mpiaij -snes_monitor_cancel -snes_monitor_short -ksp_gmres_cgs_refinement_type refine_always
Output:
atol=1e-50, rtol=1e-08, stol=1e-08, maxit=50, maxf=10000
0 SNES Function norm 5.41468
1 SNES Function norm 0.295258
2 SNES Function norm 0.000450229
3 SNES Function norm 1.38967e-09
Number of SNES iterations = 3
Norm of error 1.49751e-10 Iterations 3
make checkprogs
salloc -C cpu --qos debug -N 1 -t 30 -c 4 -A m3363
make check MPIEXEC="srun -n 2"
(most tests pass, some fail)
salloc -C cpu --qos debug -N 1 -t 10 -c 4 -A m3363
module load gpu
module load cmake
module load python
export PETSC_DIR=$HOME/gpbuildpm/petsc-3.18.1/
export PETSC_ARCH=pm
cd ~/gpbuildpm/GridPACK/src/build/applications/powerflow
srun -n 3 pf.x input_118.xml
GridPACK math module configured on 3 processors
Maximum number of iterations: 50
Convergence tolerance: 0.000001
0: I have 49 buses and 63 branches
1: I have 53 buses and 70 branches
2: I have 42 buses and 59 branches
repeat time = 1
----------test Iteration 0, before PF solve, Tol: 9.638903e-02
Residual norms for nrs_ solve.
0 KSP Residual norm 7.190970950247e-03
1 KSP Residual norm 1.347853257515e-03
2 KSP Residual norm 4.532559877150e-04
3 KSP Residual norm 2.913746016136e-04
4 KSP Residual norm 1.634291034444e-04
5 KSP Residual norm 7.794085477059e-05
6 KSP Residual norm 6.512994360601e-05
7 KSP Residual norm 4.197672279641e-05
8 KSP Residual norm 2.146565270298e-05
9 KSP Residual norm 1.425396909915e-05
...
salloc -C gpu --qos debug -N 1 -t 30 -c 4 -A m3363
module load gpu
module load cmake
module load python
cd ~/gpbuildpm/GridPACK/python
GRIDPACK_DIR=$HOME/gpbuildpm/GridPACK/src/gridpack-install
export GRIDPACK_DIR
PYTHONPATH="${GRIDPACK_DIR}/lib/python/gridpack_hadrec-0.0.1-py3.9-linux-x86_64.egg:${PYTHONPATH}” // add egg’s path to pythonpath
export PYTHONPATH
export PATH=$HOME/gpbuildpm/GridPACK/src/gridpack-install/bin:$PATH
hello.py:
srun -n 3 python hello.py
hello.py: hello from process 0 of 3
hello.py: hello from process 1 of 3
hello.py: hello from process 2 of 3
GridPACK math module configured on 3 processors
python setup.py test:
export BOOST_DIR=$HOME/gpbuildpm
export LD_LIBRARY_PATH=$BOOST_DIR/lib:$LD_LIBRARY_PATH
python setup.py test
running test
WARNING: Testing via this command is deprecated and will be removed in a future version. Users looking for a generic test entry point independent of test runner are encouraged to use tox.
/global/common/software/nersc/pm-2022q3/sw/python/3.9-anaconda-2021.11/lib/python3.9/site-packages/setuptools/installer.py:27: SetuptoolsDeprecationWarning: setuptools.installer is deprecated. Requirements should be satisfied by a PEP 517 installer.
warnings.warn(
running egg_info
writing gridpack_hadrec.egg-info/PKG-INFO
writing dependency_links to gridpack_hadrec.egg-info/dependency_links.txt
writing top-level names to gridpack_hadrec.egg-info/top_level.txt
reading manifest file 'gridpack_hadrec.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
adding license file 'LICENSE'
writing manifest file 'gridpack_hadrec.egg-info/SOURCES.txt'
running build_ext
-- Cray Programming Environment 2.7.16 C
-- NOTE: LOADEDMODULES changed since initial config!
-- NOTE: this may cause unexpected build errors.
-- Cray Programming Environment 2.7.16 CXX
-- pybind11 v2.4.3
-- GRIDPACK_HAVE_GOSS: OFF
-- GRIDPACK_GOSS_LIBRARY:
-- Include directories for GridPACK: /global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/include;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/petsc-3.18.1/pm/include;/global/u2/s/sumathil/PM_powergridenv/gpbuildpm/petsc-3.18.1/pm/include;/global/u2/s/sumathil/PM_powergridenv/gpbuildpm/petsc-3.18.1/include;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/ga-5.8/build_ts_so/include;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/include
-- Libraries for GridPACK: /global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_powerflow_module.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_dynamic_simulation_module.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_dynamic_simulation_full_y_module.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_kalmands_module.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_state_estimation_module.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_hadrec_module.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_ymatrix_components.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_pfmatrix_components.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_dsmatrix_components.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_kdsmatrix_components.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_sematrix_components.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_components.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_configuration.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_environment.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_math.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_expression.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_optimum.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_partition.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_timer.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_stream.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_analysis.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install/lib/libgridpack_parallel.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/petsc-3.18.1/pm/lib/libparmetis.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/petsc-3.18.1/pm/lib/libmetis.so;-L/global/u2/s/sumathil/PM_powergridenv/gpbuildpm/petsc-3.18.1/pm/lib;-lpetsc;-L/global/u2/s/sumathil/PM_powergridenv/gpbuildpm/petsc-3.18.1/pm/lib;-L/opt/cray/pe/mpich/8.1.17/ofi/gnu/9.1/lib;-L/opt/cray/pe/mpich/8.1.17/gtl/lib;-L/opt/cray/pe/libsci/21.08.1.2/GNU/9.1/x86_64/lib;-L/opt/nvidia/hpc_sdk/Linux_x86_64/22.5/cuda/11.7/lib64/stubs;-L/opt/nvidia/hpc_sdk/Linux_x86_64/22.5/cuda/11.7/lib64;-L/opt/nvidia/hpc_sdk/Linux_x86_64/22.5/cuda/11.7/nvvm/lib64;-L/opt/nvidia/hpc_sdk/Linux_x86_64/22.5/cuda/11.7/extras/CUPTI/lib64;-L/opt/nvidia/hpc_sdk/Linux_x86_64/22.5/cuda/11.7/extras/Debugger/lib64;-L/opt/nvidia/hpc_sdk/Linux_x86_64/22.5/math_libs/11.7/lib64;-L/global/common/software/nersc/pm-2022q3/sw/darshan/3.4.0/lib;-L/opt/cray/pe/dsmml/0.2.2/dsmml/lib;-L/opt/cray/xpmem/2.4.4-2.3_13.8__gff0e1d9.shasta/lib64;-L/opt/cray/pe/gcc/11.2.0/snos/lib/gcc/x86_64-suse-linux/11.2.0;-L/opt/cray/pe/gcc/11.2.0/snos/lib64;-L/opt/cray/pe/gcc/11.2.0/snos/lib;-lpetsc;-lspqr;-lumfpack;-lklu;-lcholmod;-lbtf;-lccolamd;-lcolamd;-lcamd;-lamd;-lsuitesparseconfig;-lsuperlu;-lsuperlu_dist;-lf2clapack;-lf2cblas;-lparmetis;-lmetis;-lstdc++;-ldl;-lcuda;-ldarshan;-lz;-lmpi_gtl_cuda;-lxpmem;-lgfortran;-lm;-lcupti;-lcudart;-lsci_gnu_82_mpi;-lsci_gnu_82;-lmpifort_gnu_91;-lmpi_gnu_91;-ldsmml;-lgfortran;-lquadmath;-lpthread;-lm;-lgcc_s;-lquadmath;-lstdc++;-ldl;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/ga-5.8/build_ts_so/lib/libga++.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/ga-5.8/build_ts_so/lib/libga.so;/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/ga-5.8/build_ts_so/lib/libarmci.so;-lrt;-ldl;Boost::mpi;Boost::serialization;Boost::random;Boost::filesystem;Boost::system
-- Configuring done
-- Generating done
-- Build files have been written to: /global/u2/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/python/build/temp.linux-x86_64-cpython-39
[ 0%] Built target parallel_scripts
Consolidate compiler generated dependencies of target gridpack
[ 50%] Linking CXX shared module ../../../gridpack.cpython-39-x86_64-linux-gnu.so
[100%] Built target gridpack
GridPACK math module configured on 1 processors
hadrec_test (tests.gridpack_test.GridPACKTester) ... ** PETSc DEPRECATION WARNING ** : the option -pc_factor_mat_solver_package is deprecated as of version 3.9 and will be removed in a future release. Please use the option -pc_factor_mat_solver_type instead. (Silence this warning with -options_suppress_deprecated_warnings)
** PETSc DEPRECATION WARNING ** : the option -pc_factor_mat_solver_package is deprecated as of version 3.9 and will be removed in a future release. Please use the option -pc_factor_mat_solver_type instead. (Silence this warning with -options_suppress_deprecated_warnings)
** PETSc DEPRECATION WARNING ** : the option -pc_factor_mat_solver_package is deprecated as of version 3.9 and will be removed in a future release. Please use the option -pc_factor_mat_solver_type instead. (Silence this warning with -options_suppress_deprecated_warnings)
** PETSc DEPRECATION WARNING ** : the option -pc_factor_mat_solver_package is deprecated as of version 3.9 and will be removed in a future release. Please use the option -pc_factor_mat_solver_type instead. (Silence this warning with -options_suppress_deprecated_warnings)
** PETSc DEPRECATION WARNING ** : the option -pc_factor_mat_solver_package is deprecated as of version 3.9 and will be removed in a future release. Please use the option -pc_factor_mat_solver_type instead. (Silence this warning with -options_suppress_deprecated_warnings)
** PETSc DEPRECATION WARNING ** : the option -pc_factor_mat_solver_package is deprecated as of version 3.9 and will be removed in a future release. Please use the option -pc_factor_mat_solver_type instead. (Silence this warning with -options_suppress_deprecated_warnings)
** PETSc DEPRECATION WARNING ** : the option -pc_factor_mat_solver_package is deprecated as of version 3.9 and will be removed in a future release. Please use the option -pc_factor_mat_solver_type instead. (Silence this warning with -options_suppress_deprecated_warnings)
** PETSc DEPRECATION WARNING ** : the option -pc_factor_mat_solver_package is deprecated as of version 3.9 and will be removed in a future release. Please use the option -pc_factor_mat_solver_type instead. (Silence this warning with -options_suppress_deprecated_warnings)
ok
hello_test (tests.gridpack_test.GridPACKTester) ... ok
task_test (tests.gridpack_test.GridPACKTester) ... ok
----------------------------------------------------------------------
Ran 3 tests in 4.499s
OK
Make a new conda environment -- Runs only on a single node with mpirun for async workflow (not srun). If run with srun, it won't train and exit immediately.
module load python
conda create --name gpenvpm --clone base
conda activate gpenvpm
conda install -c conda-forge gym
pip install xmltodict ray importlib-metadata tensorflow
pip3 install torch
Make a new conda environment with srun (multi-rank and multi-node) using async workflow.
module load python
module load tensorflow
conda create --name gpenvpm --clone base
conda activate gpenvpm
conda install -c conda-forge gym
pip install xmltodict ray importlib-metadata
pip3 install torch
git clone --recurse-submodules --branch PARS_300bus --single-branch https://github.com/exalearn/EXARL.git
salloc -C gpu --qos debug -N 1 -t 30 -c 4 -A m3363
cd EXARL
module load cmake gpu python
conda activate gpenvpm
export GRIDPACK_DIR=/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/GridPACK/src/gridpack-install
export PYTHONPATH="${GRIDPACK_DIR}/lib/python/gridpack_hadrec-0.0.1-py3.9-linux-x86_64.egg:${PYTHONPATH}"
export PATH=${GRIDPACK_DIR}/bin:$PATH
export PETSC_DIR=/global/u2/s/sumathil/PM_powergridenv/gpbuildpm/petsc-3.18.1
export PETSC_ARCH=pm
export GA_DIR="/global/homes/s/sumathil/PM_powergridenv/gpbuildpm/ga-5.8/build_ts_so"
export RHEL_OPENMPI_HACK=yes
export PYTHONPATH="$PWD:$PYTHONPATH"
srun -n 1 python exarl/driver/__main__.py --agent PARS-v1 --env Pendulum-v1 --n_episodes 10000 --n_steps 200 --workflow sync_ars --model_type nonlinear --policy_network_size 16 --delta_std 0.1 --decay 1.0 --sgd_step_size 0.018 --num_l 3 --n_delta 4 --n_top 1