Skip to content

Commit

Permalink
Merge pull request #2 from patricialarsen/add_xrays
Browse files Browse the repository at this point in the history
Add xrays and updated grav-only maps
  • Loading branch information
patricialarsen authored Jul 29, 2024
2 parents d673a96 + c4b4ead commit a53638d
Show file tree
Hide file tree
Showing 209 changed files with 11,966 additions and 159 deletions.
21 changes: 21 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,3 +1,24 @@
check_halo_direct/obj/*
check_halo_direct/params_*.txt
check_halo_direct/run_test

# log files
density_maps/*.error
density_maps/*.cobaltlog
density_maps/*.output
denstiy_maps/logs/*

pixelize_halo_lightcones/LJ/*.error
pixelize_halo_lightcones/LJ/*.cobaltlog
pixelize_halo_lightcones/LJ/*.output
pixelize_halo_lightcones/LJ/pixelize_LJ

cic2gio/logs/*.error
cic2gio/logs/*.cobaltlog
cic2gio/logs/*.output
cic2gio/cic_gio
cic2gio/memtest

# executables
density_maps/dens
density_maps/hydro
18 changes: 18 additions & 0 deletions cic2gio/Cobalt_cooley.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/bin/sh
NODES=`cat $COBALT_NODEFILE | wc -l`
PROCS=6

LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/prlarsen/usr/cfitsio-4.0.0/build/lib/
#LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/prlarsen/usr/Healpix_3.82/lib
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/prlarsen/codes/healpix/Healpix_3.82/lib
OMP_NUM_THREADS=1
export OMP_NUM_THREADS

export LD_LIBRARY_PATH


mpirun -f $COBALT_NODEFILE -n $PROCS ./memtest /lus/eagle/projects/CosDiscover/nfrontiere/576MPC_RUNS/challenge_problem_576MPC_ADIABATIC/output/m000p.lc.mpicosmo. /eagle/LastJourney/prlarsen/LC_hydro/2023/ksz_310_400.fits /eagle/LastJourney/prlarsen/LC_hydro/2023/tsz_310_400.fits 0.01 2 600 2048

#8192

#correct_massfunc.py
45 changes: 36 additions & 9 deletions cic2gio/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -5,41 +5,68 @@ CXX = mpicxx

FLAGS = -I.

HPXINCDIR = -I/home/prlarsen/usr/Healpix_3.80/include
HPXLIBDIR = -I/home/prlarsen/usr/Healpix_3.80/lib
HACC_DIR = /home/prlarsen/hacc_fresh/HACC

HPXINCDIR2 = -I/home/prlarsen/usr/Healpix_3.80/include/healpix_cxx
HPXDIR = /home/prlarsen/codes/healpix/Healpix_3.82
HPXINCDIR = -I${HPXDIR}/include
HPXLIBDIR = ${HPXDIR}/lib
HPXINCDIR2 = -I${HPXDIR}/include/healpix_cxx

#HPXINCDIR = -I/home/prlarsen/usr/Healpix_3.80/include
#HPXLIBDIR = -I/home/prlarsen/usr/Healpix_3.80/lib

#HPXINCDIR2 = -I/home/prlarsen/usr/Healpix_3.80/include/healpix_cxx
FITSINCDIR = -I/home/prlarsen/usr/cfitsio-4.0.0/build/include
FITSLIBDIR = /home/prlarsen/usr/cfitsio-4.0.0/build/lib

GIOINCDIR = -I/home/prlarsen/hacc_fresh/trunk/genericio
BLOSCDIR = -I/home/prlarsen/hacc_fresh/trunk/genericio/thirdparty/blosc
GIOINCDIR = -I${HACC_DIR}/genericio
BLOSCDIR = -I${HACC_DIR}/genericio/thirdparty/blosc
BLOSCINCDIR = -I${HACC_DIR}/cooley.cpu/mpi/genericio/thirdparty/blosc

HACCINCDIR = -I/home/prlarsen/hacc_fresh/trunk/cosmotools/algorithms/halofinder
HACCINCDIR = -I${HACC_DIR}/cosmotools/algorithms/halofinder
HACCINCDIR2 = -I${HACC_DIR}/cosmotools/common

GIOLIBDIR=/home/prlarsen/hacc_fresh/trunk/cooley.cpu/mpi/lib
GIOINCDIR2= -I/home/prlarsen/hacc_fresh/trunk/cooley.cpu/mpi/genericio


#CXXHPXLIBS = -L${HPXLIBDIR} -lhealpix_cxx
#CXXHPXDIRS = -I/home/prlarsen/usr/Healpix_3.80/src/cxx/include
CXXHPXLIBS = -L/home/prlarsen/usr/Healpix_3.80/lib -lhealpix_cxx
#CXXHPXLIBS = -L/home/prlarsen/usr/Healpix_3.80/lib -lhealpix_cxx
#-lcxxsupport -lsharp -lfftpack -lc_utils -lcfitsio


CFLAGS = $(FITSINCDIR) $(HPXINCDIR) $(HPXINCDIR2) $(CXXHPXDIRS) $(GIOINCDIR) $(GIOINCDIR2) $(BLOSCDIR) $(HACCINCDIR) -fopenmp
LDFLAGS = -lm $(CXXHPXLIBS) -L$(HPXLIBDIR) -L$(FITSLIBDIR) -lGenericIOMPI -L$(GIOLIBDIR) -lGenericIOMPI
CFLAGS = $(FITSINCDIR) $(HPXINCDIR) $(HPXINCDIR2) $(CXXHPXDIRS) $(GIOINCDIR) $(GIOINCDIR2) $(BLOSCDIR) $(HACCINCDIR) $(HACCINCDIR2) -fopenmp
LDFLAGS = -lm $(CXXHPXLIBS) -L$(HPXLIBDIR) -L$(FITSLIBDIR) -lGenericIOMPI -L$(GIOLIBDIR) -lGenericIOMPI -lchealpix
#-L/usr/lib/

SRCDIR = src
OBJDIR = obj
BINDIR = bin
#LIBDIR = usr/lib64/

HACC_INC_DIRS = ${GIOINCDIR} ${HACCINCDIR} ${HACCINCDIR2} ${BLOSCDIR} ${GIOINCDIR2}
HACC_LIB_DIRS = -L${GIOLIBDIR}

HPX_INC_DIRS = ${HPXINCDIR2} ${HPXINCDIR} ${FITSINCDIR}
HPX_LIB_DIRS = -L${HPXLIBDIR} -L${FITSLIBDIR}

cictogio: cic_gio2healpix.cxx
$(CXX) -O3 -o cic_gio /home/prlarsen/hacc_fresh/trunk/cooley.cpu/mpi/genericio/GenericIO.o ${HPXOFILES} ${CXXHPXDIRS} ${CFLAGS} ${LDFLAGS} ${FLAGS} cic_gio2healpix.cxx -lcurl

cictocart: cic_gio2cart.cxx
$(CXX) -O3 -o cic_cart /home/prlarsen/hacc_fresh/trunk/cooley.cpu/mpi/genericio/GenericIO.o ${HPXOFILES} ${CXXHPXDIRS} ${CFLAGS} ${LDFLAGS} ${FLAGS} cic_gio2cart.cxx -lcurl

memtest2: cic_gio2healpix_memoryupdate.cxx
$(CXX) -O3 -o memtest /home/prlarsen/hacc_fresh/trunk/cooley.cpu/mpi/genericio/GenericIO.o ${HPXOFILES} ${CFLAGS} ${LDFLAGS} ${FLAGS} cic_gio2healpix_memoryupdate.cxx -lcurl

memtest: cic_gio2healpix_memoryupdate.cxx
$(CXX) -fopenmp -O3 -o memtest /home/prlarsen/hacc_fresh/HACC/cooley.cpu/mpi/genericio/GenericIO.o -lm -lGenericIOMPI ${HACC_INC_DIRS} ${HACC_LIB_DIRS} ${HPX_INC_DIRS} ${HPX_LIB_DIRS} -I${BLOSCINCDIR} -lchealpix cic_gio2healpix_memoryupdate.cxx -lchealpix -lcfitsio -lcurl

memtest3: cic_gio2healpix_memoryupdate.cxx
$(CXX) -fopenmp -O3 -o memtest /home/prlarsen/hacc_fresh/HACC/cooley.cpu/mpi/genericio/GenericIO.o -lm -lGenericIOMPI ${HACC_INC_DIRS} ${HACC_LIB_DIRS} ${HPX_INC_DIRS} ${HPX_LIB_DIRS} -I${BLOSCINCDIR} -lhealpix_cxx -lchealpix cic_gio2healpix_memoryupdate.cxx -lchealpix -lcfitsio -lcurl


#-dynamic


Expand Down
45 changes: 45 additions & 0 deletions cic2gio/SZ_class.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
/*
* Use this file to access and allocate SZ-related data.
*/

#include <string>
#include <vector>
#include <map>
#include "AllocatedVector.h"
#include "GenericIO.h"

using namespace std;

struct sz_props{
int64_t pix_num;
double tsz;
double ksz;
int rank;
};

class SZ_class {

public:

bool is_allocated;
int step_number;
size_t num_parts;
MPI_Datatype sz_properties_MPI_Type;

vector<int64_t>* pix_num;
vector<double>* tsz;
vector<double>* ksz;
// destination rank for redistribution
vector<int>* rank;

SZ_class() : num_parts(0), is_allocated(false),\
step_number(-1);
~SZ_class() { };

void Allocate(size_t n=0);
void Deallocate();
sz_props GetProperties(size_t idx);
void Resize(size_t);
void Set_MPIType();
};

101 changes: 101 additions & 0 deletions cic2gio/SZ_props.cxx
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
#include <iostream>
#include "PartitionPlus.h"
#include "SZ_class.h"

using namespace std;
using namespace gio;

// Allocate actually allocates memory with optional vector size
void SZ_class::Allocate(size_t n) {
if (!this->is_allocated) {
this->is_allocated = true;
this->num_parts = n;

this->pix_num = new vector<int64_t>(this->num_parts);
this->rank = new vector<int>(this->num_parts);
this->tsz = new vector<double>(this->num_parts);
this->ksz = new vector<double>(this->num_parts);
}
else {
cerr << "ERROR: Can't allocate new vectors --\
vectors are already allocated." << endl;
;// do some graceful exit of the program
}
}

void SZ_class::Set_MPIType(){
MPI_Datatype type[4] = { MPI_INT64_T, MPI_DOUBLE, MPI_DOUBLE, MPI_INT };
int blocklen[4] = {1,1,1,1};

sz_props sz;

MPI_Aint base;
MPI_Aint disp[4];

MPI_Get_address(&sz, &base);
MPI_Get_address(&sz.pix_num, &disp[0]);
MPI_Get_address(&sz.tsz, &disp[1]);
MPI_Get_address(&sz.ksz, &disp[2]);
MPI_Get_address(&sz.rank, &disp[3]);

disp[0]-=base; disp[1]-=base; disp[2]-=base; disp[3]-=base;

MPI_Type_struct(4,blocklen,disp,type,&this->sz_properties_MPI_Type);
MPI_Type_commit(&this->sz_properties_MPI_Type);

}

void SZ_class::Deallocate() {

if (this->is_allocated) {

this->num_parts = 0;

delete this-> pix_num;
delete this-> rank;
delete this-> tsz;
delete this-> ksz;

this->is_allocated=false;
}
else {
cerr << "ERROR: Can't deallocate -- vectors are not allocated."\
<< endl;
;// do some graceful exit of the program
}
}

sz_props SZ_class::GetProperties(size_t idx) {
if (this->is_allocated) {
sz_props sz_properties;

sz_properties.rank = this->rank->at(idx);
sz_properties.pix_num = this->pix_num->at(idx);
sz_properties.tsz = this->tsz->at(idx);
sz_properties.ksz = this->ksz->at(idx);

return sz_properties;
}
else {
cerr << "ERROR: Can't get properties -- vectors are not allocated."\
<< endl;
;// do some graceful exit of the program
}
}


// use Resize(0) as a substitute for std::vector clear()
void SZ_class::Resize(size_t n) {
if (this->is_allocated) {
this->num_parts = n;
this->pix_num->resize(this->num_parts);
this->rank->resize(this->num_parts);
this->tsz->resize(this->num_parts);
this->ksz->resize(this->num_parts);
}
else {
cerr << "ERROR: Can't resize -- vectors are not allocated." << endl;
;// do some graceful exit of the program
}
}

9 changes: 7 additions & 2 deletions cic2gio/cic_gio2healpix.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ int main(int argc, char *argv[]) {
vector<POSVEL_T> xx, yy, zz;
vector<POSVEL_T> vx, vy, vz;
vector<POSVEL_T> a;
vector<MASK_T> mask;


#ifdef HYBRID_SG
Expand Down Expand Up @@ -158,6 +159,7 @@ int main(int argc, char *argv[]) {
vz.resize(Np + GIO.requestedExtraSpace()/sizeof(POSVEL_T));
uu.resize(Np + GIO.requestedExtraSpace()/sizeof(POSVEL_T));
mass.resize(Np + GIO.requestedExtraSpace()/sizeof(POSVEL_T));
mask.resize(Np + GIO.requestedExtraSpace()/sizeof(MASK_T));
#ifndef HYBRID_SG
mu.resize(Np + GIO.requestedExtraSpace()/sizeof(POSVEL_T));
#endif
Expand All @@ -177,6 +179,7 @@ int main(int argc, char *argv[]) {
GIO.addVariable("mu", mu, true);
#endif
GIO.addVariable("id", id, true);
GIO.addVariable("mask",mask,true);

GIO.readData();
} // destroy GIO prior to calling MPI_Finalize
Expand All @@ -190,6 +193,8 @@ int main(int argc, char *argv[]) {
vz.resize(Np);
uu.resize(Np);
mass.resize(Np);
mask.resize(Np);

#ifndef HYBRID_SG
mu.resize(Np);
#endif
Expand Down Expand Up @@ -287,9 +292,9 @@ int main(int argc, char *argv[]) {
const double SIGMAT = 6.652e-25;
const double MP = 1.6737236e-24;
const double ME = 9.1093836e-28;
const double CLIGHT = 2.99792458e10;
//const double CLIGHT = 2.99792458e10;
const double MUE = 1.14;
const double GAMMA = 5.0/3.0;
//const double GAMMA = 5.0/3.0;
const double YP = 0.24;
const double NHE = 0.0; // assume helium is neutral
const double CHIE = (1.0-YP*(1.0-NHE/4.0))/(1.0-YP/2.0);
Expand Down
Loading

0 comments on commit a53638d

Please sign in to comment.