diff --git a/matchup/Makefile b/matchup/Makefile new file mode 100644 index 0000000..bd38124 --- /dev/null +++ b/matchup/Makefile @@ -0,0 +1,21 @@ +CPP := mpicxx +CPPFLAGS := -O3 +SRCDIR := ./src +OBJDIR := ./obj +INCLUDES := -I $(SRCDIR) -I/home/prlarsen/hacc3/trunk/genericio -I/home/prlarsen/hacc3/trunk/cosmotools/algorithms/halofinder -I/home/prlarsen/hacc3/trunk/cosmotools/algorithms/mergertree_ct -I/home/prlarsen/hacc3/trunk/cosmotools/common +# + +#/home/prlarsen/trunk_steve/trunk/genericio -I/home/prlarsen/trunk_steve/trunk/cosmotools/algorithms/halofinder -I/home/prlarsen/trunk_steve/trunk/cosmotools/algorithms/mergertree_ct -I/home/prlarsen/trunk_steve/trunk/cosmotools/common +OBJECTS := $(OBJDIR)/main.o + + +#linking +match : $(OBJECTS) $(LIBS) + $(CPP) $(CPPFLAGS) $(OBJECTS) -L/home/prlarsen/hacc3/trunk/cooley.cpu/mpi/lib -lCosmoTools -lGenericIOMPI -o match -fopenmp + +#compilation +$(OBJDIR)/main.o: $(SRCDIR)/main.cpp + $(CPP) $(CPPFLAGS) $(INCLUDES) -c $(SRCDIR)/main.cpp -o $(OBJDIR)/main.o -fopenmp + +clean: + rm $(OBJDIR)/*.o match diff --git a/matchup/Makefile_backup b/matchup/Makefile_backup new file mode 100644 index 0000000..b4c069f --- /dev/null +++ b/matchup/Makefile_backup @@ -0,0 +1,18 @@ +CPP := mpicxx +CPPFLAGS := -O3 +SRCDIR := ./src +OBJDIR := ./obj +INCLUDES := -I $(SRCDIR) -I/home/prlarsen/trunk_steve/trunk/genericio -I/home/prlarsen/trunk_steve/trunk/cosmotools/algorithms/halofinder -I/home/prlarsen/trunk_steve/trunk/cosmotools/algorithms/mergertree_ct -I/home/prlarsen/trunk_steve/trunk/cosmotools/common +OBJECTS := $(OBJDIR)/main.o + + +#linking +match : $(OBJECTS) $(LIBS) + $(CPP) $(CPPFLAGS) $(OBJECTS) -L/home/prlarsen/trunk_steve/trunk/cooley.cpu/mpi/lib -lCosmoTools -lGenericIOMPI -o match -fopenmp + +#compilation +$(OBJDIR)/main.o: $(SRCDIR)/main.cpp + $(CPP) $(CPPFLAGS) $(INCLUDES) -c $(SRCDIR)/main.cpp -o $(OBJDIR)/main.o -fopenmp + +clean: + rm $(OBJDIR)/*.o match diff --git a/matchup/README b/matchup/README new file mode 100644 index 0000000..ec58ab3 --- /dev/null +++ b/matchup/README @@ -0,0 +1 @@ +mpirun -f $COBALT_NODEFILE -n 64 ./match /projects/LastJourney/prlarsen/halo_lightcone_LJ/output/lcHalos76/lc_intrp_halos.76 /projects/LastJourney/heitmann/L5025/HACC000/analysis/Halos/b0168/Props/STEP77/m000p-77.haloproperties /projects/LastJourney/prlarsen/halo_lightcone_LJ/matched_output/test.gio diff --git a/matchup/backup/Makefile b/matchup/backup/Makefile new file mode 100644 index 0000000..838bf48 --- /dev/null +++ b/matchup/backup/Makefile @@ -0,0 +1,24 @@ +CPP := mpicxx +CPPFLAGS := -O3 +SRCDIR := ./src +OBJDIR := ./obj +INCLUDES := -I $(SRCDIR) -I/home/prlarsen/trunk_steve/trunk/genericio -I/home/prlarsen/trunk_steve/trunk/cosmotools/algorithms/halofinder -I/home/prlarsen/trunk_steve/trunk/cosmotools/algorithms/mergertree_ct -I/home/prlarsen/trunk_steve/trunk/cosmotools/common +#INCLUDES := -I $(SRCDIR) -I/home/prlarsen/hacc3/trunk/genericio -I/home/prlarsen/hacc3/trunk/cosmotools/algorithms/halofinder -I/home/prlarsen/hacc3/trunk/cosmotools/algorithms/mergertree_ct -I/home/prlarsen/hacc3/trunk/cosmotools/common +OBJECTS := $(OBJDIR)/main.o +#LIBS := /home/prlarsen/hacc3/trunk/cooley.cpu/mpi/lib/libGenericIOMPI.a +#LIBS := /home/prlarsen/trunk_steve/trunk/cooley.cpu/mpi/lib/libGenericIOMPI.a + + +#linking +match : $(OBJECTS) $(LIBS) + $(CPP) $(CPPFLAGS) $(OBJECTS) -L/home/prlarsen/trunk_steve/trunk/cooley.cpu/mpi/lib -lCosmoTools -lGenericIOMPI -o match -fopenmp +# +# $(CPP) $(CPPFLAGS) $(OBJECTS) -L/home/prlarsen/hacc3/trunk/cooley.cpu/mpi/lib -lCosmoTools -lGenericIOMPI -o match -fopenmp + +#compilation +$(OBJDIR)/main.o: $(SRCDIR)/main.cpp +#$(LIBS) + $(CPP) $(CPPFLAGS) $(INCLUDES) -c $(SRCDIR)/main.cpp -o $(OBJDIR)/main.o -fopenmp + +clean: + rm $(OBJDIR)/*.o match diff --git a/matchup/backup/match b/matchup/backup/match new file mode 100755 index 0000000..35826ab Binary files /dev/null and b/matchup/backup/match differ diff --git a/matchup/cobalt_pl.sh b/matchup/cobalt_pl.sh new file mode 100755 index 0000000..5ffffd9 --- /dev/null +++ b/matchup/cobalt_pl.sh @@ -0,0 +1,33 @@ +#!/bin/sh +NODES=`cat $COBALT_NODEFILE | wc -l` +PROCS=$((NODES*12)) +OMP_NUM_THREADS=1 +export OMP_NUM_THREADS +# probably want to divide this based on memory requirements - 98 is step 42-43, 1 is step 487-499 +for i in {35..36} +# 36.. 80 +#36..98} +#{1..74} +#8} +do +echo $i +i2=`expr $i + 1` +echo $i2 +step=$(tail -n+$i steps_lj.txt | head -n1) +step2=$(tail -n+$i2 steps_lj.txt | head -n1) +echo $step +echo $step2 +#mkdir /projects/LastJourney/prlarsen/halo_lightcone_LJ/output/lcHalos$step2 +mkdir /projects/LastJourney/prlarsen/halo_lightcone_LJ/matched_output2/lcHalos$step2 +LC_IN_FILE=/projects/LastJourney/prlarsen/halo_lightcone_LJ/output/lcHalos$step2/lc_intrp_halos.$step2 +LC_OUT_FILE=/projects/LastJourney/prlarsen/halo_lightcone_LJ/matched_output2/lcHalos$step2/lc_intrp_halos_matched.$step2 +#FOF_FILE=/projects/LastJourney/rangel/LastJourney/MergerTrees_updated_004/m000p-${step}.treenodes +FOF_FILE=/projects/LastJourney/heitmann/L5025/HACC000/analysis/Halos/b0168/Props/STEP${step}/m000p-${step}.haloproperties +echo $LC_IN_FILE +echo $LC_OUT_FILE +echo $FOF_FILE + +mpirun --env LD_PRELOAD=$DARSHAN_PRELOAD -f $COBALT_NODEFILE -n $PROCS /home/prlarsen/lc_codes/rangel/matchup2/match $LC_IN_FILE $FOF_FILE $LC_OUT_FILE + +done + diff --git a/matchup/cobalt_test.sh b/matchup/cobalt_test.sh new file mode 100755 index 0000000..f98796b --- /dev/null +++ b/matchup/cobalt_test.sh @@ -0,0 +1,13 @@ +#!/bin/sh +NODES=`cat $COBALT_NODEFILE | wc -l` +PROCS=$((NODES*12)) + + +mpirun -f $COBALT_NODEFILE -n $PROCS /home/rangel/matchup/match /projects/LastJourney/prlarsen/halo_lightcone_LJ/output/lcHalos76/lc_intrp_halos.76 /projects/LastJourney/heitmann/L5025/HACC000/analysis/Halos/b0168/Props/STEP77/m000p-77.haloproperties /projects/LastJourney/prlarsen/halo_lightcone_LJ/matched_output/a.out + + + +#/projects/LastJourney/rangel/LastJourney/MergerTrees_updated_004/m000p-148.treenodes#6539 + + +#mpirun -f $COBALT_NODEFILE -n $PROCS /home/prlarsen/lc_codes/rangel/matchup2/match /projects/LastJourney/prlarsen/halo_lightcone_LJ/output/lcHalos76/lc_intrp_halos.76 /projects/LastJourney/rangel/LastJourney/MergerTrees_updated_004/m000p-77.treenodes /projects/LastJourney/prlarsen/halo_lightcone_LJ/matched_output/a.out diff --git a/matchup/match b/matchup/match new file mode 100755 index 0000000..907bb30 Binary files /dev/null and b/matchup/match differ diff --git a/matchup/obj/main.o b/matchup/obj/main.o new file mode 100644 index 0000000..a325658 Binary files /dev/null and b/matchup/obj/main.o differ diff --git a/matchup/src/MurmurHashNeutral2.cpp b/matchup/src/MurmurHashNeutral2.cpp new file mode 100644 index 0000000..d66dfef --- /dev/null +++ b/matchup/src/MurmurHashNeutral2.cpp @@ -0,0 +1,49 @@ +//----------------------------------------------------------------------------- +// MurmurHashNeutral2, by Austin Appleby + +// Same as MurmurHash2, but endian- and alignment-neutral. +// Half the speed though, alas. + +unsigned int MurmurHashNeutral2 ( const void * key, int len, unsigned int seed ) +{ + const unsigned int m = 0x5bd1e995; + const int r = 24; + + unsigned int h = seed ^ len; + + const unsigned char * data = (const unsigned char *)key; + + while(len >= 4) + { + unsigned int k; + + k = data[0]; + k |= data[1] << 8; + k |= data[2] << 16; + k |= data[3] << 24; + + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + data += 4; + len -= 4; + } + + switch(len) + { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; + h *= m; + }; + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} diff --git a/matchup/src/main.cpp b/matchup/src/main.cpp new file mode 100644 index 0000000..53d440c --- /dev/null +++ b/matchup/src/main.cpp @@ -0,0 +1,410 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // std::sort +#include +#include +#include + +// Generic IO +#include "GenericIO.h" +#include "Partition.h" + +#include "HaloDistribute_mt.h" +#include "Halos.h" + +#include "MurmurHashNeutral2.cpp" + +// Cosmotools +using namespace std; +using namespace gio; +using namespace cosmotk; + +//NEW +Halos H0; +cosmotk::HaloDistribute HD(&H0); + +typedef struct lc_halo { + float posvel_a_m[7]; + int rr[1]; + int64_t id; + unsigned int destination_rank; +} lc_halo; + +typedef struct fof_halo { + float mass; + int64_t id; + unsigned int destination_rank; +} fof_halo; + +bool comp_by_lc_dest(const lc_halo &a, const lc_halo &b) { + return a.destination_rank < b.destination_rank; +} + +//bool comp_by_fof_dest(const fof_halo &a, const fof_halo &b) { +bool comp_by_fof_dest(const halo_properties_t &a, const halo_properties_t &b) { + return a.rank < b.rank; +} + + +//bool comp_by_fof_id(const fof_halo &a, const fof_halo &b) { +bool comp_by_fof_id(const halo_properties_t &a, const halo_properties_t &b) { + return a.fof_halo_tag < b.fof_halo_tag; +} + +struct IO_Buffers { // this is the order of lc_halo struct above + + // LC halo data + vector x; + vector y; + vector z; + vector vx; + vector vy; + vector vz; + vector a; + vector replication; + vector id; + double box_size[3]; + double origin[3]; + + // +}; + +IO_Buffers IOB; + +void clear_IO_buffers() { + IOB.x.clear(); + IOB.y.clear(); + IOB.z.clear(); + IOB.vx.clear(); + IOB.vy.clear(); + IOB.vz.clear(); + IOB.a.clear(); + IOB.replication.clear(); + IOB.id.clear(); +} + +inline unsigned int tag_to_rank(int64_t fof_tag, int n_ranks) { + return MurmurHashNeutral2((void*)(&fof_tag),sizeof(int64_t),0) % n_ranks; +} + +void read_lc_file(string file_name) { + GenericIO GIO(Partition::getComm(),file_name,GenericIO::FileIOMPI); + GIO.openAndReadHeader(GenericIO::MismatchRedistribute); + size_t num_elems = GIO.readNumElems(); + GIO.readPhysScale(IOB.box_size); + GIO.readPhysOrigin(IOB.origin); + IOB.x.resize(num_elems + GIO.requestedExtraSpace()/sizeof(float)); + IOB.y.resize(num_elems + GIO.requestedExtraSpace()/sizeof(float)); + IOB.z.resize(num_elems + GIO.requestedExtraSpace()/sizeof(float)); + IOB.vx.resize(num_elems + GIO.requestedExtraSpace()/sizeof(float)); + IOB.vy.resize(num_elems + GIO.requestedExtraSpace()/sizeof(float)); + IOB.vz.resize(num_elems + GIO.requestedExtraSpace()/sizeof(float)); + IOB.a.resize(num_elems + GIO.requestedExtraSpace()/sizeof(float)); + IOB.replication.resize(num_elems + GIO.requestedExtraSpace()/sizeof(int)); + IOB.id.resize(num_elems + GIO.requestedExtraSpace()/sizeof(int64_t)); + GIO.addVariable("x", IOB.x, true); + GIO.addVariable("y", IOB.y, true); + GIO.addVariable("z", IOB.z, true); + GIO.addVariable("vx", IOB.vx, true); + GIO.addVariable("vy", IOB.vy, true); + GIO.addVariable("vz", IOB.vz, true); + GIO.addVariable("a", IOB.a, true); + GIO.addVariable("replication", IOB.replication, true); + GIO.addVariable("id", IOB.id, true); + GIO.readData(); + IOB.x.resize(num_elems); + IOB.y.resize(num_elems); + IOB.z.resize(num_elems); + IOB.vx.resize(num_elems); + IOB.vy.resize(num_elems); + IOB.vz.resize(num_elems); + IOB.a.resize(num_elems); + IOB.replication.resize(num_elems); + IOB.id.resize(num_elems); +} + + +void write_lc_file(string file_name) { + GenericIO GIO(Partition::getComm(), file_name); + GIO.setNumElems(IOB.x.size()); + GIO.setPhysOrigin(IOB.origin[0]); + GIO.setPhysScale(IOB.box_size[0]); + GIO.addVariable("x",IOB.x); + GIO.addVariable("y",IOB.y); + GIO.addVariable("z",IOB.z); + GIO.addVariable("vx",IOB.vx); + GIO.addVariable("vy",IOB.vy); + GIO.addVariable("vz",IOB.vz); + GIO.addVariable("a",IOB.a); + GIO.addVariable("replication",IOB.replication); + + + GIO.addVariable("fof_halo_tag", *(H0.fof_halo_tag)); + //GIO.addVariable("fof_halo_count", *(H0.fof_halo_count)); + + //if (H0.has_sod) + //GIO.addVariable("sod_halo_count", *(H0.sod_halo_count)); + + for (int i=0; i &fof_halo_recv) { + halo_properties_t tmp; + tmp.fof_halo_tag = tag; + vector::iterator item = lower_bound(fof_halo_recv.begin(),fof_halo_recv.end(), tmp, comp_by_fof_id); + if (item!=fof_halo_recv.end() && !comp_by_fof_id(tmp, *item)){ + return *item; + } + else{ + cout << "warning - halo not found " << endl; + return tmp; +} +} + +int main( int argc, char** argv ) { + MPI_Init( &argc, &argv ); + Partition::initialize(); + GenericIO::setNaturalDefaultPartition(); + + int rank, n_ranks; + rank = Partition::getMyProc(); + n_ranks = Partition::getNumProc(); + + string lc_file = string(argv[1]); + string fof_file = string(argv[2]); + string lc_out_file = string(argv[3]); + + + + + +#ifdef _OPENMP + + + + if (rank==0){ + int nthreads, tid; + + + +#pragma omp parallel private(nthreads, tid) + { + + /* Obtain thread number */ + tid = omp_get_thread_num(); + printf("Hello World from thread = %d\n", tid); + + /* Only master thread does this */ + if (tid == 0) + { + nthreads = omp_get_num_threads(); + printf("Number of threads = %d\n", nthreads); + } + + } /* All threads join master thread and disband */ + + + printf("OpenMP is defined \n"); + } +#endif + + + + //NEW + //HD.set_parameters(redistribute, Program.pad_factor, Simulation.rl, Program.halo_loss_threshold); + HD.set_parameters(true, 0.0, 0.0, 0.0); + HD.initialize(); + + H0.Allocate(); + H0.has_sod = true; + + + // LC HALOS + read_lc_file(lc_file); // read the file + + if (rank == 0) + cout << "Done reading LC" << endl; + + vector lc_halo_send; + vector lc_halo_send_cnt; + lc_halo_send_cnt.resize(n_ranks,0); + for (size_t i=0;i fof_halo_send; + vector fof_halo_send_cnt(n_ranks,0); + + // pack a buffer of halos with tag_to_rank for destinations and redistribute + for (int i=0; i lc_halo_recv_cnt; + lc_halo_recv_cnt.resize(n_ranks,0); + MPI_Alltoall(&lc_halo_send_cnt[0],1,MPI_INT,&lc_halo_recv_cnt[0],1,MPI_INT,Partition::getComm()); + + vector fof_halo_recv_cnt; + fof_halo_recv_cnt.resize(n_ranks,0); + MPI_Alltoall(&fof_halo_send_cnt[0],1,MPI_INT,&fof_halo_recv_cnt[0],1,MPI_INT,Partition::getComm()); + + // each rank now knows how many items it will receive from every other rank + + // calculate the offsets + vector lc_halo_send_off; + lc_halo_send_off.resize(n_ranks,0); + vector fof_halo_send_off; + fof_halo_send_off.resize(n_ranks,0); + vector lc_halo_recv_off; + lc_halo_recv_off.resize(n_ranks,0); + vector fof_halo_recv_off; + fof_halo_recv_off.resize(n_ranks,0); + + lc_halo_send_off[0] = lc_halo_recv_off[0] = 0; + fof_halo_send_off[0] = fof_halo_recv_off[0] = 0; + for (int i=1; i lc_halo_recv; + lc_halo_recv.resize(lc_halo_recv_total); + //vector fof_halo_recv; + vector fof_halo_recv; + fof_halo_recv.resize(fof_halo_recv_total); + MPI_Barrier(Partition::getComm()); + + if (rank == 0) + cout << "About to send data" << endl; + + + // send the actual data + MPI_Alltoallv(&lc_halo_send[0],&lc_halo_send_cnt[0],&lc_halo_send_off[0],lc_halo_type,\ + &lc_halo_recv[0],&lc_halo_recv_cnt[0],&lc_halo_recv_off[0],lc_halo_type,Partition::getComm()); + + MPI_Alltoallv(&fof_halo_send[0],&fof_halo_send_cnt[0],&fof_halo_send_off[0], HD.halo_properties_MPI_Type,\ + &fof_halo_recv[0],&fof_halo_recv_cnt[0],&fof_halo_recv_off[0], HD.halo_properties_MPI_Type, Partition::getComm()); + if (rank == 0) + cout << "About to sort" << endl; + + std::sort(fof_halo_recv.begin(),fof_halo_recv.end(),comp_by_fof_id); + + + clear_IO_buffers(); + H0.Resize(0); + for (int i=0; i