Skip to content

Commit

Permalink
fixed coredump in landsat
Browse files Browse the repository at this point in the history
  • Loading branch information
@elidwa
elidwa committed Oct 17, 2024
1 parent 81d0aa2 commit fca672a
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 23 deletions.
3 changes: 1 addition & 2 deletions datasets/landsat/package/LandsatHlsRaster.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -374,9 +374,8 @@ uint32_t LandsatHlsRaster::_getGroupSamples(sample_mode_t mode, const rasters_gr
if(returnBandSample)
{
RasterSample* s;
if(!ps.sampleReturned)
if(!ps.sampleReturned.exchange(true))
{
ps.sampleReturned = true;
s = ps.sample;
}
else
Expand Down
24 changes: 14 additions & 10 deletions packages/geo/GeoIndexedRaster.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -395,9 +395,8 @@ uint32_t GeoIndexedRaster::getBatchGroupSamples(const rasters_group_t* rgroup, L
if(ps.sample == NULL) break;

RasterSample* s;
if(!ps.sampleReturned)
if(!ps.sampleReturned.exchange(true))
{
ps.sampleReturned = true;
s = ps.sample;
}
else
Expand Down Expand Up @@ -1048,7 +1047,7 @@ void* GeoIndexedRaster::groupsFinderThread(void *param)
}
}

mlog(DEBUG, "Found %zu point groups for points range: %u - %u", gf->pointsGroups.size(), start, end);
mlog(DEBUG, "Found %zu point groups for range: %u - %u", gf->pointsGroups.size(), start, end);

/* Thread must initialize GEOS context */
GeoRtree::deinit(threadGeosContext);
Expand All @@ -1066,7 +1065,7 @@ void* GeoIndexedRaster::samplesCollectThread(void* param)
const uint32_t start = sc->pGroupsRange.start;
const uint32_t end = sc->pGroupsRange.end;

mlog(DEBUG, "Finding samples for range: %u - %u", start, end);
mlog(DEBUG, "Collecting samples for range: %u - %u", start, end);

u_int32_t numSamples = 0;
for(uint32_t pointIndx = start; pointIndx < end; pointIndx++)
Expand Down Expand Up @@ -1099,7 +1098,7 @@ void* GeoIndexedRaster::samplesCollectThread(void* param)
sc->slvector.push_back(slist);
}

mlog(DEBUG, "Found %u samples for points range: %u - %u", numSamples, start, end);
mlog(DEBUG, "Collected %u samples for range: %u - %u", numSamples, start, end);

return NULL;
}
Expand Down Expand Up @@ -1395,7 +1394,7 @@ OGRGeometry* GeoIndexedRaster::getConvexHull(const std::vector<point_info_t>* po
*----------------------------------------------------------------------------*/
void GeoIndexedRaster::applySpatialFilter(OGRLayer* layer, const std::vector<point_info_t>* points)
{
mlog(INFO, "Features before spatial filter: %lld", layer->GetFeatureCount());
mlog(DEBUG, "Features before spatial filter: %lld", layer->GetFeatureCount());

const double startTime = TimeLib::latchtime();

Expand All @@ -1414,8 +1413,8 @@ void GeoIndexedRaster::applySpatialFilter(OGRLayer* layer, const std::vector<poi
}
perfStats.spatialFilterTime = TimeLib::latchtime() - startTime;

mlog(INFO, "Features after spatial filter: %lld", layer->GetFeatureCount());
mlog(INFO, "Spatial filter time: %.3lf", perfStats.spatialFilterTime);
mlog(DEBUG, "Features after spatial filter: %lld", layer->GetFeatureCount());
mlog(DEBUG, "Spatial filter time: %.3lf", perfStats.spatialFilterTime);
}

/*----------------------------------------------------------------------------
Expand Down Expand Up @@ -1568,7 +1567,7 @@ bool GeoIndexedRaster::findUniqueRasters(std::vector<unique_raster_t*>& uniqueRa
for(const uint32_t pointIndx : it->second)
{
const point_groups_t& pg = pointsGroups[pointIndx];
ur->pointSamples.push_back({ pg.point, pg.pointIndex, NULL, false, SS_NO_ERRORS });
ur->pointSamples.emplace_back(pg.point, pg.pointIndex);
}
ur->pointSamples.shrink_to_fit();
}
Expand Down Expand Up @@ -1682,6 +1681,7 @@ bool GeoIndexedRaster::sampleUniqueRasters(const std::vector<unique_raster_t*>&
breader->sync.unlock();
}

mlog(DEBUG, "Done Sampling %u rasters", numRasters);
status = true;
}
catch(const RunTimeException& e)
Expand All @@ -1703,7 +1703,6 @@ bool GeoIndexedRaster::collectSamples(const std::vector<point_groups_t>& pointsG
if(!sampling()) return true;

const double start = TimeLib::latchtime();
mlog(DEBUG, "Populating sllist with samples");

/* Sanity check for pointsGroups, internal pointIndex should be the same as the index in pointsGroups vector */
assert(std::all_of(pointsGroups.cbegin(), pointsGroups.cend(),
Expand All @@ -1720,6 +1719,8 @@ bool GeoIndexedRaster::collectSamples(const std::vector<point_groups_t>& pointsG
getThreadsRanges(pGroupRanges, pointsGroups.size(), minPointGroupsPerThread, numMaxThreads);
const uint32_t numThreads = pGroupRanges.size();

mlog(INFO, "Collecting samples for %zu points with %u threads", pointsGroups.size(), numThreads);

for(uint32_t i = 0; i < numThreads; i++)
{
SampleCollector* sc = new SampleCollector(this, pointsGroups);
Expand All @@ -1736,6 +1737,8 @@ bool GeoIndexedRaster::collectSamples(const std::vector<point_groups_t>& pointsG
}

/* Merge sample lists from all sample collection threads */
const double mergeStart = TimeLib::latchtime();
mlog(DEBUG, "Merging sample lists");
for(SampleCollector* sc : sampleCollectors)
{
const std::vector<sample_list_t*>& slvector = sc->slvector;
Expand All @@ -1746,6 +1749,7 @@ bool GeoIndexedRaster::collectSamples(const std::vector<point_groups_t>& pointsG
ssErrors |= sc->ssErrors;
delete sc;
}
mlog(DEBUG, "Merged %d sample lists, time: %lf", sllist.length(), TimeLib::latchtime() - mergeStart);

perfStats.collectSamplesTime = TimeLib::latchtime() - start;
mlog(DEBUG, "Populated sllist with %d lists of samples, time: %lf", sllist.length(), perfStats.collectSamplesTime);
Expand Down
13 changes: 10 additions & 3 deletions packages/geo/GeoIndexedRaster.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,12 +73,19 @@ class GeoIndexedRaster: public RasterObject
* Typedefs
*--------------------------------------------------------------------*/

typedef struct {
typedef struct PointSample {
OGRPoint point;
int64_t pointIndex;
int64_t pointIndex;
RasterSample* sample;
bool sampleReturned;
std::atomic<bool> sampleReturned;
uint32_t ssErrors;

PointSample(const OGRPoint& _point, int64_t _pointIndex):
point(_point), pointIndex(_pointIndex), sample(NULL), sampleReturned(false), ssErrors(SS_NO_ERRORS) {}

PointSample(const PointSample& ps):
point(ps.point), pointIndex(ps.pointIndex), sample(ps.sample), sampleReturned(ps.sampleReturned.load()), ssErrors(ps.ssErrors) {}

} point_sample_t;

struct unique_raster_t;
Expand Down
24 changes: 16 additions & 8 deletions packages/geo/RasterObject.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ const struct luaL_Reg RasterObject::LUA_META_TABLE[] = {

Mutex RasterObject::factoryMut;
Dictionary<RasterObject::factory_t> RasterObject::factories;
Mutex RasterObject::fileDictMut;

/******************************************************************************
* PUBLIC METHODS
Expand Down Expand Up @@ -347,12 +348,15 @@ uint64_t RasterObject::fileDictAdd(const string& fileName)
{
uint64_t id;

if(!fileDict.find(fileName.c_str(), &id))
fileDictMut.lock();
{
id = (parms->key_space << 32) | fileDict.length();
fileDict.add(fileName.c_str(), id);
if(!fileDict.find(fileName.c_str(), &id))
{
id = (parms->key_space << 32) | fileDict.length();
fileDict.add(fileName.c_str(), id);
}
}

fileDictMut.unlock();
return id;
}

Expand All @@ -361,12 +365,16 @@ uint64_t RasterObject::fileDictAdd(const string& fileName)
*----------------------------------------------------------------------------*/
const char* RasterObject::fileDictGetFile (uint64_t fileId)
{
Dictionary<uint64_t>::Iterator iterator(fileDict);
for(int i = 0; i < iterator.length; i++)
fileDictMut.lock();
{
if(fileId == iterator[i].value)
return iterator[i].key;
Dictionary<uint64_t>::Iterator iterator(fileDict);
for(int i = 0; i < iterator.length; i++)
{
if(fileId == iterator[i].value)
return iterator[i].key;
}
}
fileDictMut.unlock();
return NULL;
}

Expand Down
1 change: 1 addition & 0 deletions packages/geo/RasterObject.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@ class RasterObject: public LuaObject

static Mutex factoryMut;
static Dictionary<factory_t> factories;
static Mutex fileDictMut;
Dictionary<uint64_t> fileDict;

Mutex readersMut;
Expand Down

0 comments on commit fca672a

Please sign in to comment.