Skip to content

Commit

Permalink
Merge branch 'main' of github.com:SlideRuleEarth/sliderule
Browse files Browse the repository at this point in the history
  • Loading branch information
@jpswinski
jpswinski committed Oct 23, 2024
2 parents ef445a1 + 2f7c245 commit ec1b1ea
Show file tree
Hide file tree
Showing 10 changed files with 69 additions and 56 deletions.
6 changes: 3 additions & 3 deletions datasets/gebco/package/GebcoBathyRaster.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ void GebcoBathyRaster::getIndexFile(const OGRGeometry* geo, std::string& file, c
/*----------------------------------------------------------------------------
* findRasters
*----------------------------------------------------------------------------*/
bool GebcoBathyRaster::findRasters(finder_t* finder)
bool GebcoBathyRaster::findRasters(raster_finder_t* finder)
{
const std::vector<OGRFeature*>* flist = finder->featuresList;
const OGRGeometry* geo = finder->geo;
Expand All @@ -86,7 +86,7 @@ bool GebcoBathyRaster::findRasters(finder_t* finder)
if (!rastergeo->Intersects(geo)) continue;

rasters_group_t* rgroup = new rasters_group_t;
rgroup->id = feature->GetFieldAsString("id");
rgroup->featureId = feature->GetFieldAsString("id");
rgroup->gpsTime = getGmtDate(feature, DATE_TAG, rgroup->gmtDate);

const char* dataFile = feature->GetFieldAsString("data_raster");
Expand All @@ -113,7 +113,7 @@ bool GebcoBathyRaster::findRasters(finder_t* finder)
}
rgroup->infovect.shrink_to_fit();

mlog(DEBUG, "Added group: %s with %ld rasters", rgroup->id.c_str(), rgroup->infovect.size());
mlog(DEBUG, "Added group: %s with %ld rasters", rgroup->featureId.c_str(), rgroup->infovect.size());
for(unsigned j = 0; j < rgroup->infovect.size(); j++)
mlog(DEBUG, " %s", rgroup->infovect[j].fileName.c_str());

Expand Down
2 changes: 1 addition & 1 deletion datasets/gebco/package/GebcoBathyRaster.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ class GebcoBathyRaster: public GeoIndexedRaster
~GebcoBathyRaster (void) override;

void getIndexFile (const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points) final;
bool findRasters (finder_t* finder) final;
bool findRasters (raster_finder_t* finder) final;

/*--------------------------------------------------------------------
* Data
Expand Down
14 changes: 7 additions & 7 deletions datasets/landsat/package/LandsatHlsRaster.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ void LandsatHlsRaster::getIndexFile(const OGRGeometry* geo, std::string& file, c
/*----------------------------------------------------------------------------
* findRasters
*----------------------------------------------------------------------------*/
bool LandsatHlsRaster::findRasters(finder_t* finder)
bool LandsatHlsRaster::findRasters(raster_finder_t* finder)
{
const std::vector<OGRFeature*>* flist = finder->featuresList;
const OGRGeometry* geo = finder->geo;
Expand All @@ -166,9 +166,9 @@ bool LandsatHlsRaster::findRasters(finder_t* finder)

if (!rastergeo->Intersects(geo)) continue;

/* Set raster group time and group id */
/* Set raster group time and group featureId */
rasters_group_t* rgroup = new rasters_group_t;
rgroup->id = feature->GetFieldAsString("id");
rgroup->featureId = feature->GetFieldAsString("id");
rgroup->gpsTime = getGmtDate(feature, DATE_TAG, rgroup->gmtDate);

/* Find each requested band in the index file */
Expand Down Expand Up @@ -207,7 +207,7 @@ bool LandsatHlsRaster::findRasters(finder_t* finder)
}
}

// mlog(DEBUG, "Added group: %s with %ld rasters", rgroup->id.c_str(), rgroup->infovect.size());
// mlog(DEBUG, "Added group: %s with %ld rasters", rgroup->featureId.c_str(), rgroup->infovect.size());
finder->rasterGroups.push_back(rgroup);
}
// mlog(DEBUG, "Found %ld raster groups", finder->rasterGroups.size());
Expand Down Expand Up @@ -274,10 +274,10 @@ uint32_t LandsatHlsRaster::_getGroupSamples(sample_mode_t mode, const rasters_gr
bool isS2 = false;
std::size_t pos;

pos = rgroup->id.find("HLS.L30");
pos = rgroup->featureId.find("HLS.L30");
if(pos != std::string::npos) isL8 = true;

pos = rgroup->id.find("HLS.S30");
pos = rgroup->featureId.find("HLS.S30");
if(pos != std::string::npos) isS2 = true;

if(!isL8 && !isS2)
Expand Down Expand Up @@ -407,7 +407,7 @@ uint32_t LandsatHlsRaster::_getGroupSamples(sample_mode_t mode, const rasters_gr
}

const double groupTime = rgroup->gpsTime / 1000;
const std::string groupName = rgroup->id + " {\"algo\": \"";
const std::string groupName = rgroup->featureId + " {\"algo\": \"";

/* Calculate algos - make sure that all the necessary bands were read */
if(ndsi)
Expand Down
2 changes: 1 addition & 1 deletion datasets/landsat/package/LandsatHlsRaster.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ class LandsatHlsRaster: public GeoIndexedRaster
~LandsatHlsRaster (void) override;

void getIndexFile (const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points) final;
bool findRasters (finder_t* finder) final;
bool findRasters (raster_finder_t* finder) final;

void getGroupSamples (const rasters_group_t* rgroup, List<RasterSample*>& slist, uint32_t flags) final
{ _getGroupSamples(SERIAL, rgroup, &slist, flags);}
Expand Down
2 changes: 1 addition & 1 deletion datasets/pgc/package/PgcDemStripsRaster.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ void PgcDemStripsRaster::getIndexFile(const OGRGeometry* geo, std::string& file,
/*----------------------------------------------------------------------------
* findRasters
*----------------------------------------------------------------------------*/
bool PgcDemStripsRaster::findRasters(finder_t* finder)
bool PgcDemStripsRaster::findRasters(raster_finder_t* finder)
{
const std::vector<OGRFeature*>* flist = finder->featuresList;
const OGRGeometry* geo = finder->geo;
Expand Down
2 changes: 1 addition & 1 deletion datasets/pgc/package/PgcDemStripsRaster.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ class PgcDemStripsRaster: public GeoIndexedRaster
~PgcDemStripsRaster (void) override;
bool getFeatureDate (const OGRFeature* feature, TimeLib::gmt_time_t& gmtDate) final;
void getIndexFile (const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points) final;
bool findRasters (finder_t* finder) final;
bool findRasters (raster_finder_t* finder) final;

private:
void _getIndexFile (double lon, double lat, std::string& file);
Expand Down
6 changes: 3 additions & 3 deletions datasets/usgs3dep/package/Usgs3dep1meterDemRaster.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ void Usgs3dep1meterDemRaster::getIndexFile(const OGRGeometry* geo, std::string&
/*----------------------------------------------------------------------------
* findRasters
*----------------------------------------------------------------------------*/
bool Usgs3dep1meterDemRaster::findRasters(finder_t* finder)
bool Usgs3dep1meterDemRaster::findRasters(raster_finder_t* finder)
{
const std::vector<OGRFeature*>* flist = finder->featuresList;
const OGRGeometry* geo = finder->geo;
Expand All @@ -112,7 +112,7 @@ bool Usgs3dep1meterDemRaster::findRasters(finder_t* finder)
if (!rastergeo->Intersects(geo)) continue;

rasters_group_t* rgroup = new rasters_group_t;
rgroup->id = feature->GetFieldAsString("id");
rgroup->featureId = feature->GetFieldAsString("id");
rgroup->gpsTime = getGmtDate(feature, DATE_TAG, rgroup->gmtDate);

const char* fname = feature->GetFieldAsString("url");
Expand All @@ -128,7 +128,7 @@ bool Usgs3dep1meterDemRaster::findRasters(finder_t* finder)
rgroup->infovect.push_back(rinfo);
}

// mlog(DEBUG, "Added group: %s with %ld rasters", rgroup->id.c_str(), rgroup->infovect.size());
// mlog(DEBUG, "Added group: %s with %ld rasters", rgroup->featureId.c_str(), rgroup->infovect.size());
finder->rasterGroups.push_back(rgroup);
}
// mlog(DEBUG, "Found %ld raster groups", finder->rasterGroups.size());
Expand Down
2 changes: 1 addition & 1 deletion datasets/usgs3dep/package/Usgs3dep1meterDemRaster.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ class Usgs3dep1meterDemRaster: public GeoIndexedRaster
~Usgs3dep1meterDemRaster (void) override;

void getIndexFile (const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points) final;
bool findRasters (finder_t* finder) final;
bool findRasters (raster_finder_t* finder) final;

static OGRErr overrideTargetCRS(OGRSpatialReference& target);

Expand Down
10 changes: 5 additions & 5 deletions packages/geo/GeoIndexedRaster.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,9 @@ GeoIndexedRaster::Reader::~Reader (void)
}

/*----------------------------------------------------------------------------
* Finder Constructor
* RasterFinder Constructor
*----------------------------------------------------------------------------*/
GeoIndexedRaster::Finder::Finder (const OGRGeometry* _geo, const std::vector<OGRFeature*>* _featuresList):
GeoIndexedRaster::RasterFinder::RasterFinder (const OGRGeometry* _geo, const std::vector<OGRFeature*>* _featuresList):
geo(_geo),
featuresList(_featuresList)
{
Expand Down Expand Up @@ -749,7 +749,7 @@ bool GeoIndexedRaster::sample(OGRGeometry* geo, int64_t gps, GroupOrdering* grou
/* Query the R-tree with the OGRPoint and get the result features */
geoRtree.query(geo, foundFeatures);

finder_t finder(geo, &foundFeatures);
raster_finder_t finder(geo, &foundFeatures);
if(!findRasters(&finder))
return false;

Expand Down Expand Up @@ -1021,7 +1021,7 @@ void* GeoIndexedRaster::groupsFinderThread(void *param)
}

/* Set finder for the found features */
Finder finder(&ogrPoint, &localFeatures);
RasterFinder finder(&ogrPoint, &localFeatures);

/* Find rasters intersecting with ogrPoint */
gf->obj->findRasters(&finder);
Expand Down Expand Up @@ -1419,7 +1419,7 @@ void GeoIndexedRaster::applySpatialFilter(OGRLayer* layer, const std::vector<poi
}
perfStats.spatialFilterTime = TimeLib::latchtime() - startTime;

mlog(DEBUG, "Features after spatial filter: %lld", layer->GetFeatureCount());
mlog(INFO, "Features after spatial filter: %lld", layer->GetFeatureCount());
mlog(DEBUG, "Spatial filter time: %.3lf", perfStats.spatialFilterTime);
}

Expand Down
79 changes: 46 additions & 33 deletions packages/geo/GeoIndexedRaster.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,14 @@ class GeoIndexedRaster: public RasterObject
* Typedefs
*--------------------------------------------------------------------*/

/* Raster Sample used by batch sampling */
typedef struct PointSample {
OGRPoint point;
int64_t pointIndex;
RasterSample* sample;
std::atomic<bool> sampleReturned;
uint32_t ssErrors;
int64_t pointIndex; // index to the user provided list of points to sample
RasterSample* sample; // sample created by the batch reader thread
std::atomic<bool> sampleReturned;// multiple rasters may share the same sample,
// this flag is used to avoid returning the same sample, if set a copy of the sample is returned
uint32_t ssErrors; // sampling errors

PointSample(const OGRPoint& _point, int64_t _pointIndex):
point(_point), pointIndex(_pointIndex), sample(NULL), sampleReturned(false), ssErrors(SS_NO_ERRORS) {}
Expand All @@ -88,35 +90,40 @@ class GeoIndexedRaster: public RasterObject
} point_sample_t;

struct UniqueRaster;

/** Raster information needed for sampling */
typedef struct RasterInfo {
bool dataIsElevation;
std::string tag;
std::string fileName;
UniqueRaster* uraster; // only used for batch reading
std::string tag; // "Value", "Flags", "Date"
std::string fileName; // Raster file name
UniqueRaster* uraster; // Pointer to the unique raster which contains the sample for this raster

RasterInfo(void): dataIsElevation(false), uraster(NULL) {}
} raster_info_t;

/* Group of rasters belonging to the same geojson stac catalog feature */
typedef struct RaserGroup {
std::string featureId; // stac catalog feature id
std::vector<raster_info_t> infovect; // vector of rasters belonging to the same stac catalog feature
TimeLib::gmt_time_t gmtDate; // feature date (can be computed from start/end dates)
int64_t gpsTime; // feature gps time

RaserGroup(void): gmtDate{0,0,0,0,0,0}, gpsTime(0) {}
} rasters_group_t;

/* Raster and associated points to sample, used by batch sampling */
typedef struct UniqueRaster {
bool dataIsElevation;
const std::string& fileName;
uint64_t fileId;
std::vector<point_sample_t> pointSamples;
uint64_t fileId; // fileDictionary id
std::vector<point_sample_t> pointSamples; // vector of samples for each point in this raster
explicit UniqueRaster(bool _dataIsElevation, const std::string& _fileName):
dataIsElevation(_dataIsElevation), fileName(_fileName), fileId(0) {}
} unique_raster_t;

typedef struct RaserGroup {
std::string id;
std::vector<raster_info_t> infovect;
TimeLib::gmt_time_t gmtDate;
int64_t gpsTime;

RaserGroup(void): gmtDate{0,0,0,0,0,0}, gpsTime(0) {}
} rasters_group_t;

typedef Ordering<rasters_group_t*, unsigned long> GroupOrdering;

/* Batch reader thread info used by batch sampling code */
typedef struct BatchReader {
GeoIndexedRaster* obj;
unique_raster_t* uraster;
Expand All @@ -127,7 +134,7 @@ class GeoIndexedRaster: public RasterObject
~BatchReader(void);
} batch_reader_t;


/* Cache used by serial sampling code */
typedef struct CacheItem {
bool enabled;
RasterSample* sample;
Expand All @@ -136,6 +143,7 @@ class GeoIndexedRaster: public RasterObject
~CacheItem(void) {delete raster;}
} cacheitem_t;

/* Reader thread info used by serial sampling code */
typedef struct Reader {
GeoIndexedRaster* obj;
OGRGeometry* geo;
Expand All @@ -148,31 +156,28 @@ class GeoIndexedRaster: public RasterObject
} reader_t;

typedef RasterObject::range_t range_t;
typedef struct Finder {
const OGRGeometry* geo;
const std::vector<OGRFeature*>* featuresList;
std::vector<rasters_group_t*> rasterGroups;
explicit Finder(const OGRGeometry* geo, const std::vector<OGRFeature*>* _featuresList);
} finder_t;

/* Point and it's asociated group list */
typedef struct {
OGRPoint point;
int64_t pointIndex;
GroupOrdering* groupList;
int64_t pointIndex; // index to the user provided list of points to sample
GroupOrdering* groupList; // all raster groups that intersect with this point
} point_groups_t;

/* Samples collector thread info used by batch sampling code */
typedef struct SampleCollector {
GeoIndexedRaster* obj;
range_t pGroupsRange;
range_t pGroupsRange; // range of point groups to process for this thread
const std::vector<point_groups_t>& pointsGroups;
std::vector<sample_list_t*> slvector;
uint32_t ssErrors;
std::vector<sample_list_t*> slvector; // vecotor of sample lists to be returned to the user
uint32_t ssErrors; // sampling errors
explicit SampleCollector(GeoIndexedRaster* _obj, const std::vector<point_groups_t>& _pointsGroups);
} sample_collector_t;

/* Typedef for the map of raster file name -> set of unique ordered points */
/* Map of raster file name and unique ordered points to be sampled in that raster */
typedef std::unordered_map<std::string, std::set<uint32_t>> raster_points_map_t;

/* GroupsFinder thread info used by batch sampling code */
typedef struct GroupsFinder {
GeoIndexedRaster* obj;
range_t pointsRange;
Expand All @@ -183,6 +188,14 @@ class GeoIndexedRaster: public RasterObject
explicit GroupsFinder (GeoIndexedRaster* _obj, const std::vector<point_info_t>* _points);
} groups_finder_t;

/* Used by GroupsFinder */
typedef struct RasterFinder {
const OGRGeometry* geo;
const std::vector<OGRFeature*>* featuresList; // features to test for intersection with geo
std::vector<rasters_group_t*> rasterGroups; // result raster groups which intersect with geo
explicit RasterFinder(const OGRGeometry* geo, const std::vector<OGRFeature*>* _featuresList);
} raster_finder_t;

/*--------------------------------------------------------------------
* Methods
*--------------------------------------------------------------------*/
Expand Down Expand Up @@ -217,15 +230,15 @@ class GeoIndexedRaster: public RasterObject
bool openGeoIndex (const OGRGeometry* geo, const std::vector<point_info_t>* points);
virtual bool getFeatureDate (const OGRFeature* feature, TimeLib::gmt_time_t& gmtDate);
virtual void getIndexFile (const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points=NULL) = 0;
virtual bool findRasters (finder_t* finder) = 0;
virtual bool findRasters (raster_finder_t* finder) = 0;
void sampleRasters (OGRGeometry* geo);
bool sample (OGRGeometry* geo, int64_t gps, GroupOrdering* groupList);

/*--------------------------------------------------------------------
* Data
*--------------------------------------------------------------------*/

CacheDictionary cache;
CacheDictionary cache; // Used by serial sampling
uint32_t ssErrors;

private:
Expand Down

0 comments on commit ec1b1ea

Please sign in to comment.