Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf: optimize CSVStat by pre-calculating header column count #249

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

perf: optimize CSVStat by pre-calculating header column count #249

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions include/internal/csv_stat.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ namespace csv {
/** Return current means */
CSV_INLINE std::vector<long double> CSVStat::get_mean() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->rolling_means[i]);
}
return ret;
Expand All @@ -33,7 +33,7 @@ namespace csv {
/** Return current variances */
CSV_INLINE std::vector<long double> CSVStat::get_variance() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->rolling_vars[i]/(this->n[i] - 1));
}
return ret;
Expand All @@ -42,7 +42,7 @@ namespace csv {
/** Return current mins */
CSV_INLINE std::vector<long double> CSVStat::get_mins() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->mins[i]);
}
return ret;
Expand All @@ -51,7 +51,7 @@ namespace csv {
/** Return current maxes */
CSV_INLINE std::vector<long double> CSVStat::get_maxes() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->maxes[i]);
}
return ret;
Expand All @@ -60,7 +60,7 @@ namespace csv {
/** Get counts for each column */
CSV_INLINE std::vector<CSVStat::FreqCount> CSVStat::get_counts() const {
std::vector<FreqCount> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->counts[i]);
}
return ret;
Expand All @@ -69,7 +69,7 @@ namespace csv {
/** Get data type counts for each column */
CSV_INLINE std::vector<CSVStat::TypeCount> CSVStat::get_dtypes() const {
std::vector<TypeCount> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->dtypes[i]);
}
return ret;
Expand All @@ -79,7 +79,7 @@ namespace csv {
/** Only create stats counters the first time **/
if (dtypes.empty()) {
/** Go through all records and calculate specified statistics */
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
dtypes.push_back({});
counts.push_back({});
rolling_means.push_back(0);
Expand All @@ -92,7 +92,7 @@ namespace csv {

// Start threads
std::vector<std::thread> pool;
for (size_t i = 0; i < this->get_col_names().size(); i++)
for (size_t i = 0; i < this->n_cols; i++)
pool.push_back(std::thread(&CSVStat::calc_worker, this, i));

// Block until done
Expand All @@ -105,6 +105,8 @@ namespace csv {
CSV_INLINE void CSVStat::calc() {
constexpr size_t CALC_CHUNK_SIZE = 5000;

this->n_cols = reader.get_col_names().size();

for (auto& row : reader) {
this->records.push_back(std::move(row));

Expand All @@ -128,7 +130,7 @@ namespace csv {
auto current_record = this->records.begin();

for (size_t processed = 0; current_record != this->records.end(); processed++) {
if (current_record->size() == this->get_col_names().size()) {
if (current_record->size() == this->n_cols) {
auto current_field = (*current_record)[i];

// Optimization: Don't count() if there's too many distinct values in the first 1000 rows
Expand Down
1 change: 1 addition & 0 deletions include/internal/csv_stat.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ namespace csv {
void calc_worker(const size_t&);

CSVReader reader;
size_t n_cols;
std::deque<CSVRow> records = {};
};
}
21 changes: 12 additions & 9 deletions single_include/csv.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6494,6 +6494,7 @@ namespace csv {
void calc_worker(const size_t&);

CSVReader reader;
size_t n_cols;
std::deque<CSVRow> records = {};
};
}
Expand Down Expand Up @@ -8261,7 +8262,7 @@ namespace csv {
/** Return current means */
CSV_INLINE std::vector<long double> CSVStat::get_mean() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->rolling_means[i]);
}
return ret;
Expand All @@ -8270,7 +8271,7 @@ namespace csv {
/** Return current variances */
CSV_INLINE std::vector<long double> CSVStat::get_variance() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->rolling_vars[i]/(this->n[i] - 1));
}
return ret;
Expand All @@ -8279,7 +8280,7 @@ namespace csv {
/** Return current mins */
CSV_INLINE std::vector<long double> CSVStat::get_mins() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->mins[i]);
}
return ret;
Expand All @@ -8288,7 +8289,7 @@ namespace csv {
/** Return current maxes */
CSV_INLINE std::vector<long double> CSVStat::get_maxes() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->maxes[i]);
}
return ret;
Expand All @@ -8297,7 +8298,7 @@ namespace csv {
/** Get counts for each column */
CSV_INLINE std::vector<CSVStat::FreqCount> CSVStat::get_counts() const {
std::vector<FreqCount> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->counts[i]);
}
return ret;
Expand All @@ -8306,7 +8307,7 @@ namespace csv {
/** Get data type counts for each column */
CSV_INLINE std::vector<CSVStat::TypeCount> CSVStat::get_dtypes() const {
std::vector<TypeCount> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->dtypes[i]);
}
return ret;
Expand All @@ -8316,7 +8317,7 @@ namespace csv {
/** Only create stats counters the first time **/
if (dtypes.empty()) {
/** Go through all records and calculate specified statistics */
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
dtypes.push_back({});
counts.push_back({});
rolling_means.push_back(0);
Expand All @@ -8329,7 +8330,7 @@ namespace csv {

// Start threads
std::vector<std::thread> pool;
for (size_t i = 0; i < this->get_col_names().size(); i++)
for (size_t i = 0; i < this->n_cols; i++)
pool.push_back(std::thread(&CSVStat::calc_worker, this, i));

// Block until done
Expand All @@ -8342,6 +8343,8 @@ namespace csv {
CSV_INLINE void CSVStat::calc() {
constexpr size_t CALC_CHUNK_SIZE = 5000;

this->n_cols = reader.get_col_names().size();

for (auto& row : reader) {
this->records.push_back(std::move(row));

Expand All @@ -8365,7 +8368,7 @@ namespace csv {
auto current_record = this->records.begin();

for (size_t processed = 0; current_record != this->records.end(); processed++) {
if (current_record->size() == this->get_col_names().size()) {
if (current_record->size() == this->n_cols) {
auto current_field = (*current_record)[i];

// Optimization: Don't count() if there's too many distinct values in the first 1000 rows
Expand Down
21 changes: 12 additions & 9 deletions single_include_test/csv.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6494,6 +6494,7 @@ namespace csv {
void calc_worker(const size_t&);

CSVReader reader;
size_t n_cols;
std::deque<CSVRow> records = {};
};
}
Expand Down Expand Up @@ -8261,7 +8262,7 @@ namespace csv {
/** Return current means */
CSV_INLINE std::vector<long double> CSVStat::get_mean() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->rolling_means[i]);
}
return ret;
Expand All @@ -8270,7 +8271,7 @@ namespace csv {
/** Return current variances */
CSV_INLINE std::vector<long double> CSVStat::get_variance() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->rolling_vars[i]/(this->n[i] - 1));
}
return ret;
Expand All @@ -8279,7 +8280,7 @@ namespace csv {
/** Return current mins */
CSV_INLINE std::vector<long double> CSVStat::get_mins() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->mins[i]);
}
return ret;
Expand All @@ -8288,7 +8289,7 @@ namespace csv {
/** Return current maxes */
CSV_INLINE std::vector<long double> CSVStat::get_maxes() const {
std::vector<long double> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->maxes[i]);
}
return ret;
Expand All @@ -8297,7 +8298,7 @@ namespace csv {
/** Get counts for each column */
CSV_INLINE std::vector<CSVStat::FreqCount> CSVStat::get_counts() const {
std::vector<FreqCount> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->counts[i]);
}
return ret;
Expand All @@ -8306,7 +8307,7 @@ namespace csv {
/** Get data type counts for each column */
CSV_INLINE std::vector<CSVStat::TypeCount> CSVStat::get_dtypes() const {
std::vector<TypeCount> ret;
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
ret.push_back(this->dtypes[i]);
}
return ret;
Expand All @@ -8316,7 +8317,7 @@ namespace csv {
/** Only create stats counters the first time **/
if (dtypes.empty()) {
/** Go through all records and calculate specified statistics */
for (size_t i = 0; i < this->get_col_names().size(); i++) {
for (size_t i = 0; i < this->n_cols; i++) {
dtypes.push_back({});
counts.push_back({});
rolling_means.push_back(0);
Expand All @@ -8329,7 +8330,7 @@ namespace csv {

// Start threads
std::vector<std::thread> pool;
for (size_t i = 0; i < this->get_col_names().size(); i++)
for (size_t i = 0; i < this->n_cols; i++)
pool.push_back(std::thread(&CSVStat::calc_worker, this, i));

// Block until done
Expand All @@ -8342,6 +8343,8 @@ namespace csv {
CSV_INLINE void CSVStat::calc() {
constexpr size_t CALC_CHUNK_SIZE = 5000;

this->n_cols = reader.get_col_names().size();

for (auto& row : reader) {
this->records.push_back(std::move(row));

Expand All @@ -8365,7 +8368,7 @@ namespace csv {
auto current_record = this->records.begin();

for (size_t processed = 0; current_record != this->records.end(); processed++) {
if (current_record->size() == this->get_col_names().size()) {
if (current_record->size() == this->n_cols) {
auto current_field = (*current_record)[i];

// Optimization: Don't count() if there's too many distinct values in the first 1000 rows
Expand Down