From bdaf557e2ca1b6458894f458923d0efc6272f679 Mon Sep 17 00:00:00 2001 From: Paul-Edouard Sarlin <15985472+sarlinpe@users.noreply.github.com> Date: Mon, 5 Feb 2024 22:27:01 +0100 Subject: [PATCH] Cleanup the logging and helper utils (#33) * Rename functions * Improve readability * Update helpers * Update logging * Make Logging local * Make logging.Level local * Pull changes from COLMAP --- _pyceres/bindings.cc | 11 +- _pyceres/core/bindings.h | 18 +- _pyceres/core/callbacks.h | 2 +- _pyceres/core/cost_functions.h | 19 +- _pyceres/core/covariance.h | 42 ++- _pyceres/core/loss_functions.h | 6 +- _pyceres/core/manifold.h | 14 +- _pyceres/core/problem.h | 4 +- _pyceres/core/solver.h | 507 ++++++++++++++++----------------- _pyceres/core/types.h | 4 +- _pyceres/factors/bindings.h | 2 +- _pyceres/glog.h | 92 ------ _pyceres/helpers.h | 386 +++++++++++++++---------- _pyceres/log_exceptions.h | 125 -------- _pyceres/logging.h | 215 ++++++++++++++ 15 files changed, 750 insertions(+), 697 deletions(-) delete mode 100644 _pyceres/glog.h delete mode 100644 _pyceres/log_exceptions.h create mode 100644 _pyceres/logging.h diff --git a/_pyceres/bindings.cc b/_pyceres/bindings.cc index 64f4644..740ab71 100644 --- a/_pyceres/bindings.cc +++ b/_pyceres/bindings.cc @@ -1,8 +1,8 @@ #include "_pyceres/core/bindings.h" #include "_pyceres/factors/bindings.h" -#include "_pyceres/glog.h" #include "_pyceres/helpers.h" +#include "_pyceres/logging.h" #include #include @@ -12,12 +12,13 @@ namespace py = pybind11; PYBIND11_MODULE(pyceres, m) { - m.doc() = "PyCeres"; + m.doc() = "PyCeres - Python bindings for the Ceres solver."; + m.attr("__version__") = py::str(VERSION_INFO); py::add_ostream_redirect(m, "ostream_redirect"); - init_glog(m); - bind_core(m); + BindLogging(m); + BindCore(m); py::module_ f = m.def_submodule("factors"); - bind_factors(f); + BindFactors(f); } diff --git a/_pyceres/core/bindings.h b/_pyceres/core/bindings.h index 54a2013..872bd44 100644 --- a/_pyceres/core/bindings.h +++ b/_pyceres/core/bindings.h @@ -13,13 +13,13 @@ namespace py = pybind11; -void bind_core(py::module& m) { - init_types(m); - init_callbacks(m); - init_covariance(m); - init_solver(m); - init_loss_functions(m); - init_cost_functions(m); - init_manifold(m); - init_problem(m); +void BindCore(py::module& m) { + BindTypes(m); + BindCallbacks(m); + BindCovariance(m); + BindSolver(m); + BindLossFunctions(m); + BindCostFunctions(m); + BindManifold(m); + BindProblem(m); } diff --git a/_pyceres/core/callbacks.h b/_pyceres/core/callbacks.h index 1c0c0cf..8616d2c 100644 --- a/_pyceres/core/callbacks.h +++ b/_pyceres/core/callbacks.h @@ -42,7 +42,7 @@ class PyIterationCallback : public ceres::IterationCallback { PYBIND11_MAKE_OPAQUE(std::vector); -void init_callbacks(py::module& m) { +void BindCallbacks(py::module& m) { py::class_(m, "IterationCallback") .def(py::init<>()) diff --git a/_pyceres/core/cost_functions.h b/_pyceres/core/cost_functions.h index 234c112..6564a41 100644 --- a/_pyceres/core/cost_functions.h +++ b/_pyceres/core/cost_functions.h @@ -1,7 +1,7 @@ #pragma once #include "_pyceres/helpers.h" -#include "_pyceres/log_exceptions.h" +#include "_pyceres/logging.h" #include #include @@ -12,7 +12,7 @@ namespace py = pybind11; // This allows use to create python based cost functions. class PyCostFunction : public ceres::CostFunction { public: - /* Inherit the constructors */ + // Inherit the constructors. using ceres::CostFunction::CostFunction; using ceres::CostFunction::set_num_residuals; @@ -82,16 +82,15 @@ class PyCostFunction : public ceres::CostFunction { // Mutable so they can be modified by the const function. mutable std::vector> parameters_vec; mutable std::vector> jacobians_vec; - mutable bool cached_flag = false; // Flag used to determine if the vectors - // need to be resized - mutable py::array_t - residuals_wrap; // Buffer to contain the residuals - // pointer - mutable py::str no_copy; // Dummy variable for pybind11 to avoid copy - // copy + // Flag used to determine if the vectors need to be resized. + mutable bool cached_flag = false; + // Buffer to contain the residuals pointer. + mutable py::array_t residuals_wrap; + // Dummy variable for pybind11 to avoid a copy. + mutable py::str no_copy; }; -void init_cost_functions(py::module& m) { +void BindCostFunctions(py::module& m) { py::class_( m, "CostFunction") .def(py::init<>()) diff --git a/_pyceres/core/covariance.h b/_pyceres/core/covariance.h index 6fc97fc..05a9917 100644 --- a/_pyceres/core/covariance.h +++ b/_pyceres/core/covariance.h @@ -1,34 +1,32 @@ #pragma once #include "_pyceres/helpers.h" -#include "_pyceres/log_exceptions.h" +#include "_pyceres/logging.h" #include #include namespace py = pybind11; -void init_covariance(py::module& m) { - using c_options = ceres::Covariance::Options; - auto co = py::class_(m, "CovarianceOptions") - .def(py::init<>()) - .def_property( - "num_threads", - [](const c_options& self) { return self.num_threads; }, - [](c_options& self, int n_threads) { - int effective_n_threads = - GetEffectiveNumThreads(n_threads); - self.num_threads = effective_n_threads; - }) - .def_readwrite("sparse_linear_algebra_library_type", - &c_options::sparse_linear_algebra_library_type) - .def_readwrite("algorithm_type", &c_options::algorithm_type) - .def_readwrite("min_reciprocal_condition_number", - &c_options::min_reciprocal_condition_number) - .def_readwrite("null_space_rank", &c_options::null_space_rank) - .def_readwrite("apply_loss_function", - &c_options::apply_loss_function); - make_dataclass(co); +void BindCovariance(py::module& m) { + using Options = ceres::Covariance::Options; + py::class_ PyOptions(m, "CovarianceOptions"); + PyOptions.def(py::init<>()) + .def_property( + "num_threads", + [](const Options& self) { return self.num_threads; }, + [](Options& self, int n_threads) { + int effective_n_threads = GetEffectiveNumThreads(n_threads); + self.num_threads = effective_n_threads; + }) + .def_readwrite("sparse_linear_algebra_library_type", + &Options::sparse_linear_algebra_library_type) + .def_readwrite("algorithm_type", &Options::algorithm_type) + .def_readwrite("min_reciprocal_condition_number", + &Options::min_reciprocal_condition_number) + .def_readwrite("null_space_rank", &Options::null_space_rank) + .def_readwrite("apply_loss_function", &Options::apply_loss_function); + MakeDataclass(PyOptions); py::class_(m, "Covariance") .def(py::init()) diff --git a/_pyceres/core/loss_functions.h b/_pyceres/core/loss_functions.h index d40fd60..cfeb5bc 100644 --- a/_pyceres/core/loss_functions.h +++ b/_pyceres/core/loss_functions.h @@ -1,7 +1,7 @@ #pragma once #include "_pyceres/helpers.h" -#include "_pyceres/log_exceptions.h" +#include "_pyceres/logging.h" #include @@ -29,7 +29,7 @@ class PyLossFunction : public ceres::LossFunction { if (overload) { overload.operator()(sq_norm, out_arr); } else { - THROW_EXCEPTION(std::runtime_error, " not implemented.") + LOG_FATAL_THROW(std::runtime_error) << " not implemented."; } } @@ -87,7 +87,7 @@ std::shared_ptr CreateLossFunctionFromDict(py::dict dict) { } } -void init_loss_functions(py::module& m) { +void BindLossFunctions(py::module& m) { py::class_>(m, "LossFunction") diff --git a/_pyceres/core/manifold.h b/_pyceres/core/manifold.h index adf48a8..409faeb 100644 --- a/_pyceres/core/manifold.h +++ b/_pyceres/core/manifold.h @@ -1,7 +1,7 @@ #pragma once #include "_pyceres/helpers.h" -#include "_pyceres/log_exceptions.h" +#include "_pyceres/logging.h" #include #include @@ -14,23 +14,25 @@ class PyManifold : public ceres::Manifold { bool Plus(const double* x, const double* delta, double* x_plus_delta) const override { - THROW_EXCEPTION(std::runtime_error, " not implemented."); + LOG_FATAL_THROW(std::runtime_error) << " not implemented."; return true; } bool PlusJacobian(const double* x, double* jacobian) const override { - THROW_EXCEPTION(std::runtime_error, " not implemented."); + LOG_FATAL_THROW(std::runtime_error) << " not implemented."; + return true; } bool Minus(const double* y, const double* x, double* y_minus_x) const override { - THROW_EXCEPTION(std::runtime_error, " not implemented."); + LOG_FATAL_THROW(std::runtime_error) << " not implemented."; return true; } bool MinusJacobian(const double* x, double* jacobian) const override { - THROW_EXCEPTION(std::runtime_error, " not implemented."); + LOG_FATAL_THROW(std::runtime_error) << " not implemented."; + return true; } // Size of x. @@ -56,7 +58,7 @@ class PyManifold : public ceres::Manifold { using namespace ceres; -void init_manifold(py::module& m) { +void BindManifold(py::module& m) { py::class_(m, "Manifold") .def(py::init<>()) .def("ambient_size", &Manifold::AmbientSize) diff --git a/_pyceres/core/problem.h b/_pyceres/core/problem.h index 64eca6e..585071e 100644 --- a/_pyceres/core/problem.h +++ b/_pyceres/core/problem.h @@ -2,7 +2,7 @@ #include "_pyceres/core/wrappers.h" #include "_pyceres/helpers.h" -#include "_pyceres/log_exceptions.h" +#include "_pyceres/logging.h" #include #include @@ -26,7 +26,7 @@ std::unique_ptr CreatePythonProblem() { return std::unique_ptr(new ceres::Problem(options)); } -void init_problem(py::module& m) { +void BindProblem(py::module& m) { using options = ceres::Problem::Options; py::class_(m, "ProblemOptions") .def(py::init(&CreateProblemOptions)) // Ensures default is that diff --git a/_pyceres/core/solver.h b/_pyceres/core/solver.h index 6df14a8..5eaca60 100644 --- a/_pyceres/core/solver.h +++ b/_pyceres/core/solver.h @@ -1,282 +1,263 @@ #pragma once #include "_pyceres/helpers.h" -#include "_pyceres/log_exceptions.h" +#include "_pyceres/logging.h" #include #include namespace py = pybind11; -void init_solver(py::module& m) { +void BindSolver(py::module& m) { m.def("solve", - overload_cast_()(&ceres::Solve), + py::overload_cast(&ceres::Solve), py::call_guard()); - using s_options = ceres::Solver::Options; - auto so = - py::class_(m, "SolverOptions") - .def(py::init<>()) - .def("IsValid", &s_options::IsValid) - .def_property( - "callbacks", - [](const s_options& self) { return self.callbacks; }, - py::cpp_function( - [](s_options& self, py::list list) { - std::vector callbacks; - for (auto& handle : list) { - self.callbacks.push_back( - handle.cast()); - } - }, - py::keep_alive<1, 2>())) - .def_readwrite("minimizer_type", &s_options::minimizer_type) - .def_readwrite("line_search_direction_type", - &s_options::line_search_direction_type) - .def_readwrite("line_search_type", &s_options::line_search_type) - .def_readwrite("nonlinear_conjugate_gradient_type", - &s_options::nonlinear_conjugate_gradient_type) - .def_readwrite("max_lbfgs_rank", &s_options::max_lbfgs_rank) - .def_readwrite("use_approximate_eigenvalue_bfgs_scaling", - &s_options::use_approximate_eigenvalue_bfgs_scaling) - .def_readwrite("line_search_interpolation_type", - &s_options::line_search_interpolation_type) - .def_readwrite("min_line_search_step_size", - &s_options::min_line_search_step_size) - .def_readwrite("line_search_sufficient_function_decrease", - &s_options::line_search_sufficient_function_decrease) - .def_readwrite("max_line_search_step_contraction", - &s_options::max_line_search_step_contraction) - .def_readwrite("min_line_search_step_contraction", - &s_options::min_line_search_step_contraction) - .def_readwrite("max_num_line_search_step_size_iterations", - &s_options::max_num_line_search_step_size_iterations) - .def_readwrite("max_num_line_search_direction_restarts", - &s_options::max_num_line_search_direction_restarts) - .def_readwrite("line_search_sufficient_curvature_decrease", - &s_options::line_search_sufficient_curvature_decrease) - .def_readwrite("max_line_search_step_expansion", - &s_options::max_line_search_step_expansion) - .def_readwrite("trust_region_strategy_type", - &s_options::trust_region_strategy_type) - .def_readwrite("dogleg_type", &s_options::dogleg_type) - .def_readwrite("use_nonmonotonic_steps", - &s_options::use_nonmonotonic_steps) - .def_readwrite("max_consecutive_nonmonotonic_steps", - &s_options::max_consecutive_nonmonotonic_steps) - .def_readwrite("max_num_iterations", &s_options::max_num_iterations) - .def_readwrite("max_solver_time_in_seconds", - &s_options::max_solver_time_in_seconds) - .def_property( - "num_threads", - [](const s_options& self) { return self.num_threads; }, - [](s_options& self, int n_threads) { - int effective_n_threads = GetEffectiveNumThreads(n_threads); - self.num_threads = effective_n_threads; + using Options = ceres::Solver::Options; + py::class_ PyOptions(m, "SolverOptions"); + PyOptions.def(py::init<>()) + .def("IsValid", &Options::IsValid) + .def_property( + "callbacks", + [](const Options& self) { return self.callbacks; }, + py::cpp_function( + [](Options& self, py::list list) { + std::vector callbacks; + for (auto& handle : list) { + self.callbacks.push_back( + handle.cast()); + } + }, + py::keep_alive<1, 2>())) + .def_readwrite("minimizer_type", &Options::minimizer_type) + .def_readwrite("line_search_direction_type", + &Options::line_search_direction_type) + .def_readwrite("line_search_type", &Options::line_search_type) + .def_readwrite("nonlinear_conjugate_gradient_type", + &Options::nonlinear_conjugate_gradient_type) + .def_readwrite("max_lbfgs_rank", &Options::max_lbfgs_rank) + .def_readwrite("use_approximate_eigenvalue_bfgs_scaling", + &Options::use_approximate_eigenvalue_bfgs_scaling) + .def_readwrite("line_search_interpolation_type", + &Options::line_search_interpolation_type) + .def_readwrite("min_line_search_step_size", + &Options::min_line_search_step_size) + .def_readwrite("line_search_sufficient_function_decrease", + &Options::line_search_sufficient_function_decrease) + .def_readwrite("max_line_search_step_contraction", + &Options::max_line_search_step_contraction) + .def_readwrite("min_line_search_step_contraction", + &Options::min_line_search_step_contraction) + .def_readwrite("max_num_line_search_step_size_iterations", + &Options::max_num_line_search_step_size_iterations) + .def_readwrite("max_num_line_search_direction_restarts", + &Options::max_num_line_search_direction_restarts) + .def_readwrite("line_search_sufficient_curvature_decrease", + &Options::line_search_sufficient_curvature_decrease) + .def_readwrite("max_line_search_step_expansion", + &Options::max_line_search_step_expansion) + .def_readwrite("trust_region_strategy_type", + &Options::trust_region_strategy_type) + .def_readwrite("dogleg_type", &Options::dogleg_type) + .def_readwrite("use_nonmonotonic_steps", &Options::use_nonmonotonic_steps) + .def_readwrite("max_consecutive_nonmonotonic_steps", + &Options::max_consecutive_nonmonotonic_steps) + .def_readwrite("max_num_iterations", &Options::max_num_iterations) + .def_readwrite("max_solver_time_in_seconds", + &Options::max_solver_time_in_seconds) + .def_property( + "num_threads", + [](const Options& self) { return self.num_threads; }, + [](Options& self, int n_threads) { + int effective_n_threads = GetEffectiveNumThreads(n_threads); + self.num_threads = effective_n_threads; #if CERES_VERSION_MAJOR < 2 - self.num_linear_solver_threads = effective_n_threads; + self.num_linear_solver_threads = effective_n_threads; #endif // CERES_VERSION_MAJOR - }) - .def_readwrite("initial_trust_region_radius", - &s_options::initial_trust_region_radius) - .def_readwrite("max_trust_region_radius", - &s_options::max_trust_region_radius) - .def_readwrite("min_trust_region_radius", - &s_options::min_trust_region_radius) - .def_readwrite("min_relative_decrease", - &s_options::min_relative_decrease) - .def_readwrite("min_lm_diagonal", &s_options::min_lm_diagonal) - .def_readwrite("max_lm_diagonal", &s_options::max_lm_diagonal) - .def_readwrite("max_num_consecutive_invalid_steps", - &s_options::max_num_consecutive_invalid_steps) - .def_readwrite("function_tolerance", &s_options::function_tolerance) - .def_readwrite("gradient_tolerance", &s_options::gradient_tolerance) - .def_readwrite("parameter_tolerance", &s_options::parameter_tolerance) - .def_readwrite("linear_solver_type", &s_options::linear_solver_type) - .def_readwrite("preconditioner_type", &s_options::preconditioner_type) - .def_readwrite("visibility_clustering_type", - &s_options::visibility_clustering_type) - .def_readwrite("dense_linear_algebra_library_type", - &s_options::dense_linear_algebra_library_type) - .def_readwrite("sparse_linear_algebra_library_type", - &s_options::sparse_linear_algebra_library_type) - // .def_readwrite("num_linear_solver_threads", - // &s_options::num_linear_solver_threads) - .def_readwrite("use_explicit_schur_complement", - &s_options::use_explicit_schur_complement) - .def_readwrite("dynamic_sparsity", &s_options::dynamic_sparsity) - .def_readwrite("use_inner_iterations", - &s_options::use_inner_iterations) - .def_readwrite("inner_iteration_tolerance", - &s_options::inner_iteration_tolerance) - .def_readwrite("min_linear_solver_iterations", - &s_options::min_linear_solver_iterations) - .def_readwrite("max_linear_solver_iterations", - &s_options::max_linear_solver_iterations) - .def_readwrite("eta", &s_options::eta) - .def_readwrite("jacobi_scaling", &s_options::jacobi_scaling) - .def_readwrite("logging_type", &s_options::logging_type) - .def_readwrite("minimizer_progress_to_stdout", - &s_options::minimizer_progress_to_stdout) - .def_readwrite("trust_region_problem_dump_directory", - &s_options::trust_region_problem_dump_directory) - .def_readwrite("trust_region_problem_dump_format_type", - &s_options::trust_region_problem_dump_format_type) - .def_readwrite("check_gradients", &s_options::check_gradients) - .def_readwrite("gradient_check_relative_precision", - &s_options::gradient_check_relative_precision) - .def_readwrite( - "gradient_check_numeric_derivative_relative_step_size", - &s_options::gradient_check_numeric_derivative_relative_step_size) - .def_readwrite("update_state_every_iteration", - &s_options::update_state_every_iteration); - make_dataclass(so); + }) + .def_readwrite("initial_trust_region_radius", + &Options::initial_trust_region_radius) + .def_readwrite("max_trust_region_radius", + &Options::max_trust_region_radius) + .def_readwrite("min_trust_region_radius", + &Options::min_trust_region_radius) + .def_readwrite("min_relative_decrease", &Options::min_relative_decrease) + .def_readwrite("min_lm_diagonal", &Options::min_lm_diagonal) + .def_readwrite("max_lm_diagonal", &Options::max_lm_diagonal) + .def_readwrite("max_num_consecutive_invalid_steps", + &Options::max_num_consecutive_invalid_steps) + .def_readwrite("function_tolerance", &Options::function_tolerance) + .def_readwrite("gradient_tolerance", &Options::gradient_tolerance) + .def_readwrite("parameter_tolerance", &Options::parameter_tolerance) + .def_readwrite("linear_solver_type", &Options::linear_solver_type) + .def_readwrite("preconditioner_type", &Options::preconditioner_type) + .def_readwrite("visibility_clustering_type", + &Options::visibility_clustering_type) + .def_readwrite("dense_linear_algebra_library_type", + &Options::dense_linear_algebra_library_type) + .def_readwrite("sparse_linear_algebra_library_type", + &Options::sparse_linear_algebra_library_type) + // .def_readwrite("num_linear_solver_threads", + // &Options::num_linear_solver_threads) + .def_readwrite("use_explicit_schur_complement", + &Options::use_explicit_schur_complement) + .def_readwrite("dynamic_sparsity", &Options::dynamic_sparsity) + .def_readwrite("use_inner_iterations", &Options::use_inner_iterations) + .def_readwrite("inner_iteration_tolerance", + &Options::inner_iteration_tolerance) + .def_readwrite("min_linear_solver_iterations", + &Options::min_linear_solver_iterations) + .def_readwrite("max_linear_solver_iterations", + &Options::max_linear_solver_iterations) + .def_readwrite("eta", &Options::eta) + .def_readwrite("jacobi_scaling", &Options::jacobi_scaling) + .def_readwrite("logging_type", &Options::logging_type) + .def_readwrite("minimizer_progress_to_stdout", + &Options::minimizer_progress_to_stdout) + .def_readwrite("trust_region_problem_dump_directory", + &Options::trust_region_problem_dump_directory) + .def_readwrite("trust_region_problem_dump_format_type", + &Options::trust_region_problem_dump_format_type) + .def_readwrite("check_gradients", &Options::check_gradients) + .def_readwrite("gradient_check_relative_precision", + &Options::gradient_check_relative_precision) + .def_readwrite( + "gradient_check_numeric_derivative_relative_step_size", + &Options::gradient_check_numeric_derivative_relative_step_size) + .def_readwrite("update_state_every_iteration", + &Options::update_state_every_iteration); + MakeDataclass(PyOptions); - using s_summary = ceres::Solver::Summary; - auto summary = - py::class_(m, "SolverSummary") - .def(py::init<>()) - .def("BriefReport", &s_summary::BriefReport) - .def("FullReport", &s_summary::FullReport) - .def("IsSolutionUsable", &s_summary::IsSolutionUsable) - .def_readwrite("minimizer_type", &s_summary::minimizer_type) - .def_readwrite("termination_type", &s_summary::termination_type) - .def_readwrite("message", &s_summary::message) - .def_readwrite("initial_cost", &s_summary::initial_cost) - .def_readwrite("final_cost", &s_summary::final_cost) - .def_readwrite("fixed_cost", &s_summary::fixed_cost) - .def_readwrite("num_successful_steps", - &s_summary::num_successful_steps) - .def_readwrite("num_unsuccessful_steps", - &s_summary::num_unsuccessful_steps) - .def_readwrite("num_inner_iteration_steps", - &s_summary::num_inner_iteration_steps) - .def_readwrite("num_line_search_steps", - &s_summary::num_line_search_steps) - .def_readwrite("preprocessor_time_in_seconds", - &s_summary::preprocessor_time_in_seconds) - .def_readwrite("minimizer_time_in_seconds", - &s_summary::minimizer_time_in_seconds) - .def_readwrite("postprocessor_time_in_seconds", - &s_summary::postprocessor_time_in_seconds) - .def_readwrite("total_time_in_seconds", - &s_summary::total_time_in_seconds) - .def_readwrite("linear_solver_time_in_seconds", - &s_summary::linear_solver_time_in_seconds) - .def_readwrite("num_linear_solves", &s_summary::num_linear_solves) - .def_readwrite("residual_evaluation_time_in_seconds", - &s_summary::residual_evaluation_time_in_seconds) - .def_readwrite("num_residual_evaluations", - &s_summary::num_residual_evaluations) - .def_readwrite("jacobian_evaluation_time_in_seconds", - &s_summary::jacobian_evaluation_time_in_seconds) - .def_readwrite("num_jacobian_evaluations", - &s_summary::num_jacobian_evaluations) - .def_readwrite("inner_iteration_time_in_seconds", - &s_summary::inner_iteration_time_in_seconds) - .def_readwrite( - "line_search_cost_evaluation_time_in_seconds", - &s_summary::line_search_cost_evaluation_time_in_seconds) - .def_readwrite( - "line_search_gradient_evaluation_time_in_seconds", - &s_summary::line_search_gradient_evaluation_time_in_seconds) - .def_readwrite( - "line_search_polynomial_minimization_time_in_seconds", - &s_summary::line_search_polynomial_minimization_time_in_seconds) - .def_readwrite("line_search_total_time_in_seconds", - &s_summary::line_search_total_time_in_seconds) - .def_readwrite("num_parameter_blocks", - &s_summary::num_parameter_blocks) - .def_readwrite("num_parameters", &s_summary::num_parameters) - .def_readwrite("num_effective_parameters", - &s_summary::num_effective_parameters) - .def_readwrite("num_residual_blocks", &s_summary::num_residual_blocks) - .def_readwrite("num_residuals", &s_summary::num_residuals) - .def_readwrite("num_parameter_blocks_reduced", - &s_summary::num_parameter_blocks_reduced) - .def_readwrite("num_parameters_reduced", - &s_summary::num_parameters_reduced) - .def_readwrite("num_effective_parameters_reduced", - &s_summary::num_effective_parameters_reduced) - .def_readwrite("num_residual_blocks_reduced", - &s_summary::num_residual_blocks_reduced) - .def_readwrite("num_residuals_reduced", - &s_summary::num_residuals_reduced) - .def_readwrite("is_constrained", &s_summary::is_constrained) - .def_readwrite("num_threads_given", &s_summary::num_threads_given) - .def_readwrite("num_threads_used", &s_summary::num_threads_used) + using Summary = ceres::Solver::Summary; + py::class_ PySummary(m, "SolverSummary"); + PySummary.def(py::init<>()) + .def("BriefReport", &Summary::BriefReport) + .def("FullReport", &Summary::FullReport) + .def("IsSolutionUsable", &Summary::IsSolutionUsable) + .def_readwrite("minimizer_type", &Summary::minimizer_type) + .def_readwrite("termination_type", &Summary::termination_type) + .def_readwrite("message", &Summary::message) + .def_readwrite("initial_cost", &Summary::initial_cost) + .def_readwrite("final_cost", &Summary::final_cost) + .def_readwrite("fixed_cost", &Summary::fixed_cost) + .def_readwrite("num_successful_steps", &Summary::num_successful_steps) + .def_readwrite("num_unsuccessful_steps", &Summary::num_unsuccessful_steps) + .def_readwrite("num_inner_iteration_steps", + &Summary::num_inner_iteration_steps) + .def_readwrite("num_line_search_steps", &Summary::num_line_search_steps) + .def_readwrite("preprocessor_time_in_seconds", + &Summary::preprocessor_time_in_seconds) + .def_readwrite("minimizer_time_in_seconds", + &Summary::minimizer_time_in_seconds) + .def_readwrite("postprocessor_time_in_seconds", + &Summary::postprocessor_time_in_seconds) + .def_readwrite("total_time_in_seconds", &Summary::total_time_in_seconds) + .def_readwrite("linear_solver_time_in_seconds", + &Summary::linear_solver_time_in_seconds) + .def_readwrite("num_linear_solves", &Summary::num_linear_solves) + .def_readwrite("residual_evaluation_time_in_seconds", + &Summary::residual_evaluation_time_in_seconds) + .def_readwrite("num_residual_evaluations", + &Summary::num_residual_evaluations) + .def_readwrite("jacobian_evaluation_time_in_seconds", + &Summary::jacobian_evaluation_time_in_seconds) + .def_readwrite("num_jacobian_evaluations", + &Summary::num_jacobian_evaluations) + .def_readwrite("inner_iteration_time_in_seconds", + &Summary::inner_iteration_time_in_seconds) + .def_readwrite("line_search_cost_evaluation_time_in_seconds", + &Summary::line_search_cost_evaluation_time_in_seconds) + .def_readwrite("line_search_gradient_evaluation_time_in_seconds", + &Summary::line_search_gradient_evaluation_time_in_seconds) + .def_readwrite( + "line_search_polynomial_minimization_time_in_seconds", + &Summary::line_search_polynomial_minimization_time_in_seconds) + .def_readwrite("line_search_total_time_in_seconds", + &Summary::line_search_total_time_in_seconds) + .def_readwrite("num_parameter_blocks", &Summary::num_parameter_blocks) + .def_readwrite("num_parameters", &Summary::num_parameters) + .def_readwrite("num_effective_parameters", + &Summary::num_effective_parameters) + .def_readwrite("num_residual_blocks", &Summary::num_residual_blocks) + .def_readwrite("num_residuals", &Summary::num_residuals) + .def_readwrite("num_parameter_blocks_reduced", + &Summary::num_parameter_blocks_reduced) + .def_readwrite("num_parameters_reduced", &Summary::num_parameters_reduced) + .def_readwrite("num_effective_parameters_reduced", + &Summary::num_effective_parameters_reduced) + .def_readwrite("num_residual_blocks_reduced", + &Summary::num_residual_blocks_reduced) + .def_readwrite("num_residuals_reduced", &Summary::num_residuals_reduced) + .def_readwrite("is_constrained", &Summary::is_constrained) + .def_readwrite("num_threads_given", &Summary::num_threads_given) + .def_readwrite("num_threads_used", &Summary::num_threads_used) #if CERES_VERSION_MAJOR < 2 - .def_readwrite("num_linear_solver_threads_given", - &s_summary::num_linear_solver_threads_given) - .def_readwrite("num_linear_solver_threads_used", - &s_summary::num_linear_solver_threads_used) + .def_readwrite("num_linear_solver_threads_given", + &Summary::num_linear_solver_threads_given) + .def_readwrite("num_linear_solver_threads_used", + &Summary::num_linear_solver_threads_used) #endif - .def_readwrite("linear_solver_type_given", - &s_summary::linear_solver_type_given) - .def_readwrite("linear_solver_type_used", - &s_summary::linear_solver_type_used) - .def_readwrite("schur_structure_given", - &s_summary::schur_structure_given) - .def_readwrite("schur_structure_used", - &s_summary::schur_structure_used) - .def_readwrite("inner_iterations_given", - &s_summary::inner_iterations_given) - .def_readwrite("inner_iterations_used", - &s_summary::inner_iterations_used) - .def_readwrite("preconditioner_type_given", - &s_summary::preconditioner_type_given) - .def_readwrite("preconditioner_type_used", - &s_summary::preconditioner_type_used) - .def_readwrite("visibility_clustering_type", - &s_summary::visibility_clustering_type) - .def_readwrite("trust_region_strategy_type", - &s_summary::trust_region_strategy_type) - .def_readwrite("dogleg_type", &s_summary::dogleg_type) - .def_readwrite("dense_linear_algebra_library_type", - &s_summary::dense_linear_algebra_library_type) - .def_readwrite("sparse_linear_algebra_library_type", - &s_summary::sparse_linear_algebra_library_type) - .def_readwrite("line_search_direction_type", - &s_summary::line_search_direction_type) - .def_readwrite("line_search_type", &s_summary::line_search_type) - .def_readwrite("line_search_interpolation_type", - &s_summary::line_search_interpolation_type) - .def_readwrite("nonlinear_conjugate_gradient_type", - &s_summary::nonlinear_conjugate_gradient_type) - .def_readwrite("max_lbfgs_rank", &s_summary::max_lbfgs_rank); - make_dataclass(summary); + .def_readwrite("linear_solver_type_given", + &Summary::linear_solver_type_given) + .def_readwrite("linear_solver_type_used", + &Summary::linear_solver_type_used) + .def_readwrite("schur_structure_given", &Summary::schur_structure_given) + .def_readwrite("schur_structure_used", &Summary::schur_structure_used) + .def_readwrite("inner_iterations_given", &Summary::inner_iterations_given) + .def_readwrite("inner_iterations_used", &Summary::inner_iterations_used) + .def_readwrite("preconditioner_type_given", + &Summary::preconditioner_type_given) + .def_readwrite("preconditioner_type_used", + &Summary::preconditioner_type_used) + .def_readwrite("visibility_clustering_type", + &Summary::visibility_clustering_type) + .def_readwrite("trust_region_strategy_type", + &Summary::trust_region_strategy_type) + .def_readwrite("dogleg_type", &Summary::dogleg_type) + .def_readwrite("dense_linear_algebra_library_type", + &Summary::dense_linear_algebra_library_type) + .def_readwrite("sparse_linear_algebra_library_type", + &Summary::sparse_linear_algebra_library_type) + .def_readwrite("line_search_direction_type", + &Summary::line_search_direction_type) + .def_readwrite("line_search_type", &Summary::line_search_type) + .def_readwrite("line_search_interpolation_type", + &Summary::line_search_interpolation_type) + .def_readwrite("nonlinear_conjugate_gradient_type", + &Summary::nonlinear_conjugate_gradient_type) + .def_readwrite("max_lbfgs_rank", &Summary::max_lbfgs_rank); + MakeDataclass(PySummary); - using it_sum = ceres::IterationSummary; - auto it_summary = - py::class_(m, "IterationSummary") - .def(py::init<>()) - .def_readonly("iteration", &it_sum::iteration) - .def_readonly("step_is_valid", &it_sum::step_is_valid) - .def_readonly("step_is_nonmonotonic", &it_sum::step_is_nonmonotonic) - .def_readonly("step_is_successful", &it_sum::step_is_successful) - .def_readonly("cost", &it_sum::cost) - .def_readonly("cost_change", &it_sum::cost_change) - .def_readonly("gradient_max_norm", &it_sum::gradient_max_norm) - .def_readonly("gradient_norm", &it_sum::gradient_norm) - .def_readonly("step_norm", &it_sum::step_norm) - .def_readonly("relative_decrease", &it_sum::relative_decrease) - .def_readonly("trust_region_radius", &it_sum::trust_region_radius) - .def_readonly("eta", &it_sum::eta) - .def_readonly("step_size", &it_sum::step_size) - .def_readonly("line_search_function_evaluations", - &it_sum::line_search_function_evaluations) - .def_readonly("line_search_gradient_evaluations", - &it_sum::line_search_gradient_evaluations) - .def_readonly("line_search_iterations", - &it_sum::line_search_iterations) - .def_readonly("linear_solver_iterations", - &it_sum::linear_solver_iterations) - .def_readonly("iteration_time_in_seconds", - &it_sum::iteration_time_in_seconds) - .def_readonly("step_solver_time_in_seconds", - &it_sum::step_solver_time_in_seconds) - .def_readonly("cumulative_time_in_seconds", - &it_sum::cumulative_time_in_seconds); + using IterSummary = ceres::IterationSummary; + py::class_ PyIterSummary(m, "IterationSummary"); + PyIterSummary.def(py::init<>()) + .def_readonly("iteration", &IterSummary::iteration) + .def_readonly("step_is_valid", &IterSummary::step_is_valid) + .def_readonly("step_is_nonmonotonic", &IterSummary::step_is_nonmonotonic) + .def_readonly("step_is_successful", &IterSummary::step_is_successful) + .def_readonly("cost", &IterSummary::cost) + .def_readonly("cost_change", &IterSummary::cost_change) + .def_readonly("gradient_max_norm", &IterSummary::gradient_max_norm) + .def_readonly("gradient_norm", &IterSummary::gradient_norm) + .def_readonly("step_norm", &IterSummary::step_norm) + .def_readonly("relative_decrease", &IterSummary::relative_decrease) + .def_readonly("trust_region_radius", &IterSummary::trust_region_radius) + .def_readonly("eta", &IterSummary::eta) + .def_readonly("step_size", &IterSummary::step_size) + .def_readonly("line_search_function_evaluations", + &IterSummary::line_search_function_evaluations) + .def_readonly("line_search_gradient_evaluations", + &IterSummary::line_search_gradient_evaluations) + .def_readonly("line_search_iterations", + &IterSummary::line_search_iterations) + .def_readonly("linear_solver_iterations", + &IterSummary::linear_solver_iterations) + .def_readonly("iteration_time_in_seconds", + &IterSummary::iteration_time_in_seconds) + .def_readonly("step_solver_time_in_seconds", + &IterSummary::step_solver_time_in_seconds) + .def_readonly("cumulative_time_in_seconds", + &IterSummary::cumulative_time_in_seconds); } diff --git a/_pyceres/core/types.h b/_pyceres/core/types.h index 427743b..b9c4fe7 100644 --- a/_pyceres/core/types.h +++ b/_pyceres/core/types.h @@ -1,7 +1,7 @@ #pragma once #include "_pyceres/helpers.h" -#include "_pyceres/log_exceptions.h" +#include "_pyceres/logging.h" #include #include @@ -12,7 +12,7 @@ namespace py = pybind11; -void init_types(py::module& m) { +void BindTypes(py::module& m) { auto ownt = py::enum_(m, "Ownership") .value("DO_NOT_TAKE_OWNERSHIP", ceres::Ownership::DO_NOT_TAKE_OWNERSHIP) diff --git a/_pyceres/factors/bindings.h b/_pyceres/factors/bindings.h index 31e25b7..e2aca1d 100644 --- a/_pyceres/factors/bindings.h +++ b/_pyceres/factors/bindings.h @@ -9,7 +9,7 @@ namespace py = pybind11; -void bind_factors(py::module& m) { +void BindFactors(py::module& m) { m.def( "NormalPrior", [](const Eigen::VectorXd& mean, diff --git a/_pyceres/glog.h b/_pyceres/glog.h deleted file mode 100644 index d63a4c1..0000000 --- a/_pyceres/glog.h +++ /dev/null @@ -1,92 +0,0 @@ -#pragma once - -#include -#include -#include - -#include -#include -#include -#include - -namespace py = pybind11; - -class glog_dummy {}; // dummy class -// Issue #7: Glog version > 0.5.0 requires T=size_t, <= 0.5.0 T=int -template -void PyBindLogStack(const char* data, T size) { - std::chrono::milliseconds timespan(2000); // or whatever - py::scoped_estream_redirect stream( - std::cerr, // std::ostream& - py::module::import("sys").attr("stderr") // Python output - ); - std::this_thread::sleep_for(timespan); - std::this_thread::sleep_for(timespan); - - std::cerr << data << std::endl; - std::cerr << std::endl; - std::cerr - << "ERROR: C++ code terminated. Kernel Died. See log files for details."; - std::cerr << std::endl << std::endl << std::endl; -} - -__attribute__((noreturn)) void PyBindLogTermination() { - std::chrono::milliseconds timespan(2000); // or whatever - py::scoped_estream_redirect stream( - std::cerr, // std::ostream& - py::module::import("sys").attr("stderr") // Python output - ); - std::this_thread::sleep_for(timespan); - std::this_thread::sleep_for(timespan); - - std::cerr << std::endl; - std::cerr - << "ERROR: C++ code terminated. Kernel Died. See log files for details."; - std::cerr << std::endl << std::endl << std::endl; - exit(1); -} - -void init_glog(py::module& m) { - // google::InstallFailureSignalHandler(); - // google::InitGoogleLogging(""); - // google::InstallFailureFunction(&PyBindLogTermination); //Important to warn - // User in jupyter-notebook about FATAL failure (segfault, LOG(FATAL), - // CHECK(), ...) - - py::class_(m, "glog") - .def_property_static( - "minloglevel", - [](py::object) { return FLAGS_minloglevel; }, - [](py::object, int a) { FLAGS_minloglevel = a; }) - .def_property_static( - "stderrthreshold", - [](py::object) { return FLAGS_stderrthreshold; }, - [](py::object, int a) { FLAGS_stderrthreshold = a; }) - .def_property_static( - "log_dir", - [](py::object) { return FLAGS_log_dir; }, - [](py::object, std::string a) { FLAGS_log_dir = a; }) - .def_property_static( - "logtostderr", - [](py::object) { return FLAGS_logtostderr; }, - [](py::object, bool a) { FLAGS_logtostderr = a; }) - .def_property_static( - "alsologtostderr", - [](py::object) { return FLAGS_alsologtostderr; }, - [](py::object, bool a) { FLAGS_alsologtostderr = a; }) - .def("init", - [](std::string path) { - google::ShutdownGoogleLogging(); - google::InitGoogleLogging(path.c_str()); - }) - .def("init", - []() { - google::InstallFailureSignalHandler(); - google::InitGoogleLogging(""); - google::InstallFailureFunction(&PyBindLogTermination); - }) - .def("install_failure_writer", - []() { google::InstallFailureWriter(&PyBindLogStack); }) - .def("install_failure_function", - []() { google::InstallFailureFunction(&PyBindLogTermination); }); -} diff --git a/_pyceres/helpers.h b/_pyceres/helpers.h index f76172d..0022a0b 100644 --- a/_pyceres/helpers.h +++ b/_pyceres/helpers.h @@ -1,7 +1,8 @@ #pragma once -#include "_pyceres/log_exceptions.h" +#include "_pyceres/logging.h" +#include #include #include #include @@ -12,183 +13,256 @@ #include #include +using namespace pybind11::literals; namespace py = pybind11; -template -using overload_cast_ = pybind11::detail::overload_cast_impl; - template -inline T pyStringToEnum(const py::enum_& enm, const std::string& value) { - auto values = enm.attr("__members__").template cast(); - auto strVal = py::str(value); - if (values.contains(strVal)) { - return T(values[strVal].template cast()); +T pyStringToEnum(const py::enum_& enm, const std::string& value) { + const auto values = enm.attr("__members__").template cast(); + const auto str_val = py::str(value); + if (!values.contains(str_val)) { + LOG(FATAL_THROW) << "Invalid string value " << value << " for enum " + << enm.attr("__name__").template cast(); } - std::string msg = - "ERROR: Invalid string value " + value + " for enum " + - std::string(enm.attr("__name__").template cast()); - THROW_EXCEPTION(std::out_of_range, msg.c_str()); - T t; - return t; + return T(values[str_val].template cast()); } template -inline void AddStringToEnumConstructor(py::enum_& enm) { +void AddStringToEnumConstructor(py::enum_& enm) { enm.def(py::init([enm](const std::string& value) { return pyStringToEnum(enm, py::str(value)); // str constructor })); py::implicitly_convertible(); } -template -inline void make_dataclass(py::class_ cls) { - cls.def(py::init([cls](py::dict dict) { - auto self = py::object(cls()); +void UpdateFromDict(py::object& self, const py::dict& dict) { + for (const auto& it : dict) { + if (!py::isinstance(it.first)) { + LOG(FATAL_THROW) << "Dictionary key is not a string: " + << py::str(it.first); + } + const py::str name = py::reinterpret_borrow(it.first); + const py::handle& value = it.second; + const auto attr = self.attr(name); + try { + if (py::hasattr(attr, "mergedict") && py::isinstance(value)) { + attr.attr("mergedict").attr("__call__")(value); + } else { + self.attr(name) = value; + } + } catch (const py::error_already_set& ex) { + if (ex.matches(PyExc_TypeError)) { + // If fail we try bases of the class + const py::list bases = + attr.attr("__class__").attr("__bases__").cast(); + bool success_on_base = false; + for (auto& base : bases) { + try { + self.attr(name) = base(value); + success_on_base = true; + break; + } catch (const py::error_already_set&) { + continue; // We anyway throw afterwards + } + } + if (success_on_base) { + continue; + } + std::stringstream ss; + ss << self.attr("__class__") + .attr("__name__") + .template cast() + << "." << name.template cast() << ": Could not convert " + << py::type::of(value.cast()) + .attr("__name__") + .template cast() + << ": " << py::str(value).template cast() << " to '" + << py::type::of(attr).attr("__name__").template cast() + << "'."; + // We write the err message to give info even if exceptions + // is catched outside, e.g. in function overload resolve + LOG(ERROR) << "Internal TypeError: " << ss.str(); + throw(py::type_error(std::string("Failed to merge dict into class: ") + + "Could not assign " + + name.template cast())); + } else if (ex.matches(PyExc_AttributeError) && + py::str(ex.value()).cast() == + std::string("can't set attribute")) { + std::stringstream ss; + ss << self.attr("__class__") + .attr("__name__") + .template cast() + << "." << name.template cast() << " defined readonly."; + throw py::attribute_error(ss.str()); + } else if (ex.matches(PyExc_AttributeError)) { + LOG(ERROR) << "Internal AttributeError: " + << py::str(ex.value()).cast(); + throw; + } else { + LOG(ERROR) << "Internal Error: " + << py::str(ex.value()).cast(); + throw; + } + } + } +} + +bool AttributeIsFunction(const std::string& name, const py::object& value) { + return (name.find("__") == 0 || name.rfind("__") != std::string::npos || + py::hasattr(value, "__func__") || py::hasattr(value, "__call__")); +} + +std::vector ListObjectAttributes(const py::object& pyself) { + std::vector attributes; + for (const auto& handle : pyself.attr("__dir__")()) { + const py::str attribute = py::reinterpret_borrow(handle); + const auto value = pyself.attr(attribute); + if (AttributeIsFunction(attribute, value)) { + continue; + } + attributes.push_back(attribute); + } + return attributes; +} + +template +py::dict ConvertToDict(const T& self, + std::vector attributes, + const bool recursive) { + const py::object pyself = py::cast(self); + if (attributes.empty()) { + attributes = ListObjectAttributes(pyself); + } + py::dict dict; + for (const auto& attr : attributes) { + const auto value = pyself.attr(attr.c_str()); + if (recursive && py::hasattr(value, "todict")) { + dict[attr.c_str()] = + value.attr("todict").attr("__call__")().template cast(); + } else { + dict[attr.c_str()] = value; + } + } + return dict; +} + +template +std::string CreateSummary(const T& self, bool write_type) { + std::stringstream ss; + auto pyself = py::cast(self); + const std::string prefix = " "; + bool after_subsummary = false; + ss << pyself.attr("__class__").attr("__name__").template cast() + << ":"; + for (auto& handle : pyself.attr("__dir__")()) { + const py::str name = py::reinterpret_borrow(handle); + py::object attribute; + try { + attribute = pyself.attr(name); + } catch (const std::exception& e) { + // Some properties are not valid for some uninitialized objects. + continue; + } + if (AttributeIsFunction(name, attribute)) { + continue; + } + ss << "\n"; + if (!after_subsummary) { + ss << prefix; + } + ss << name.template cast(); + if (py::hasattr(attribute, "summary")) { + std::string summ = attribute.attr("summary") + .attr("__call__")(write_type) + .template cast(); + // NOLINTNEXTLINE(performance-inefficient-string-concatenation) + summ = std::regex_replace(summ, std::regex("\n"), "\n" + prefix); + ss << ": " << summ; + } else { + if (write_type) { + const std::string type_str = + py::str(py::type::of(attribute).attr("__name__")); + ss << ": " << type_str; + after_subsummary = true; + } + std::string value = py::str(attribute); + if (value.length() > 80 && py::hasattr(attribute, "__len__")) { + const int length = attribute.attr("__len__")().template cast(); + value = value.front() + " ... " + std::to_string(length) + + " elements ... " + value.back(); + } + ss << " = " << value; + after_subsummary = false; + } + } + return ss.str(); +} + +template +void AddDefaultsToDocstrings(py::class_ cls) { + auto obj = cls(); + for (auto& handle : obj.attr("__dir__")()) { + const std::string attribute = py::str(handle); + py::object member; + try { + member = obj.attr(attribute.c_str()); + } catch (const std::exception& e) { + // Some properties are not valid for some uninitialized objects. + continue; + } + auto prop = cls.attr(attribute.c_str()); + if (AttributeIsFunction(attribute, member)) { + continue; + } + const auto type_name = py::type::of(member).attr("__name__"); + const std::string doc = + py::str(prop.doc()).cast() + " (" + + type_name.template cast() + + ", default: " + py::str(member).cast() + ")"; + prop.doc() = py::str(doc); + } +} + +template +void MakeDataclass(py::class_ cls, + const std::vector& attributes = {}) { + AddDefaultsToDocstrings(cls); + if (!py::hasattr(cls, "summary")) { + cls.def("summary", &CreateSummary, "write_type"_a = false); + } + cls.def("mergedict", &UpdateFromDict); + cls.def( + "todict", + [attributes](const T& self, const bool recursive) { + return ConvertToDict(self, attributes, recursive); + }, + "recursive"_a = true); + + cls.def(py::init([cls](const py::dict& dict) { + py::object self = cls(); self.attr("mergedict").attr("__call__")(dict); return self.cast(); })); - py::implicitly_convertible(); - - cls.def(py::init([cls](py::kwargs kwargs) { + cls.def(py::init([cls](const py::kwargs& kwargs) { py::dict dict = kwargs.cast(); - auto self = py::object(cls(dict)); - return self.cast(); + return cls(dict).template cast(); })); + py::implicitly_convertible(); py::implicitly_convertible(); - cls.def("mergedict", [cls](py::object& self, py::dict dict) { - for (auto& it : dict) { - try { - if (py::hasattr(self.attr(it.first), "mergedict")) { - self.attr(it.first).attr("mergedict").attr("__call__")(it.second); - } else { - self.attr(it.first) = it.second; - } - } catch (const py::error_already_set& ex) { - if (ex.matches(PyExc_TypeError)) { - // If fail we try bases of the class - py::list bases = self.attr(it.first) - .attr("__class__") - .attr("__bases__") - .cast(); - bool success_on_base = false; - for (auto& base : bases) { - try { - self.attr(it.first) = base(it.second); - success_on_base = true; - break; - } catch (const py::error_already_set& ex) { - continue; // We anyway throw afterwards - } - } - if (success_on_base) { - continue; - } - std::stringstream ss; - ss << cls.attr("__name__").template cast() << "." - << py::str(it.first).template cast() - << ": Could not convert " - << py::type::of(it.second.cast()) - .attr("__name__") - .template cast() - << ": " << py::str(it.second).template cast() - << " to '" - << py::type::of(self.attr(it.first)) - .attr("__name__") - .template cast() - << "'."; - // We write the err message to give info even if exceptions - // is catched outside, e.g. in function overload resolve - std::cerr << "Internal TypeError: " << ss.str() << std::endl; - throw( - py::type_error(std::string("Failed to merge dict into class: ") + - "Could not assign " + - py::str(it.first).template cast())); - } else if (ex.matches(PyExc_AttributeError) && - py::str(ex.value()).cast() == - std::string("can't set attribute")) { - std::stringstream ss; - ss << cls.attr("__name__").template cast() << "." - << py::str(it.first).template cast() - << " defined readonly."; - throw py::attribute_error(ss.str()); - } else if (ex.matches(PyExc_AttributeError)) { - std::cerr << "Internal AttributeError: " - << py::str(ex.value()).cast() << std::endl; - throw; - } else { - throw; - } - } - } - }); + cls.def("__copy__", [](const T& self) { return T(self); }); + cls.def("__deepcopy__", + [](const T& self, const py::dict&) { return T(self); }); - cls.def( - "summary", - [cls](const T& self, bool write_type) { - std::stringstream ss; - auto pyself = py::cast(self); - std::string prefix = " "; - bool after_subsummary = false; - ss << cls.attr("__name__").template cast() << ":\n"; - for (auto& handle : pyself.attr("__dir__")()) { - std::string attribute = py::str(handle); - auto member = pyself.attr(attribute.c_str()); - - if (attribute.find("__") != 0 && - attribute.rfind("__") == std::string::npos && - !py::hasattr(member, "__func__")) { - if (py::hasattr(member, "summary")) { - std::string summ = member.attr("summary") - .attr("__call__")(write_type) - .template cast(); - summ = std::regex_replace(summ, std::regex("\n"), "\n" + prefix); - if (!after_subsummary) { - ss << prefix; - } - ss << attribute << ": " << summ; - after_subsummary = true; - } else { - if (!after_subsummary) { - ss << prefix; - } - ss << attribute; - if (write_type) { - ss << ": " - << py::type::of(member) - .attr("__name__") - .template cast(); - } - ss << " = " << py::str(member).template cast() - << "\n"; - after_subsummary = false; - } - } - } - return ss.str(); + cls.def(py::pickle( + [attributes](const T& self) { + return ConvertToDict(self, attributes, /*recursive=*/false); }, - py::arg("write_type") = false); - - cls.def("todict", [cls](const T& self) { - auto pyself = py::cast(self); - py::dict dict; - for (auto& handle : pyself.attr("__dir__")()) { - std::string attribute = py::str(handle); - auto member = pyself.attr(attribute.c_str()); - if (attribute.find("__") != 0 && - attribute.rfind("__") == std::string::npos && - !py::hasattr(member, "__func__")) { - if (py::hasattr(member, "todict")) { - dict[attribute.c_str()] = member.attr("todict") - .attr("__call__")() - .template cast(); - } else { - dict[attribute.c_str()] = member; - } - } - } - return dict; - }); + [cls](const py::dict& dict) { + py::object self = cls(); + self.attr("mergedict").attr("__call__")(dict); + return self.cast(); + })); } int GetEffectiveNumThreads(const int num_threads) { diff --git a/_pyceres/log_exceptions.h b/_pyceres/log_exceptions.h deleted file mode 100644 index eb1f75d..0000000 --- a/_pyceres/log_exceptions.h +++ /dev/null @@ -1,125 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -namespace py = pybind11; - -template -inline std::string ToString(T msg) { - return std::to_string(msg); -} - -inline std::string ToString(std::string msg) { return msg; } - -inline std::string ToString(const char* msg) { return std::string(msg); } - -inline const char* __ColmapGetConstFileBaseName(const char* file) { - const char* base = strrchr(file, '/'); - if (!base) { - base = strrchr(file, '\\'); - } - return base ? (base + 1) : file; -} - -template -inline T TemplateException(const char* file, const int line, std::string txt) { - std::stringstream ss; - ss << "[" << __ColmapGetConstFileBaseName(file) << ":" << line << "] " << txt; - return T(ss.str()); -} - -inline std::string __GetConditionString(const char* cond_str) { - std::stringstream ss; - ss << "Condition Failed: " << cond_str; - return ss.str(); -} - -inline std::string __GetCheckString(const char* cond_str) { - std::stringstream ss; - ss << "Check Failed: " << cond_str; - return ss.str(); -} - -inline std::string __MergeTwoConstChar(const char* expr1, const char* expr2) { - return (std::string(expr1) + std::string(" ") + expr2); -} - -inline void __ThrowCheckImpl(const char* file, - const int line, - const bool result, - const char* expr_str) { - if (!result) { - throw TemplateException( - file, line, __GetCheckString(expr_str).c_str()); - } -} - -inline void __ThrowCheckImplMsg(const char* file, - const int line, - const bool result, - const char* expr_str, - std::string msg) { - if (!result) { - std::stringstream ss; - ss << expr_str << " : " << msg; - std::string m = ss.str(); - throw TemplateException( - file, line, __GetCheckString(m.c_str())); - } -} - -template -void __ThrowCheckOpImpl(const char* file, - const int line, - const bool result, - const T1& val1, - const T2& val2, - const char* val1_str, - const char* val2_str, - const char* op_str) { - if (!result) { - std::stringstream ss; - ss << val1_str << " " << op_str << " " << val2_str << " (" << val1 - << " vs. " << val2 << ")"; - std::string msg = ss.str(); - throw TemplateException( - file, line, __GetCheckString(msg.c_str())); - } -} - -// Option checker macros. In contrast to glog, this function does not abort the -// program, but simply throws an exception on failure. -#define THROW_EXCEPTION(exception, msg) \ - throw TemplateException(__FILE__, __LINE__, ToString(msg)); - -#define THROW_CUSTOM_CHECK_MSG(condition, exception, msg) \ - if (!condition) \ - throw TemplateException( \ - __FILE__, \ - __LINE__, \ - __GetCheckString(#condition) + std::string(" ") + ToString(msg)); - -#define THROW_CUSTOM_CHECK(condition, exception) \ - if (!condition) \ - throw TemplateException( \ - __FILE__, __LINE__, __GetCheckString(#condition)); - -#define THROW_CHECK(expr) __ThrowCheckImpl(__FILE__, __LINE__, (expr), #expr); - -#define THROW_CHECK_MSG(expr, msg) \ - __ThrowCheckImplMsg(__FILE__, __LINE__, (expr), #expr, ToString(msg)) - -#define THROW_CHECK_OP(name, op, val1, val2) \ - __ThrowCheckOpImpl( \ - __FILE__, __LINE__, (val1 op val2), val1, val2, #val1, #val2, #op); - -#define THROW_CHECK_EQ(val1, val2) THROW_CHECK_OP(_EQ, ==, val1, val2) -#define THROW_CHECK_NE(val1, val2) THROW_CHECK_OP(_NE, !=, val1, val2) -#define THROW_CHECK_LE(val1, val2) THROW_CHECK_OP(_LE, <=, val1, val2) -#define THROW_CHECK_LT(val1, val2) THROW_CHECK_OP(_LT, <, val1, val2) -#define THROW_CHECK_GE(val1, val2) THROW_CHECK_OP(_GE, >=, val1, val2) -#define THROW_CHECK_GT(val1, val2) THROW_CHECK_OP(_GT, >, val1, val2) diff --git a/_pyceres/logging.h b/_pyceres/logging.h new file mode 100644 index 0000000..22f2a13 --- /dev/null +++ b/_pyceres/logging.h @@ -0,0 +1,215 @@ +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +namespace py = pybind11; + +// Issue #7: Glog version > 0.5.0 requires T=size_t, <= 0.5.0 T=int +template +void PyBindLogStack(const char* data, T size) { + std::chrono::milliseconds timespan(2000); // or whatever + py::scoped_estream_redirect stream( + std::cerr, // std::ostream& + py::module::import("sys").attr("stderr") // Python output + ); + std::this_thread::sleep_for(timespan); + std::this_thread::sleep_for(timespan); + + std::cerr << data << std::endl; + std::cerr << std::endl; + std::cerr + << "ERROR: C++ code terminated. Kernel Died. See log files for details."; + std::cerr << std::endl << std::endl << std::endl; +} + +__attribute__((noreturn)) void PyBindLogTermination() { + std::chrono::milliseconds timespan(2000); // or whatever + py::scoped_estream_redirect stream( + std::cerr, // std::ostream& + py::module::import("sys").attr("stderr") // Python output + ); + std::this_thread::sleep_for(timespan); + std::this_thread::sleep_for(timespan); + + std::cerr << std::endl; + std::cerr + << "ERROR: C++ code terminated. Kernel Died. See log files for details."; + std::cerr << std::endl << std::endl << std::endl; + exit(1); +} + +// Alternative checks to throw an exception instead of aborting the program. +// Usage: THROW_CHECK(condition) << message; +// THROW_CHECK_EQ(val1, val2) << message; +// LOG(FATAL_THROW) << message; +// These macros are copied from glog/logging.h and extended to a new severity +// level FATAL_THROW. +#define COMPACT_GOOGLE_LOG_FATAL_THROW \ + LogMessageFatalThrowDefault(__FILE__, __LINE__) + +#define LOG_TO_STRING_FATAL_THROW(message) \ + LogMessageFatalThrowDefault(__FILE__, __LINE__, message) + +#define LOG_FATAL_THROW(exception) \ + LogMessageFatalThrow(__FILE__, __LINE__).stream() + +#define THROW_CHECK(condition) \ + LOG_IF(FATAL_THROW, GOOGLE_PREDICT_BRANCH_NOT_TAKEN(!(condition))) \ + << "Check failed: " #condition " " + +#define THROW_CHECK_OP(name, op, val1, val2) \ + CHECK_OP_LOG(name, op, val1, val2, LogMessageFatalThrowDefault) + +#define THROW_CHECK_EQ(val1, val2) THROW_CHECK_OP(_EQ, ==, val1, val2) +#define THROW_CHECK_NE(val1, val2) THROW_CHECK_OP(_NE, !=, val1, val2) +#define THROW_CHECK_LE(val1, val2) THROW_CHECK_OP(_LE, <=, val1, val2) +#define THROW_CHECK_LT(val1, val2) THROW_CHECK_OP(_LT, <, val1, val2) +#define THROW_CHECK_GE(val1, val2) THROW_CHECK_OP(_GE, >=, val1, val2) +#define THROW_CHECK_GT(val1, val2) THROW_CHECK_OP(_GT, >, val1, val2) + +#define THROW_CHECK_NOTNULL(val) \ + ThrowCheckNotNull(__FILE__, __LINE__, "'" #val "' Must be non NULL", (val)) + +const char* __GetConstFileBaseName(const char* file) { + const char* base = strrchr(file, '/'); + if (!base) { + base = strrchr(file, '\\'); + } + return base ? (base + 1) : file; +} + +inline std::string __MakeExceptionPrefix(const char* file, int line) { + return "[" + std::string(__GetConstFileBaseName(file)) + ":" + + std::to_string(line) + "] "; +} + +template +class LogMessageFatalThrow : public google::LogMessage { + public: + LogMessageFatalThrow(const char* file, int line) + : google::LogMessage(file, line, google::GLOG_ERROR, &message_), + prefix_(__MakeExceptionPrefix(file, line)){}; + LogMessageFatalThrow(const char* file, int line, std::string* message) + : google::LogMessage(file, line, google::GLOG_ERROR, message), + message_(*message), + prefix_(__MakeExceptionPrefix(file, line)){}; + LogMessageFatalThrow(const char* file, + int line, + const google::CheckOpString& result) + : google::LogMessage(file, line, google::GLOG_ERROR, &message_), + prefix_(__MakeExceptionPrefix(file, line)) { + stream() << "Check failed: " << (*result.str_) << " "; + // On LOG(FATAL) glog does not bother cleaning up CheckOpString + // so we do it here. + delete result.str_; + }; + ~LogMessageFatalThrow() noexcept(false) { + Flush(); +#if defined(__cpp_lib_uncaught_exceptions) && \ + (__cpp_lib_uncaught_exceptions >= 201411L) + if (std::uncaught_exceptions() == 0) +#else + if (!std::uncaught_exception()) +#endif + { + throw T(prefix_ + message_); + } + }; + + private: + std::string message_; + std::string prefix_; +}; + +using LogMessageFatalThrowDefault = LogMessageFatalThrow; + +template +T ThrowCheckNotNull(const char* file, int line, const char* names, T&& t) { + if (t == nullptr) { + LogMessageFatalThrowDefault(file, line).stream() << names; + } + return std::forward(t); +} + +struct Logging { + enum class LogSeverity { + GLOG_INFO = google::GLOG_INFO, + GLOG_WARNING = google::GLOG_WARNING, + GLOG_ERROR = google::GLOG_ERROR, + GLOG_FATAL = google::GLOG_FATAL, + }; +}; // dummy class + +std::pair GetPythonCallFrame() { + const auto frame = py::module_::import("sys").attr("_getframe")(0); + const std::string file = py::str(frame.attr("f_code").attr("co_filename")); + const std::string function = py::str(frame.attr("f_code").attr("co_name")); + const int line = py::int_(frame.attr("f_lineno")); + return std::make_pair(file + ":" + function, line); +} + +void BindLogging(py::module& m) { + py::class_ PyLogging(m, "logging", py::module_local()); + PyLogging.def_readwrite_static("minloglevel", &FLAGS_minloglevel) + .def_readwrite_static("stderrthreshold", &FLAGS_stderrthreshold) + .def_readwrite_static("log_dir", &FLAGS_log_dir) + .def_readwrite_static("logtostderr", &FLAGS_logtostderr) + .def_readwrite_static("alsologtostderr", &FLAGS_alsologtostderr) + .def_static( + "set_log_destination", + [](const Logging::LogSeverity severity, const std::string& path) { + google::SetLogDestination( + static_cast(severity), path.c_str()); + }) + .def_static( + "info", + [](const std::string& msg) { + auto frame = GetPythonCallFrame(); + google::LogMessage(frame.first.c_str(), frame.second).stream() + << msg; + }) + .def_static("warning", + [](const std::string& msg) { + auto frame = GetPythonCallFrame(); + google::LogMessage( + frame.first.c_str(), frame.second, google::GLOG_WARNING) + .stream() + << msg; + }) + .def_static("error", + [](const std::string& msg) { + auto frame = GetPythonCallFrame(); + google::LogMessage( + frame.first.c_str(), frame.second, google::GLOG_ERROR) + .stream() + << msg; + }) + .def_static( + "fatal", + [](const std::string& msg) { + auto frame = GetPythonCallFrame(); + google::LogMessageFatal(frame.first.c_str(), frame.second).stream() + << msg; + }) + .def("install_failure_writer", + []() { google::InstallFailureWriter(&PyBindLogStack); }) + .def("install_failure_function", + []() { google::InstallFailureFunction(&PyBindLogTermination); }); + py::enum_(PyLogging, "Level", py::module_local()) + .value("INFO", Logging::LogSeverity::GLOG_INFO) + .value("WARNING", Logging::LogSeverity::GLOG_WARNING) + .value("ERROR", Logging::LogSeverity::GLOG_ERROR) + .value("FATAL", Logging::LogSeverity::GLOG_FATAL) + .export_values(); + google::InitGoogleLogging(""); + google::InstallFailureSignalHandler(); + google::InstallFailureFunction(&PyBindLogTermination); + FLAGS_alsologtostderr = true; +}