Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update arrow, pyo3, and numpy #8618

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
+70 −67
Draft

Update arrow, pyo3, and numpy #8618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b7c0a86
Update arrow, pyo3, and numpy
emilk Jan 8, 2025
a1837c6
Fix last deprecation warning
emilk Jan 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 40 additions & 45 deletions Cargo.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,9 +356,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"

[[package]]
name = "arrow"
version = "53.2.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3"
checksum = "d2ccdcc8fb14508ca20aaec7076032e5c0b0751b906036d4496786e2f227a37a"
dependencies = [
"arrow-arith",
"arrow-array",
Expand All @@ -375,40 +375,39 @@ dependencies = [

[[package]]
name = "arrow-arith"
version = "53.2.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a"
checksum = "a1aad8e27f32e411a0fc0bf5a625a35f0bf9b9f871cf4542abe31f7cef4beea2"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
"half",
"num",
]

[[package]]
name = "arrow-array"
version = "53.2.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8"
checksum = "bd6ed90c28c6f73a706c55799b8cc3a094e89257238e5b1d65ca7c70bd3ae23f"
dependencies = [
"ahash",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"chrono",
"half",
"hashbrown 0.14.5",
"hashbrown 0.15.2",
"num",
]

[[package]]
name = "arrow-buffer"
version = "53.2.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec"
checksum = "fe4a40bdc1552ea10fbdeae4e5a945d8572c32f66bce457b96c13d9c46b80447"
dependencies = [
"bytes",
"half",
Expand All @@ -417,9 +416,9 @@ dependencies = [

[[package]]
name = "arrow-cast"
version = "53.2.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1"
checksum = "430c0a21aa7f81bcf0f97c57216d7127795ea755f494d27bae2bd233be43c2cc"
dependencies = [
"arrow-array",
"arrow-buffer",
Expand All @@ -437,9 +436,9 @@ dependencies = [

[[package]]
name = "arrow-data"
version = "53.2.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5"
checksum = "09af476cfbe9879937e50b1334c73189de6039186e025b1b1ac84b283b87b20e"
dependencies = [
"arrow-buffer",
"arrow-schema",
Expand All @@ -459,26 +458,23 @@ dependencies = [

[[package]]
name = "arrow-ord"
version = "53.2.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52"
checksum = "eddf14c5f03b679ec8ceac4dfac43f63cdc4ed54dab3cc120a4ef46af38481eb"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"half",
"num",
]

[[package]]
name = "arrow-row"
version = "53.2.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be"
checksum = "e9acdc58da19f383f4ba381fa0e3583534ae2ceb31269aaf4a03f08ff13e8443"
dependencies = [
"ahash",
"arrow-array",
"arrow-buffer",
"arrow-data",
Expand All @@ -488,18 +484,18 @@ dependencies = [

[[package]]
name = "arrow-schema"
version = "53.2.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1"
checksum = "3a1822a1a952955637e85e8f9d6b0e04dd75d65492b87ec548dd593d3a1f772b"
dependencies = [
"bitflags 2.6.0",
]

[[package]]
name = "arrow-select"
version = "53.2.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125"
checksum = "5c4172e9a12dfe15303d3926269f9ead471ea93bdd067d113abc65cb6c48e246"
dependencies = [
"ahash",
"arrow-array",
Expand All @@ -511,9 +507,9 @@ dependencies = [

[[package]]
name = "arrow-string"
version = "53.2.0"
version = "54.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c"
checksum = "73683040445f4932342781926189901c9521bb1a787c35dbe628a3ce51372d3c"
dependencies = [
"arrow-array",
"arrow-buffer",
Expand Down Expand Up @@ -3871,7 +3867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4"
dependencies = [
"cfg-if",
"windows-targets 0.48.5",
"windows-targets 0.52.6",
]

[[package]]
Expand Down Expand Up @@ -4494,17 +4490,17 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"

[[package]]
name = "numpy"
version = "0.22.1"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edb929bc0da91a4d85ed6c0a84deaa53d411abfb387fc271124f91bf6b89f14e"
checksum = "b94caae805f998a07d33af06e6a3891e38556051b8045c615470a71590e13e78"
dependencies = [
"libc",
"ndarray",
"num-complex",
"num-integer",
"num-traits",
"pyo3",
"rustc-hash 1.1.0",
"rustc-hash 2.0.0",
]

[[package]]
Expand Down Expand Up @@ -5283,9 +5279,9 @@ dependencies = [

[[package]]
name = "pyo3"
version = "0.22.5"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d922163ba1f79c04bc49073ba7b32fd5a8d3b76a87c955921234b8e77333c51"
checksum = "e484fd2c8b4cb67ab05a318f1fd6fa8f199fcc30819f08f07d200809dba26c15"
dependencies = [
"cfg-if",
"indoc",
Expand All @@ -5301,29 +5297,29 @@ dependencies = [

[[package]]
name = "pyo3-build-config"
version = "0.22.5"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc38c5feeb496c8321091edf3d63e9a6829eab4b863b4a6a65f26f3e9cc6b179"
checksum = "dc0e0469a84f208e20044b98965e1561028180219e35352a2afaf2b942beff3b"
dependencies = [
"once_cell",
"target-lexicon",
]

[[package]]
name = "pyo3-ffi"
version = "0.22.5"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94845622d88ae274d2729fcefc850e63d7a3ddff5e3ce11bd88486db9f1d357d"
checksum = "eb1547a7f9966f6f1a0f0227564a9945fe36b90da5a93b3933fc3dc03fae372d"
dependencies = [
"libc",
"pyo3-build-config",
]

[[package]]
name = "pyo3-macros"
version = "0.22.5"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e655aad15e09b94ffdb3ce3d217acf652e26bbc37697ef012f5e5e348c716e5e"
checksum = "fdb6da8ec6fa5cedd1626c886fc8749bdcbb09424a86461eb8cdf096b7c33257"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
Expand All @@ -5333,9 +5329,9 @@ dependencies = [

[[package]]
name = "pyo3-macros-backend"
version = "0.22.5"
version = "0.23.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1e3f09eecd94618f60a455a23def79f79eba4dc561a97324bf9ac8c6df30ce"
checksum = "38a385202ff5a92791168b1136afae5059d3ac118457bb7bc304c197c2d33e7d"
dependencies = [
"heck 0.5.0",
"proc-macro2",
Expand Down Expand Up @@ -5542,9 +5538,8 @@ dependencies = [

[[package]]
name = "re_arrow2"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f046c5679b0f305d610f80d93fd51ad702cfc077bbe16d9553a1660a2505160"
version = "0.18.1"
source = "git+https://github.com/rerun-io/re_arrow2.git?branch=emilk/arrow-54#ab5ae4a61e7b0c0ec4bd226ec88f6a07fe629234"
dependencies = [
"ahash",
"arrow-array",
Expand Down Expand Up @@ -9788,7 +9783,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.59.0",
]

[[package]]
Expand Down
10 changes: 6 additions & 4 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ ahash = "0.8"
anyhow = { version = "1.0", default-features = false }
argh = "0.1.12"
array-init = "2.1"
arrow = { version = "53.1", default-features = false }
arrow = { version = "54", default-features = false }
arrow2 = { package = "re_arrow2", version = "0.18" }
async-executor = "1.0"
backtrace = "0.3"
Expand Down Expand Up @@ -220,7 +220,7 @@ nohash-hasher = "0.2"
notify = { version = "6.1.1", features = ["macos_kqueue"] }
num-derive = "0.4"
num-traits = "0.2"
numpy = "0.22"
numpy = "0.23"
object_store = { version = "0.10.2" }
once_cell = "1.17" # No lazy_static - use `std::sync::OnceLock` or `once_cell` instead
ordered-float = "4.3.0"
Expand All @@ -239,8 +239,8 @@ prost = "0.13.3"
prost-build = "0.13.3"
puffin = "0.19.1"
puffin_http = "0.16"
pyo3 = "0.22.5"
pyo3-build-config = "0.22.5"
pyo3 = "0.23.3"
pyo3-build-config = "0.23.3"
quote = "1.0"
rand = { version = "0.8", default-features = false }
rand_distr = { version = "0.4", default-features = false }
Expand Down Expand Up @@ -586,3 +586,5 @@ egui_commonmark = { git = "https://github.com/rerun-io/egui_commonmark.git", bra
# walkers = { git = "https://github.com/rerun-io/walkers", rev = "8939cceb3fa49ca8648ee16fe1d8432f5ab0bdcc" } # https://github.com/podusowski/walkers/pull/222

# dav1d = { path = "/home/cmc/dev/rerun-io/rav1d", package = "re_rav1d", version = "0.1.1" }

re_arrow2 = { git = "https://github.com/rerun-io/re_arrow2.git", branch = "emilk/arrow-54" } # https://github.com/rerun-io/re_arrow2/pull/14
28 changes: 17 additions & 11 deletions rerun_py/src/dataframe.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,12 +49,12 @@ pub(crate) fn register(m: &Bound<'_, PyModule>) -> PyResult<()> {
Ok(())
}

fn py_rerun_warn(msg: &str) -> PyResult<()> {
fn py_rerun_warn(msg: &std::ffi::CStr) -> PyResult<()> {
Python::with_gil(|py| {
let warning_type = PyModule::import_bound(py, "rerun")?
let warning_type = PyModule::import(py, "rerun")?
.getattr("error_utils")?
.getattr("RerunWarning")?;
PyErr::warn_bound(py, &warning_type, msg, 0)?;
PyErr::warn(py, &warning_type, msg, 0)?;
Ok(())
})
}
Expand Down Expand Up @@ -388,7 +388,7 @@ impl IndexValuesLike<'_> {
// If any has the `.chunks` attribute, we can try to try each chunk as pyarrow array
if let Ok(chunks) = any.getattr("chunks") {
let mut values = BTreeSet::new();
for chunk in chunks.iter()? {
for chunk in chunks.try_iter()? {
let chunk = chunk?.extract::<PyArrowType<ArrayData>>()?;
let array = make_array(chunk.0.clone());

Expand Down Expand Up @@ -486,13 +486,19 @@ impl PySchema {
.schema
.clone()
.into_iter()
.map(|col| match col {
ColumnDescriptor::Time(col) => PyIndexColumnDescriptor(col).into_py(py),
ColumnDescriptor::Component(col) => {
PyComponentColumnDescriptor(col).into_py(py)
}
.map(|col| {
Ok(match col {
ColumnDescriptor::Time(col) => PyIndexColumnDescriptor(col)
.into_pyobject(py)?
.into_any()
.unbind(),
ColumnDescriptor::Component(col) => PyComponentColumnDescriptor(col)
.into_pyobject(py)?
.into_any()
.unbind(),
})
})
.collect::<Vec<PyObject>>()
.collect::<PyResult<Vec<PyObject>>>()?
.into_iter(),
};
Py::new(slf.py(), iter)
Expand Down Expand Up @@ -749,7 +755,7 @@ impl PyRecordingView {
&& all_contents_are_static
&& any_selected_data_is_static
{
py_rerun_warn("RecordingView::select: tried to select static data, but no non-static contents generated an index value on this timeline. No results will be returned. Either include non-static data or consider using `select_static()` instead.")?;
py_rerun_warn(c"RecordingView::select: tried to select static data, but no non-static contents generated an index value on this timeline. No results will be returned. Either include non-static data or consider using `select_static()` instead.")?;
}

let schema = query_handle.schema();
Expand Down
Loading
Loading