diff --git a/Cargo.lock b/Cargo.lock index f4811b345362..571ebe1cfd1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6970,6 +6970,7 @@ dependencies = [ "re_web_viewer_server", "re_ws_comms", "tokio", + "tokio-stream", "tonic", "url", "uuid", diff --git a/examples/python/remote/metadata.py b/examples/python/remote/metadata.py new file mode 100644 index 000000000000..6affe479e413 --- /dev/null +++ b/examples/python/remote/metadata.py @@ -0,0 +1,41 @@ +"""Script to show how to interact with a remote storage node via python APIs.""" + +from __future__ import annotations + +import argparse + +import polars as pl +import pyarrow as pa +import rerun as rr + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + subparsers = parser.add_subparsers(dest="subcommand") + + print_cmd = subparsers.add_parser("print", help="Print everything") + update_cmd = subparsers.add_parser("update", help="Update metadata for a recording") + + update_cmd.add_argument("id", help="ID of the recording to update") + update_cmd.add_argument("key", help="Key of the metadata to update") + update_cmd.add_argument("value", help="Value of the metadata to update") + + args = parser.parse_args() + + # Register the new rrd + conn = rr.remote.connect("http://0.0.0.0:51234") + + catalog = pl.from_arrow(conn.list_recordings()) + + if args.subcommand == "print": + print(catalog) + + if args.subcommand == "update": + id = catalog.filter(catalog["id"].str.starts_with(args.id)).select(pl.first("id")).item() + + if id is None: + print("ID not found") + exit(1) + print(f"Updating metadata for {id}") + + conn.update_metadata(id, {args.key: pa.array([args.value])}) diff --git a/pixi.lock b/pixi.lock index fff4677f244d..c249dc8f45e1 100644 --- a/pixi.lock +++ b/pixi.lock @@ -2455,6 +2455,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/63/b3fc299528d7df1f678b0666002b37affe6b8751225c3d9c12cf530e73ed/pillow-11.0.0-cp311-cp311-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e3/70/67acead083a0aa6d2717fb35637ba7967d37f4a002527f3d29dc96fb445a/polars-1.15.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8d/14/619e24a4c70df2901e1f4dbc50a6291eb63a759172558df326347dce1f0d/protobuf-3.20.3-py2.py3-none-any.whl @@ -2856,6 +2857,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/e4/130c5fab4a54d3991129800dd2801feeb4b118d7630148cd67f0e6269d4c/pillow-11.0.0-cp311-cp311-manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/77/7f/eb9e25dcfdf15b2b1fa18d1d1127268300674d20faabf8a13b3979eb83b0/polars-1.15.0-cp39-abi3-manylinux_2_24_aarch64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e6/23/ed718dc18e6a561445ece1e7a17d2dda0c634ad9cf663102b47f10005d8f/protobuf-5.28.2-cp38-abi3-manylinux2014_aarch64.whl @@ -3231,6 +3233,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/eb/f7e21b113dd48a9c97d364e0915b3988c6a0b6207652f5a92372871b7aa4/pillow-11.0.0-cp311-cp311-macosx_10_10_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/03/6bdf31683d4faddc029e1c4abbffc626c46bb4e87e84a730fecaf7429430/polars-1.15.0-cp39-abi3-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8d/14/619e24a4c70df2901e1f4dbc50a6291eb63a759172558df326347dce1f0d/protobuf-3.20.3-py2.py3-none-any.whl @@ -3611,6 +3614,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/b3/2b54a1d541accebe6bd8b1358b34ceb2c509f51cb7dcda8687362490da5b/pillow-11.0.0-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/05/a3/6d7e178da328a84e610fcb1dfb22555d3d947b39a6029441bebbb58caaad/polars-1.15.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8d/14/619e24a4c70df2901e1f4dbc50a6291eb63a759172558df326347dce1f0d/protobuf-3.20.3-py2.py3-none-any.whl @@ -3989,6 +3993,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e1/2d/c5e34703c118da6dae4de89d5c9b5a2fb9fbc2f7789ac2c8d8836f6367ba/peewee-3.17.7.tar.gz - pypi: https://files.pythonhosted.org/packages/dc/83/1470c220a4ff06cd75fc609068f6605e567ea51df70557555c2ab6516b2c/pillow-11.0.0-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b9/c5/873558da23dafedda97e248238d93efae72041e09dfa134110277de4aa66/polars-1.15.0-cp39-abi3-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8d/14/619e24a4c70df2901e1f4dbc50a6291eb63a759172558df326347dce1f0d/protobuf-3.20.3-py2.py3-none-any.whl @@ -4408,6 +4413,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/39/63/b3fc299528d7df1f678b0666002b37affe6b8751225c3d9c12cf530e73ed/pillow-11.0.0-cp311-cp311-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e3/70/67acead083a0aa6d2717fb35637ba7967d37f4a002527f3d29dc96fb445a/polars-1.15.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8d/14/619e24a4c70df2901e1f4dbc50a6291eb63a759172558df326347dce1f0d/protobuf-3.20.3-py2.py3-none-any.whl @@ -4761,6 +4767,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/fc/e4/130c5fab4a54d3991129800dd2801feeb4b118d7630148cd67f0e6269d4c/pillow-11.0.0-cp311-cp311-manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/77/7f/eb9e25dcfdf15b2b1fa18d1d1127268300674d20faabf8a13b3979eb83b0/polars-1.15.0-cp39-abi3-manylinux_2_24_aarch64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e6/23/ed718dc18e6a561445ece1e7a17d2dda0c634ad9cf663102b47f10005d8f/protobuf-5.28.2-cp38-abi3-manylinux2014_aarch64.whl @@ -5089,6 +5096,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f0/eb/f7e21b113dd48a9c97d364e0915b3988c6a0b6207652f5a92372871b7aa4/pillow-11.0.0-cp311-cp311-macosx_10_10_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/03/6bdf31683d4faddc029e1c4abbffc626c46bb4e87e84a730fecaf7429430/polars-1.15.0-cp39-abi3-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8d/14/619e24a4c70df2901e1f4dbc50a6291eb63a759172558df326347dce1f0d/protobuf-3.20.3-py2.py3-none-any.whl @@ -5422,6 +5430,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/25/b3/2b54a1d541accebe6bd8b1358b34ceb2c509f51cb7dcda8687362490da5b/pillow-11.0.0-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/05/a3/6d7e178da328a84e610fcb1dfb22555d3d947b39a6029441bebbb58caaad/polars-1.15.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8d/14/619e24a4c70df2901e1f4dbc50a6291eb63a759172558df326347dce1f0d/protobuf-3.20.3-py2.py3-none-any.whl @@ -5754,6 +5763,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/e1/2d/c5e34703c118da6dae4de89d5c9b5a2fb9fbc2f7789ac2c8d8836f6367ba/peewee-3.17.7.tar.gz - pypi: https://files.pythonhosted.org/packages/dc/83/1470c220a4ff06cd75fc609068f6605e567ea51df70557555c2ab6516b2c/pillow-11.0.0-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b9/c5/873558da23dafedda97e248238d93efae72041e09dfa134110277de4aa66/polars-1.15.0-cp39-abi3-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/8d/14/619e24a4c70df2901e1f4dbc50a6291eb63a759172558df326347dce1f0d/protobuf-3.20.3-py2.py3-none-any.whl @@ -8977,6 +8987,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/16/89/818fa238e37a47a29bb8495ca2cafdd514599a89f19ada7916348a74b5f9/Pillow-10.0.0-cp311-cp311-manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/e3/70/67acead083a0aa6d2717fb35637ba7967d37f4a002527f3d29dc96fb445a/polars-1.15.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7c/6f/db31f0711c0402aa477257205ce7d29e86a75cb52cd19f7afb585f75cda0/proto_plus-1.24.0-py3-none-any.whl @@ -9472,6 +9483,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/79/53/3a7277ae95bfe86b8b4db0ed1d08c4924aa2dfbfe51b8fe0e310b160a9c6/Pillow-10.0.0-cp311-cp311-manylinux_2_28_aarch64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/77/7f/eb9e25dcfdf15b2b1fa18d1d1127268300674d20faabf8a13b3979eb83b0/polars-1.15.0-cp39-abi3-manylinux_2_24_aarch64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7c/6f/db31f0711c0402aa477257205ce7d29e86a75cb52cd19f7afb585f75cda0/proto_plus-1.24.0-py3-none-any.whl @@ -9941,6 +9953,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7a/54/f6a14d95cba8ff082c550d836c9e5c23f1641d2ac291c23efe0494219b8c/Pillow-10.0.0-cp311-cp311-macosx_10_10_x86_64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/f7/03/6bdf31683d4faddc029e1c4abbffc626c46bb4e87e84a730fecaf7429430/polars-1.15.0-cp39-abi3-macosx_10_12_x86_64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7c/6f/db31f0711c0402aa477257205ce7d29e86a75cb52cd19f7afb585f75cda0/proto_plus-1.24.0-py3-none-any.whl @@ -10414,6 +10427,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/b7/ad/71982d18fd28ed1f93c31b8648f980ebdbdbcf7d8c9c9b4af59290914ce9/Pillow-10.0.0-cp311-cp311-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/05/a3/6d7e178da328a84e610fcb1dfb22555d3d947b39a6029441bebbb58caaad/polars-1.15.0-cp39-abi3-macosx_11_0_arm64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7c/6f/db31f0711c0402aa477257205ce7d29e86a75cb52cd19f7afb585f75cda0/proto_plus-1.24.0-py3-none-any.whl @@ -10863,6 +10877,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/66/d4/054e491f0880bf0119ee79cdc03264e01d5732e06c454da8c69b83a7c8f2/Pillow-10.0.0-cp311-cp311-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/3c/a6/bc1012356d8ece4d66dd75c4b9fc6c1f6650ddd5991e421177d9f8f671be/platformdirs-4.3.6-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/b9/c5/873558da23dafedda97e248238d93efae72041e09dfa134110277de4aa66/polars-1.15.0-cp39-abi3-win_amd64.whl - pypi: https://files.pythonhosted.org/packages/84/2d/46ed6436849c2c88228c3111865f44311cff784b4aabcdef4ea2545dbc3d/prometheus_client-0.21.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/7c/6f/db31f0711c0402aa477257205ce7d29e86a75cb52cd19f7afb585f75cda0/proto_plus-1.24.0-py3-none-any.whl @@ -16913,13 +16928,13 @@ packages: sha256: 8ae055c0b8b0dd7757e4e666f6163172859044d4090830aecbec3460cdb318ee requires_dist: - accelerate - - diffusers==0.27.2 - - numpy - opencv-python - pillow - - rerun-sdk + - diffusers==0.27.2 + - numpy - torch==2.2.2 - transformers + - rerun-sdk requires_python: '>=3.10' editable: true - kind: pypi @@ -17325,7 +17340,7 @@ packages: - numpy - opencv-contrib-python>4.6 - pillow - - requests<3,>=2.31 + - requests>=2.31,<3 - rerun-sdk - timm==0.9.11 - torch==2.2.2 @@ -17340,9 +17355,9 @@ packages: - dicom-numpy==0.6.2 - numpy - pydicom==2.3.0 - - requests<3,>=2.31 + - requests>=2.31,<3 - rerun-sdk - - types-requests<3,>=2.31 + - types-requests>=2.31,<3 editable: true - kind: pypi name: dicom-numpy @@ -19104,7 +19119,7 @@ packages: - mediapipe==0.10.9 ; sys_platform == 'darwin' - numpy - opencv-python>4.9 - - requests<3,>=2.31 + - requests>=2.31,<3 - rerun-sdk - tqdm requires_python: <3.12 @@ -20195,7 +20210,7 @@ packages: - mediapipe==0.10.9 ; sys_platform == 'darwin' - numpy - opencv-python>4.6 - - requests<3,>=2.31 + - requests>=2.31,<3 - rerun-sdk requires_python: <3.12 editable: true @@ -31197,9 +31212,9 @@ packages: path: examples/python/nv12 sha256: c8ca97c5d8c04037cd5eb9a65be7b1e7d667c11d4dba3ee9aad5956ccf926dc4 requires_dist: - - numpy - - opencv-python - rerun-sdk>=0.10 + - opencv-python + - numpy editable: true - kind: pypi name: nvidia-cublas-cu12 @@ -31290,7 +31305,7 @@ packages: - betterproto[compiler] - numpy - opencv-python>4.6 - - requests<3,>=2.31 + - requests>=2.31,<3 - rerun-sdk - scipy editable: true @@ -32961,6 +32976,186 @@ packages: - pkg:pypi/pluggy?source=hash-mapping size: 23815 timestamp: 1713667175451 +- kind: pypi + name: polars + version: 1.15.0 + url: https://files.pythonhosted.org/packages/05/a3/6d7e178da328a84e610fcb1dfb22555d3d947b39a6029441bebbb58caaad/polars-1.15.0-cp39-abi3-macosx_11_0_arm64.whl + sha256: 537df3259543956248c62b636a4e00b11becb30f97d1b4bd80a9e11f46c038bd + requires_dist: + - numpy>=1.16.0 ; extra == 'numpy' + - pandas ; extra == 'pandas' + - polars[pyarrow] ; extra == 'pandas' + - pyarrow>=7.0.0 ; extra == 'pyarrow' + - pydantic ; extra == 'pydantic' + - fastexcel>=0.9 ; extra == 'calamine' + - openpyxl>=3.0.0 ; extra == 'openpyxl' + - xlsx2csv>=0.8.0 ; extra == 'xlsx2csv' + - xlsxwriter ; extra == 'xlsxwriter' + - polars[calamine,openpyxl,xlsx2csv,xlsxwriter] ; extra == 'excel' + - adbc-driver-manager[dbapi] ; extra == 'adbc' + - adbc-driver-sqlite[dbapi] ; extra == 'adbc' + - connectorx>=0.3.2 ; extra == 'connectorx' + - sqlalchemy ; extra == 'sqlalchemy' + - polars[pandas] ; extra == 'sqlalchemy' + - polars[adbc,connectorx,sqlalchemy] ; extra == 'database' + - nest-asyncio ; extra == 'database' + - fsspec ; extra == 'fsspec' + - deltalake>=0.15.0 ; extra == 'deltalake' + - pyiceberg>=0.5.0 ; extra == 'iceberg' + - gevent ; extra == 'async' + - cloudpickle ; extra == 'cloudpickle' + - matplotlib ; extra == 'graph' + - altair>=5.4.0 ; extra == 'plot' + - great-tables>=0.8.0 ; extra == 'style' + - backports-zoneinfo ; python_full_version < '3.9' and extra == 'timezone' + - tzdata ; platform_system == 'Windows' and extra == 'timezone' + - cudf-polars-cu12 ; extra == 'gpu' + - polars[async,cloudpickle,database,deltalake,excel,fsspec,graph,iceberg,numpy,pandas,plot,pyarrow,pydantic,style,timezone] ; extra == 'all' + requires_python: '>=3.9' +- kind: pypi + name: polars + version: 1.15.0 + url: https://files.pythonhosted.org/packages/77/7f/eb9e25dcfdf15b2b1fa18d1d1127268300674d20faabf8a13b3979eb83b0/polars-1.15.0-cp39-abi3-manylinux_2_24_aarch64.whl + sha256: 1d15b86b73653befd834e48418d759954daeb831a08b32cd21034c88378c4037 + requires_dist: + - numpy>=1.16.0 ; extra == 'numpy' + - pandas ; extra == 'pandas' + - polars[pyarrow] ; extra == 'pandas' + - pyarrow>=7.0.0 ; extra == 'pyarrow' + - pydantic ; extra == 'pydantic' + - fastexcel>=0.9 ; extra == 'calamine' + - openpyxl>=3.0.0 ; extra == 'openpyxl' + - xlsx2csv>=0.8.0 ; extra == 'xlsx2csv' + - xlsxwriter ; extra == 'xlsxwriter' + - polars[calamine,openpyxl,xlsx2csv,xlsxwriter] ; extra == 'excel' + - adbc-driver-manager[dbapi] ; extra == 'adbc' + - adbc-driver-sqlite[dbapi] ; extra == 'adbc' + - connectorx>=0.3.2 ; extra == 'connectorx' + - sqlalchemy ; extra == 'sqlalchemy' + - polars[pandas] ; extra == 'sqlalchemy' + - polars[adbc,connectorx,sqlalchemy] ; extra == 'database' + - nest-asyncio ; extra == 'database' + - fsspec ; extra == 'fsspec' + - deltalake>=0.15.0 ; extra == 'deltalake' + - pyiceberg>=0.5.0 ; extra == 'iceberg' + - gevent ; extra == 'async' + - cloudpickle ; extra == 'cloudpickle' + - matplotlib ; extra == 'graph' + - altair>=5.4.0 ; extra == 'plot' + - great-tables>=0.8.0 ; extra == 'style' + - backports-zoneinfo ; python_full_version < '3.9' and extra == 'timezone' + - tzdata ; platform_system == 'Windows' and extra == 'timezone' + - cudf-polars-cu12 ; extra == 'gpu' + - polars[async,cloudpickle,database,deltalake,excel,fsspec,graph,iceberg,numpy,pandas,plot,pyarrow,pydantic,style,timezone] ; extra == 'all' + requires_python: '>=3.9' +- kind: pypi + name: polars + version: 1.15.0 + url: https://files.pythonhosted.org/packages/b9/c5/873558da23dafedda97e248238d93efae72041e09dfa134110277de4aa66/polars-1.15.0-cp39-abi3-win_amd64.whl + sha256: e4602b715dc58b84e38102c9ffcc7738f078373415f55acf0760204443d62371 + requires_dist: + - numpy>=1.16.0 ; extra == 'numpy' + - pandas ; extra == 'pandas' + - polars[pyarrow] ; extra == 'pandas' + - pyarrow>=7.0.0 ; extra == 'pyarrow' + - pydantic ; extra == 'pydantic' + - fastexcel>=0.9 ; extra == 'calamine' + - openpyxl>=3.0.0 ; extra == 'openpyxl' + - xlsx2csv>=0.8.0 ; extra == 'xlsx2csv' + - xlsxwriter ; extra == 'xlsxwriter' + - polars[calamine,openpyxl,xlsx2csv,xlsxwriter] ; extra == 'excel' + - adbc-driver-manager[dbapi] ; extra == 'adbc' + - adbc-driver-sqlite[dbapi] ; extra == 'adbc' + - connectorx>=0.3.2 ; extra == 'connectorx' + - sqlalchemy ; extra == 'sqlalchemy' + - polars[pandas] ; extra == 'sqlalchemy' + - polars[adbc,connectorx,sqlalchemy] ; extra == 'database' + - nest-asyncio ; extra == 'database' + - fsspec ; extra == 'fsspec' + - deltalake>=0.15.0 ; extra == 'deltalake' + - pyiceberg>=0.5.0 ; extra == 'iceberg' + - gevent ; extra == 'async' + - cloudpickle ; extra == 'cloudpickle' + - matplotlib ; extra == 'graph' + - altair>=5.4.0 ; extra == 'plot' + - great-tables>=0.8.0 ; extra == 'style' + - backports-zoneinfo ; python_full_version < '3.9' and extra == 'timezone' + - tzdata ; platform_system == 'Windows' and extra == 'timezone' + - cudf-polars-cu12 ; extra == 'gpu' + - polars[async,cloudpickle,database,deltalake,excel,fsspec,graph,iceberg,numpy,pandas,plot,pyarrow,pydantic,style,timezone] ; extra == 'all' + requires_python: '>=3.9' +- kind: pypi + name: polars + version: 1.15.0 + url: https://files.pythonhosted.org/packages/e3/70/67acead083a0aa6d2717fb35637ba7967d37f4a002527f3d29dc96fb445a/polars-1.15.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + sha256: c6686bc8acaaed24f337ae1623536210370ab4d3099c1dbb79ccfc9ec083914a + requires_dist: + - numpy>=1.16.0 ; extra == 'numpy' + - pandas ; extra == 'pandas' + - polars[pyarrow] ; extra == 'pandas' + - pyarrow>=7.0.0 ; extra == 'pyarrow' + - pydantic ; extra == 'pydantic' + - fastexcel>=0.9 ; extra == 'calamine' + - openpyxl>=3.0.0 ; extra == 'openpyxl' + - xlsx2csv>=0.8.0 ; extra == 'xlsx2csv' + - xlsxwriter ; extra == 'xlsxwriter' + - polars[calamine,openpyxl,xlsx2csv,xlsxwriter] ; extra == 'excel' + - adbc-driver-manager[dbapi] ; extra == 'adbc' + - adbc-driver-sqlite[dbapi] ; extra == 'adbc' + - connectorx>=0.3.2 ; extra == 'connectorx' + - sqlalchemy ; extra == 'sqlalchemy' + - polars[pandas] ; extra == 'sqlalchemy' + - polars[adbc,connectorx,sqlalchemy] ; extra == 'database' + - nest-asyncio ; extra == 'database' + - fsspec ; extra == 'fsspec' + - deltalake>=0.15.0 ; extra == 'deltalake' + - pyiceberg>=0.5.0 ; extra == 'iceberg' + - gevent ; extra == 'async' + - cloudpickle ; extra == 'cloudpickle' + - matplotlib ; extra == 'graph' + - altair>=5.4.0 ; extra == 'plot' + - great-tables>=0.8.0 ; extra == 'style' + - backports-zoneinfo ; python_full_version < '3.9' and extra == 'timezone' + - tzdata ; platform_system == 'Windows' and extra == 'timezone' + - cudf-polars-cu12 ; extra == 'gpu' + - polars[async,cloudpickle,database,deltalake,excel,fsspec,graph,iceberg,numpy,pandas,plot,pyarrow,pydantic,style,timezone] ; extra == 'all' + requires_python: '>=3.9' +- kind: pypi + name: polars + version: 1.15.0 + url: https://files.pythonhosted.org/packages/f7/03/6bdf31683d4faddc029e1c4abbffc626c46bb4e87e84a730fecaf7429430/polars-1.15.0-cp39-abi3-macosx_10_12_x86_64.whl + sha256: d66ae8aae1a5b1273f7235173ad8022e479378eb36ff7ea6b1dd8224430d18bd + requires_dist: + - numpy>=1.16.0 ; extra == 'numpy' + - pandas ; extra == 'pandas' + - polars[pyarrow] ; extra == 'pandas' + - pyarrow>=7.0.0 ; extra == 'pyarrow' + - pydantic ; extra == 'pydantic' + - fastexcel>=0.9 ; extra == 'calamine' + - openpyxl>=3.0.0 ; extra == 'openpyxl' + - xlsx2csv>=0.8.0 ; extra == 'xlsx2csv' + - xlsxwriter ; extra == 'xlsxwriter' + - polars[calamine,openpyxl,xlsx2csv,xlsxwriter] ; extra == 'excel' + - adbc-driver-manager[dbapi] ; extra == 'adbc' + - adbc-driver-sqlite[dbapi] ; extra == 'adbc' + - connectorx>=0.3.2 ; extra == 'connectorx' + - sqlalchemy ; extra == 'sqlalchemy' + - polars[pandas] ; extra == 'sqlalchemy' + - polars[adbc,connectorx,sqlalchemy] ; extra == 'database' + - nest-asyncio ; extra == 'database' + - fsspec ; extra == 'fsspec' + - deltalake>=0.15.0 ; extra == 'deltalake' + - pyiceberg>=0.5.0 ; extra == 'iceberg' + - gevent ; extra == 'async' + - cloudpickle ; extra == 'cloudpickle' + - matplotlib ; extra == 'graph' + - altair>=5.4.0 ; extra == 'plot' + - great-tables>=0.8.0 ; extra == 'style' + - backports-zoneinfo ; python_full_version < '3.9' and extra == 'timezone' + - tzdata ; platform_system == 'Windows' and extra == 'timezone' + - cudf-polars-cu12 ; extra == 'gpu' + - polars[async,cloudpickle,database,deltalake,excel,fsspec,graph,iceberg,numpy,pandas,plot,pyarrow,pydantic,style,timezone] ; extra == 'all' + requires_python: '>=3.9' - kind: conda name: prettier version: 3.2.5 @@ -34770,7 +34965,7 @@ packages: sha256: 9006b1b7ca8bd9c90ba0bf0d7a00641b7dd13a6de76a2828f79ec5b853a4ef98 requires_dist: - numpy - - requests<3,>=2.31 + - requests>=2.31,<3 - rerun-sdk - trimesh==3.15.2 editable: true @@ -35114,7 +35309,7 @@ packages: requires_dist: - numpy - opencv-python>4.6 - - requests<3,>=2.31 + - requests>=2.31,<3 - rerun-sdk - tqdm editable: true @@ -36454,11 +36649,11 @@ packages: path: examples/python/segment_anything_model sha256: 85bc241bedf212c63a39d0251a9dcc0fb3a435087a024d4eafd7f49342a75926 requires_dist: + - segment-anything @ git+https://github.com/facebookresearch/segment-anything.git - numpy - opencv-python - - requests<3,>=2.31 + - requests>=2.31,<3 - rerun-sdk - - segment-anything @ git+https://github.com/facebookresearch/segment-anything.git - torch==2.2.2 - torchvision - tqdm @@ -36666,7 +36861,7 @@ packages: requires_dist: - mesh-to-sdf @ git+https://github.com/marian42/mesh_to_sdf.git - numpy - - requests<3,>=2.31 + - requests>=2.31,<3 - rerun-sdk - scikit-learn>=1.1.3 - trimesh==3.15.2 @@ -36917,9 +37112,9 @@ packages: path: examples/python/structure_from_motion sha256: b20b79aa7bb2b4225b37d3cb28872a70dc7e9ab2ca9ab138b90d60fc8d7b4c15 requires_dist: - - numpy - opencv-python>4.6 - - requests<3,>=2.31 + - numpy + - requests>=2.31,<3 - rerun-sdk - tqdm editable: true diff --git a/pixi.toml b/pixi.toml index 97766b3be9bb..510a482e9c10 100644 --- a/pixi.toml +++ b/pixi.toml @@ -605,6 +605,7 @@ python = "=3.11" [feature.examples-common.pypi-dependencies] # External deps jupyter = ">=1.0" +polars = ">=0.12.0" segment-anything = { git = "https://github.com/facebookresearch/segment-anything.git" } mesh-to-sdf = { git = "https://github.com/marian42/mesh_to_sdf.git" } diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index fd4833e1e10f..397434ddc986 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -40,6 +40,7 @@ remote = [ "dep:re_protos", "dep:re_ws_comms", "dep:tokio", + "dep:tokio-stream", "dep:tonic", "dep:url", ] @@ -87,6 +88,7 @@ uuid.workspace = true object_store = { workspace = true, optional = true, features = ["aws"] } re_protos = { workspace = true, optional = true } tokio = { workspace = true, optional = true } +tokio-stream = { workspace = true, optional = true } # Not used yet, but we will need it when we start streaming data #tokio-stream = { workspace = true, optional = true } tonic = { workspace = true, default-features = false, features = [ diff --git a/rerun_py/rerun_bindings/rerun_bindings.pyi b/rerun_py/rerun_bindings/rerun_bindings.pyi index e3f863a7d5d1..dbe45bb9e285 100644 --- a/rerun_py/rerun_bindings/rerun_bindings.pyi +++ b/rerun_py/rerun_bindings/rerun_bindings.pyi @@ -3,7 +3,7 @@ from typing import Iterator, Optional, Sequence, Union import pyarrow as pa -from .types import AnyColumn, AnyComponentColumn, ComponentLike, IndexValuesLike, ViewContentsLike +from .types import AnyColumn, AnyComponentColumn, ComponentLike, IndexValuesLike, MetadataLike, ViewContentsLike class IndexColumnDescriptor: """ @@ -567,3 +567,80 @@ def load_archive(path_to_rrd: str | os.PathLike) -> RRDArchive: """ ... + +class StorageNodeClient: + """ + A client for interfacing with a Rerun storage node. + + Required-feature: `remote` + """ + + def list_recordings(self) -> pa.RecordBatchReader: + """Get the metadata for all recordings in the storage node.""" + ... + + def register(self, storage_url: str, metadata: Optional[dict[str, MetadataLike]] = None) -> str: + """ + Register a recording along with some metadata. + + Parameters + ---------- + storage_url : str + The URL to the storage location. + metadata : dict[str, MetadataLike] + A dictionary where the keys are the metadata columns and the values are pyarrow arrays. + + """ + ... + + def update_metadata(self, id: str, metadata: dict[str, MetadataLike]) -> None: + """ + Update the metadata for the recording with the given id. + + Parameters + ---------- + id : str + The id of the recording to update. + metadata : dict[str, MetadataLike] + A dictionary where the keys are the metadata columns and the values are pyarrow arrays. + + """ + ... + + def open_recording(self, id: str) -> Recording: + """ + Open a [`Recording`][rerun.dataframe.Recording] by id to use with the dataframe APIs. + + This currently downloads the full recording to the local machine. + + Parameters + ---------- + id : str + The id of the recording to open. + + Returns + ------- + Recording + The opened recording. + + """ + ... + +def connect(addr: str) -> StorageNodeClient: + """ + Load a rerun archive from an RRD file. + + Required-feature: `remote` + + Parameters + ---------- + addr : str + The address of the storage node to connect to. + + Returns + ------- + StorageNodeClient + The connected client. + + """ + ... diff --git a/rerun_py/rerun_bindings/types.py b/rerun_py/rerun_bindings/types.py index 0477ab660527..c5ddf94e7477 100644 --- a/rerun_py/rerun_bindings/types.py +++ b/rerun_py/rerun_bindings/types.py @@ -67,3 +67,5 @@ This can be any numpy-compatible array of integers, or a [`pa.Int64Array`][] """ + +MetadataLike: TypeAlias = pa.Array diff --git a/rerun_py/rerun_sdk/rerun/remote.py b/rerun_py/rerun_sdk/rerun/remote.py index b48fba9a0516..5fad8347a3e0 100644 --- a/rerun_py/rerun_sdk/rerun/remote.py +++ b/rerun_py/rerun_sdk/rerun/remote.py @@ -2,9 +2,10 @@ try: from rerun_bindings import ( + StorageNodeClient as StorageNodeClient, connect as connect, ) except ImportError: - def connect(url: str) -> None: + def connect(addr: str) -> StorageNodeClient: raise NotImplementedError("Rerun SDK was built without the `remote` feature enabled.") diff --git a/rerun_py/src/dataframe.rs b/rerun_py/src/dataframe.rs index 31f4335326b0..57c10bf0a039 100644 --- a/rerun_py/src/dataframe.rs +++ b/rerun_py/src/dataframe.rs @@ -565,8 +565,8 @@ impl PySchema { /// to retrieve the data. #[pyclass(name = "Recording")] pub struct PyRecording { - store: ChunkStoreHandle, - cache: re_dataframe::QueryCacheHandle, + pub(crate) store: ChunkStoreHandle, + pub(crate) cache: re_dataframe::QueryCacheHandle, } /// A view of a recording restricted to a given index, containing a specific set of entities and components. diff --git a/rerun_py/src/remote.rs b/rerun_py/src/remote.rs index 87db9ca7bf59..eeaf24c652a7 100644 --- a/rerun_py/src/remote.rs +++ b/rerun_py/src/remote.rs @@ -1,15 +1,31 @@ #![allow(unsafe_op_in_unsafe_fn)] -use arrow::{array::ArrayData, pyarrow::PyArrowType}; +use arrow::{ + array::{ArrayData, RecordBatch, RecordBatchIterator, RecordBatchReader}, + datatypes::Schema, + pyarrow::PyArrowType, +}; // False positive due to #[pyfunction] macro use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyDict, Bound, PyResult}; -use re_chunk::TransportChunk; -use re_protos::v0::{ - storage_node_client::StorageNodeClient, EncoderVersion, ListRecordingsRequest, - RecordingMetadata, RecordingType, RegisterRecordingRequest, +use re_chunk::{Chunk, TransportChunk}; +use re_chunk_store::ChunkStore; +use re_dataframe::ChunkStoreHandle; +use re_log_types::{StoreInfo, StoreSource}; +use re_protos::{ + codec::decode, + v0::{ + storage_node_client::StorageNodeClient, EncoderVersion, FetchRecordingRequest, + ListRecordingsRequest, RecordingId, RecordingMetadata, RecordingType, + RegisterRecordingRequest, UpdateRecordingMetadataRequest, + }, }; +use re_sdk::{ApplicationId, StoreId, StoreKind, Time}; + +use crate::dataframe::PyRecording; /// Register the `rerun.remote` module. pub(crate) fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_function(wrap_pyfunction!(connect, m)?)?; Ok(()) @@ -26,20 +42,33 @@ async fn connect_async(addr: String) -> PyResult PyResult { +pub fn connect(addr: String) -> PyResult { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; let client = runtime.block_on(connect_async(addr))?; - Ok(PyConnection { runtime, client }) + Ok(PyStorageNodeClient { runtime, client }) } /// A connection to a remote storage node. -#[pyclass(name = "Connection")] -pub struct PyConnection { +#[pyclass(name = "StorageNodeClient")] +pub struct PyStorageNodeClient { /// A tokio runtime for async operations. This connection will currently /// block the Python interpreter while waiting for responses. /// This runtime must be persisted for the lifetime of the connection. @@ -50,10 +79,11 @@ pub struct PyConnection { } #[pymethods] -impl PyConnection { - /// List all recordings registered with the node. - fn list_recordings(&mut self) -> PyResult> { - self.runtime.block_on(async { +impl PyStorageNodeClient { + /// Get the metadata for all recordings in the storage node. + fn list_recordings(&mut self) -> PyResult>> { + let reader = self.runtime.block_on(async { + // TODO(jleibs): Support column projection let request = ListRecordingsRequest { column_projection: None, }; @@ -64,16 +94,43 @@ impl PyConnection { .await .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; - Ok(resp + let transport_chunks = resp .into_inner() .recordings .into_iter() - .map(|recording| PyRecordingMetadata { info: recording }) - .collect()) - }) + .map(|recording| recording.data()) + .collect::, _>>() + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + + let record_batches: Vec> = + transport_chunks + .into_iter() + .map(|tc| tc.try_to_arrow_record_batch()) + .collect(); + + // TODO(jleibs): surfacing this schema is awkward. This should be more explicit in + // the gRPC APIs somehow. + let schema = record_batches + .first() + .and_then(|batch| batch.as_ref().ok().map(|batch| batch.schema())) + .unwrap_or(std::sync::Arc::new(Schema::empty())); + + let reader = RecordBatchIterator::new(record_batches, schema); + + Ok::<_, PyErr>(reader) + })?; + + Ok(PyArrowType(Box::new(reader))) } - /// Register a recording along with some metadata + /// Register a recording along with some metadata. + /// + /// Parameters + /// ---------- + /// storage_url : str + /// The URL to the storage location. + /// metadata : dict[str, MetadataLike] + /// A dictionary where the keys are the metadata columns and the values are pyarrow arrays. #[pyo3(signature = ( storage_url, metadata = None @@ -146,6 +203,138 @@ impl PyConnection { Ok(recording_id) }) } + + /// Update the metadata for the recording with the given id. + /// + /// Parameters + /// ---------- + /// id : str + /// The id of the recording to update. + /// metadata : dict[str, MetadataLike] + /// A dictionary where the keys are the metadata columns and the values are pyarrow arrays. + #[pyo3(signature = ( + id, + metadata + ))] + fn update_metadata(&mut self, id: &str, metadata: &Bound<'_, PyDict>) -> PyResult<()> { + self.runtime.block_on(async { + let (schema, data): ( + Vec, + Vec>, + ) = metadata + .iter() + .map(|(key, value)| { + let key = key.to_string(); + let value = value.extract::()?; + let value_array = value.to_arrow2()?; + let field = + arrow2::datatypes::Field::new(key, value_array.data_type().clone(), true); + Ok((field, value_array)) + }) + .collect::>>()? + .into_iter() + .unzip(); + + let schema = arrow2::datatypes::Schema::from(schema); + + let data = arrow2::chunk::Chunk::new(data); + + let metadata_tc = TransportChunk { + schema: schema.clone(), + data, + }; + + let metadata = RecordingMetadata::try_from(EncoderVersion::V0, &metadata_tc) + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + + let request = UpdateRecordingMetadataRequest { + recording_id: Some(RecordingId { id: id.to_owned() }), + metadata: Some(metadata), + }; + + self.client + .update_recording_metadata(request) + .await + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + + Ok(()) + }) + } + + /// Open a [`Recording`][rerun.dataframe.Recording] by id to use with the dataframe APIs. + /// + /// This currently downloads the full recording to the local machine. + /// + /// Parameters + /// ---------- + /// id : str + /// The id of the recording to open. + /// + /// Returns + /// ------- + /// Recording + /// The opened recording. + #[pyo3(signature = ( + id, + ))] + fn open_recording(&mut self, id: &str) -> PyResult { + use tokio_stream::StreamExt as _; + let store = self.runtime.block_on(async { + let mut resp = self + .client + .fetch_recording(FetchRecordingRequest { + recording_id: Some(RecordingId { id: id.to_owned() }), + }) + .await + .map_err(|err| PyRuntimeError::new_err(err.to_string()))? + .into_inner(); + + // TODO(jleibs): Does this come from RDP? + let store_id = StoreId::from_string(StoreKind::Recording, id.to_owned()); + + let store_info = StoreInfo { + application_id: ApplicationId::from("rerun_data_platform"), + store_id: store_id.clone(), + cloned_from: None, + is_official_example: false, + started: Time::now(), + store_source: StoreSource::Unknown, + store_version: None, + }; + + let mut store = ChunkStore::new(store_id, Default::default()); + store.set_info(store_info); + + while let Some(result) = resp.next().await { + let response = result.map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + let tc = decode(EncoderVersion::V0, &response.payload) + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + + let Some(tc) = tc else { + return Err(PyRuntimeError::new_err("Stream error")); + }; + + let chunk = Chunk::from_transport(&tc) + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + + store + .insert_chunk(&std::sync::Arc::new(chunk)) + .map_err(|err| PyRuntimeError::new_err(err.to_string()))?; + } + + Ok(store) + })?; + + let handle = ChunkStoreHandle::new(store); + + let cache = + re_dataframe::QueryCacheHandle::new(re_dataframe::QueryCache::new(handle.clone())); + + Ok(PyRecording { + store: handle, + cache, + }) + } } /// A type alias for metadata. @@ -187,22 +376,3 @@ impl MetadataLike { } } } - -/// The info for a recording stored in the archive. -#[pyclass(name = "RecordingMetadata")] -pub struct PyRecordingMetadata { - info: re_protos::v0::RecordingMetadata, -} - -#[pymethods] -impl PyRecordingMetadata { - fn __repr__(&self) -> String { - format!( - "Recording(id={})", - self.info - .id() - .map(|id| id.to_string()) - .unwrap_or("Unknown".to_owned()) - ) - } -} diff --git a/scripts/ci/python_check_signatures.py b/scripts/ci/python_check_signatures.py index d6110caa196e..27746d5dde16 100644 --- a/scripts/ci/python_check_signatures.py +++ b/scripts/ci/python_check_signatures.py @@ -67,7 +67,7 @@ def __eq__(self, other): return self.name == other.name and self.signature == other.signature and self.doc == other.doc -TotalSignature = dict[str, APIDef | dict[str, APIDef]] +TotalSignature = dict[str, APIDef | dict[str, APIDef | str]] def parse_function_signature(node: Any) -> APIDef: @@ -129,8 +129,19 @@ def load_stub_signatures(pyi_file: Path) -> TotalSignature: elif node.type == "classdef": class_name = node.name.value - # Extract methods within the class + class_def = {} + + doc = None + for child in node.children: + if child.type == "suite": + first_child = child.children[1] + if first_child.type == "simple_stmt" and first_child.children[0].type == "string": + doc = first_child.children[0].value.strip('"""') + if doc is not None: + class_def["__doc__"] = doc + + # Extract methods within the class for class_node in node.iter_funcdefs(): method_name = class_node.name.value @@ -203,6 +214,8 @@ def compare_signatures(stub_signatures: TotalSignature, runtime_signatures: Tota print("Stub expected class, but runtime provided function.") continue for method_name, stub_method_signature in stub_signature.items(): + if isinstance(stub_method_signature, str): + continue if stub_method_signature.doc is None: print() print(f"{name}.{method_name} missing docstring") @@ -215,8 +228,12 @@ def compare_signatures(stub_signatures: TotalSignature, runtime_signatures: Tota result += 1 else: - print(f"Class {name} not found in runtime") - result += 1 + docstr = stub_signature.get("__doc__", "") + if isinstance(docstr, str) and "Required-feature:" in docstr: + print(f"Skipping class {name} since it requires features not enabled in default runtime") + else: + print(f"Class {name} not found in runtime") + result += 1 else: if stub_signature.doc is None: print() @@ -232,8 +249,11 @@ def compare_signatures(stub_signatures: TotalSignature, runtime_signatures: Tota result += 1 else: print() - print(f"Function {name} not found in runtime") - result += 1 + if stub_signature.doc is not None and "Required-feature:" in stub_signature.doc: + print(f"Skipping Function {name} since it requires features not enabled in default runtime") + else: + print(f"Function {name} not found in runtime") + result += 1 if result == 0: print("All stub signatures match!")