From 6ccef00deb3ea8ab4ce12d840409c1af898ae281 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 14:00:46 +0800 Subject: [PATCH] feat(bindings/python): generate python operator constructor types (#5457) --- bindings/python/Cargo.toml | 7 +- bindings/python/python/opendal/__base.pyi | 781 ++++++++++++++++++++ bindings/python/python/opendal/__init__.pyi | 9 +- dev/Cargo.lock | 142 ++++ dev/Cargo.toml | 3 +- dev/src/generate/binding_python.rs | 67 +- dev/templates/python.py.jinja2 | 62 ++ 7 files changed, 1060 insertions(+), 11 deletions(-) create mode 100644 bindings/python/python/opendal/__base.pyi create mode 100644 dev/templates/python.py.jinja2 diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 6cda36ddfb29..bf57397945e1 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -45,6 +45,9 @@ default = [ "services-webhdfs", ] +# NOTE: this is the feature we used to build pypi wheels. +# When enable or disable some features, +# also need to update dev/src/generate/binding_python.rs `enabled_service` to match it. services-all = [ "default", "services-aliyun-drive", @@ -161,9 +164,7 @@ bytes = "1.5.0" dict_derive = "0.6.0" futures = "0.3.28" # this crate won't be published, we always use the local version -opendal = { version = ">=0", path = "../../core", features = [ - "layers-blocking", -] } +opendal = { version = ">=0", path = "../../core", features = ["layers-blocking"] } pyo3 = { version = "0.23.3", features = ["generate-import-lib"] } pyo3-async-runtimes = { version = "0.23.0", features = ["tokio-runtime"] } tokio = "1" diff --git a/bindings/python/python/opendal/__base.pyi b/bindings/python/python/opendal/__base.pyi new file mode 100644 index 000000000000..c9e794b81985 --- /dev/null +++ b/bindings/python/python/opendal/__base.pyi @@ -0,0 +1,781 @@ +""" +this file is generated by opendal/dev/generate/binding_python.rs, and opendal.__base doesn't exists. + +DO NOT EDIT IT Manually +""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import overload, Literal, TypeAlias + +# `true`/`false`` in any case, for example, `true`/`True`/`TRUE` `false`/`False`/`FALSE` +_bool: TypeAlias = str +# a str represent a int, for example, `"10"`/`"0"` +_int: TypeAlias = str + +# a human readable duration string +# see https://docs.rs/humantime/latest/humantime/fn.parse_duration.html +# for more details +_duration: TypeAlias = str + +# A "," separated string, for example `"127.0.0.1:1,127.0.0.1:2"` +_strings: TypeAlias = str + +class _Base: + """this is not a real base class but typing mixin, + + The services list here is support by opendal pypi wheel. + """ + + @overload + def __init__( + self, + scheme: Literal["aliyun_drive"], + /, + *, + drive_type: str, + root: str = ..., + access_token: str = ..., + client_id: str = ..., + client_secret: str = ..., + refresh_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["alluxio"], + /, + *, + root: str = ..., + endpoint: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["atomicserver"], + /, + *, + root: str = ..., + endpoint: str = ..., + private_key: str = ..., + public_key: str = ..., + parent_resource_id: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["azblob"], + /, + *, + container: str, + root: str = ..., + endpoint: str = ..., + account_name: str = ..., + account_key: str = ..., + encryption_key: str = ..., + encryption_key_sha256: str = ..., + encryption_algorithm: str = ..., + sas_token: str = ..., + batch_max_operations: _int = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["azdls"], + /, + *, + filesystem: str, + root: str = ..., + endpoint: str = ..., + account_name: str = ..., + account_key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["azfile"], + /, + *, + share_name: str, + root: str = ..., + endpoint: str = ..., + account_name: str = ..., + account_key: str = ..., + sas_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["b2"], + /, + *, + bucket: str, + bucket_id: str, + root: str = ..., + application_key_id: str = ..., + application_key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["cacache"], + /, + *, + datadir: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["chainsafe"], + /, + *, + bucket_id: str, + root: str = ..., + api_key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["cloudflare_kv"], + /, + *, + token: str = ..., + account_id: str = ..., + namespace_id: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["compfs"], + /, + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["cos"], + /, + *, + root: str = ..., + endpoint: str = ..., + secret_id: str = ..., + secret_key: str = ..., + bucket: str = ..., + disable_config_load: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["d1"], + /, + *, + token: str = ..., + account_id: str = ..., + database_id: str = ..., + root: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["dashmap"], + /, + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["dbfs"], + /, + *, + root: str = ..., + endpoint: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["dropbox"], + /, + *, + root: str = ..., + access_token: str = ..., + refresh_token: str = ..., + client_id: str = ..., + client_secret: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["fs"], + /, + *, + root: str = ..., + atomic_write_dir: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["gcs"], + /, + *, + bucket: str, + root: str = ..., + endpoint: str = ..., + scope: str = ..., + service_account: str = ..., + credential: str = ..., + credential_path: str = ..., + predefined_acl: str = ..., + default_storage_class: str = ..., + allow_anonymous: _bool = ..., + disable_vm_metadata: _bool = ..., + disable_config_load: _bool = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["gdrive"], + /, + *, + root: str = ..., + access_token: str = ..., + refresh_token: str = ..., + client_id: str = ..., + client_secret: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["ghac"], + /, + *, + root: str = ..., + version: str = ..., + endpoint: str = ..., + runtime_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["github"], + /, + *, + owner: str, + repo: str, + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["gridfs"], + /, + *, + connection_string: str = ..., + database: str = ..., + bucket: str = ..., + chunk_size: _int = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["hdfs_native"], + /, + *, + root: str = ..., + url: str = ..., + enable_append: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["http"], + /, + *, + endpoint: str = ..., + username: str = ..., + password: str = ..., + token: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["huggingface"], + /, + *, + repo_type: str = ..., + repo_id: str = ..., + revision: str = ..., + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["icloud"], + /, + *, + root: str = ..., + apple_id: str = ..., + password: str = ..., + trust_token: str = ..., + ds_web_auth_token: str = ..., + is_china_mainland: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["ipfs"], + /, + *, + endpoint: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["ipmfs"], + /, + *, + root: str = ..., + endpoint: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["koofr"], + /, + *, + endpoint: str, + email: str, + root: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["lakefs"], + /, + *, + endpoint: str = ..., + username: str = ..., + password: str = ..., + root: str = ..., + repository: str = ..., + branch: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["libsql"], + /, + *, + connection_string: str = ..., + auth_token: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["memcached"], + /, + *, + endpoint: str = ..., + root: str = ..., + username: str = ..., + password: str = ..., + default_ttl: _duration = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["memory"], + /, + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["mini_moka"], + /, + *, + max_capacity: _int = ..., + time_to_live: _duration = ..., + time_to_idle: _duration = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["moka"], + /, + *, + name: str = ..., + max_capacity: _int = ..., + time_to_live: _duration = ..., + time_to_idle: _duration = ..., + num_segments: _int = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["mongodb"], + /, + *, + connection_string: str = ..., + database: str = ..., + collection: str = ..., + root: str = ..., + key_field: str = ..., + value_field: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["monoiofs"], + /, + *, + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["mysql"], + /, + *, + connection_string: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["nebula_graph"], + /, + *, + host: str = ..., + port: _int = ..., + username: str = ..., + password: str = ..., + space: str = ..., + tag: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["obs"], + /, + *, + root: str = ..., + endpoint: str = ..., + access_key_id: str = ..., + secret_access_key: str = ..., + bucket: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["onedrive"], + /, + *, + access_token: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["oss"], + /, + *, + bucket: str, + root: str = ..., + endpoint: str = ..., + presign_endpoint: str = ..., + server_side_encryption: str = ..., + server_side_encryption_key_id: str = ..., + allow_anonymous: _bool = ..., + access_key_id: str = ..., + access_key_secret: str = ..., + # deprecated: Please use `delete_max_size` instead of `batch_max_operations` + batch_max_operations: _int = ..., + delete_max_size: _int = ..., + role_arn: str = ..., + role_session_name: str = ..., + oidc_provider_arn: str = ..., + oidc_token_file: str = ..., + sts_endpoint: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["pcloud"], + /, + *, + endpoint: str, + root: str = ..., + username: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["persy"], + /, + *, + datafile: str = ..., + segment: str = ..., + index: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["postgresql"], + /, + *, + root: str = ..., + connection_string: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["redb"], + /, + *, + datadir: str = ..., + root: str = ..., + table: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["redis"], + /, + *, + db: _int, + endpoint: str = ..., + cluster_endpoints: str = ..., + username: str = ..., + password: str = ..., + root: str = ..., + default_ttl: _duration = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["s3"], + /, + *, + bucket: str, + root: str = ..., + enable_versioning: _bool = ..., + endpoint: str = ..., + region: str = ..., + access_key_id: str = ..., + secret_access_key: str = ..., + session_token: str = ..., + role_arn: str = ..., + external_id: str = ..., + role_session_name: str = ..., + disable_config_load: _bool = ..., + disable_ec2_metadata: _bool = ..., + allow_anonymous: _bool = ..., + server_side_encryption: str = ..., + server_side_encryption_aws_kms_key_id: str = ..., + server_side_encryption_customer_algorithm: str = ..., + server_side_encryption_customer_key: str = ..., + server_side_encryption_customer_key_md5: str = ..., + default_storage_class: str = ..., + enable_virtual_host_style: _bool = ..., + # deprecated: Please use `delete_max_size` instead of `batch_max_operations` + batch_max_operations: _int = ..., + delete_max_size: _int = ..., + disable_stat_with_override: _bool = ..., + checksum_algorithm: str = ..., + disable_write_with_if_match: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["seafile"], + /, + *, + repo_name: str, + root: str = ..., + endpoint: str = ..., + username: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["sftp"], + /, + *, + endpoint: str = ..., + root: str = ..., + user: str = ..., + key: str = ..., + known_hosts_strategy: str = ..., + enable_copy: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["sled"], + /, + *, + datadir: str = ..., + root: str = ..., + tree: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["sqlite"], + /, + *, + connection_string: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["supabase"], + /, + *, + bucket: str, + root: str = ..., + endpoint: str = ..., + key: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["surrealdb"], + /, + *, + connection_string: str = ..., + username: str = ..., + password: str = ..., + namespace: str = ..., + database: str = ..., + table: str = ..., + key_field: str = ..., + value_field: str = ..., + root: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["swift"], + /, + *, + endpoint: str = ..., + container: str = ..., + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["upyun"], + /, + *, + bucket: str, + root: str = ..., + operator: str = ..., + password: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["vercel_artifacts"], + /, + *, + access_token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["vercel_blob"], + /, + *, + root: str = ..., + token: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["webdav"], + /, + *, + endpoint: str = ..., + username: str = ..., + password: str = ..., + token: str = ..., + root: str = ..., + disable_copy: _bool = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["webhdfs"], + /, + *, + root: str = ..., + endpoint: str = ..., + delegation: str = ..., + disable_list_batch: _bool = ..., + atomic_write_dir: str = ..., + ) -> None: ... + @overload + def __init__( + self, + scheme: Literal["yandex_disk"], + /, + *, + access_token: str, + root: str = ..., + ) -> None: ... + @overload + def __init__(self, scheme: str, /, **kwargs: str) -> None: ... diff --git a/bindings/python/python/opendal/__init__.pyi b/bindings/python/python/opendal/__init__.pyi index 6d6645a60bbc..663cbc677c20 100644 --- a/bindings/python/python/opendal/__init__.pyi +++ b/bindings/python/python/opendal/__init__.pyi @@ -15,16 +15,16 @@ # specific language governing permissions and limitations # under the License. -from typing import Any, AsyncIterable, Iterable, Optional, final, Union, Type +from typing import AsyncIterable, Iterable, Optional, final, Union, Type from types import TracebackType from opendal import exceptions as exceptions from opendal import layers as layers from opendal.layers import Layer +from opendal.__base import _Base @final -class Operator: - def __init__(self, scheme: str, **kwargs: Any) -> None: ... +class Operator(_Base): def layer(self, layer: Layer) -> "Operator": ... def open(self, path: str, mode: str) -> File: ... def read(self, path: str) -> bytes: ... @@ -51,8 +51,7 @@ class Operator: def to_async_operator(self) -> AsyncOperator: ... @final -class AsyncOperator: - def __init__(self, scheme: str, **kwargs: Any) -> None: ... +class AsyncOperator(_Base): def layer(self, layer: Layer) -> "AsyncOperator": ... async def open(self, path: str, mode: str) -> AsyncFile: ... async def read(self, path: str) -> bytes: ... diff --git a/dev/Cargo.lock b/dev/Cargo.lock index bbc6ca74ea1e..f0479e85386b 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -66,6 +66,15 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "basic-toml" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "823388e228f614e9558c6804262db37960ec8821856535f5c3f59913140558f8" +dependencies = [ + "serde", +] + [[package]] name = "clap" version = "4.5.23" @@ -147,6 +156,15 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + [[package]] name = "humantime" version = "2.1.0" @@ -159,6 +177,18 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + [[package]] name = "log" version = "0.4.22" @@ -171,6 +201,38 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "odev" version = "0.0.1" @@ -180,9 +242,16 @@ dependencies = [ "env_logger", "log", "pretty_assertions", + "rinja", "syn", ] +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + [[package]] name = "pretty_assertions" version = "1.4.1" @@ -240,6 +309,73 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rinja" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dc4940d00595430b3d7d5a01f6222b5e5b51395d1120bdb28d854bb8abb17a5" +dependencies = [ + "humansize", + "itoa", + "percent-encoding", + "rinja_derive", +] + +[[package]] +name = "rinja_derive" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d9ed0146aef6e2825f1b1515f074510549efba38d71f4554eec32eb36ba18b" +dependencies = [ + "basic-toml", + "memchr", + "mime", + "mime_guess", + "proc-macro2", + "quote", + "rinja_parser", + "rustc-hash", + "serde", + "syn", +] + +[[package]] +name = "rinja_parser" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f9a866e2e00a7a1fb27e46e9e324a6f7c0e7edc4543cae1d38f4e4a100c610" +dependencies = [ + "memchr", + "nom", + "serde", +] + +[[package]] +name = "rustc-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" + +[[package]] +name = "serde" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "strsim" version = "0.11.1" @@ -257,6 +393,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + [[package]] name = "unicode-ident" version = "1.0.14" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 9cd342176c65..9d0e98acd22d 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -32,7 +32,8 @@ anyhow = "1.0.95" clap = { version = "4.5.23", features = ["derive"] } env_logger = "0.11.6" log = "0.4.22" -syn = { version = "2.0.91", features = ['parsing', 'full', 'derive', 'visit', 'extra-traits'] } +rinja = "0.3.5" +syn = { version = "2.0.91", features = ["visit", "full", "extra-traits"] } [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index c6bef062034e..68bd9b4577d5 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -17,10 +17,73 @@ use crate::generate::parser::Services; use anyhow::Result; +use rinja::Template; +use std::fs; use std::path::PathBuf; -pub fn generate(_project_root: PathBuf, services: &Services) -> Result<()> { - println!("{:?}", services); +use super::parser::{ConfigType, Service}; + +// Using the template in this path, relative +// to the `templates` dir in the crate root +#[derive(Template)] +#[template(path = "python.py.jinja2", escape = "none")] +struct PythonTemplate { + services: Vec<(String, Service)>, +} + +/// TODO: add a common utils to parse enabled features from cargo.toml +fn enabled_service(srv: &str) -> bool { + match srv { + // not enabled in bindings/python/Cargo.toml + "etcd" | "foundationdb" | "ftp" | "hdfs" | "rocksdb" | "tikv" => false, + _ => true, + } +} + +pub fn generate(project_root: PathBuf, services: &Services) -> Result<()> { + let mut v = Vec::from_iter( + services + .clone() + .into_iter() + .filter(|x| enabled_service(x.0.as_str())), + ); + + // move required options at beginning. + for srv in &mut v { + let mut v = Vec::from_iter(srv.1.config.clone().into_iter().enumerate()); + + v.sort_by_key(|a| (a.1.optional, a.0)); + + srv.1.config = v.iter().map(|f| f.1.clone()).collect(); + } + + let tmpl = PythonTemplate { services: v }; + + let t = tmpl.render().expect("should render template"); + + let output_file: String = project_root + .join("bindings/python/python/opendal/__base.pyi") + .to_str() + .expect("should build output file path") + .into(); + + fs::write(output_file, t).expect("failed to write result to file"); Ok(()) } + +impl ConfigType { + pub fn python_type(&self) -> String { + match self { + ConfigType::Bool => "_bool".into(), + ConfigType::Duration => "_duration".into(), + ConfigType::I64 + | ConfigType::Usize + | ConfigType::U64 + | ConfigType::U32 + | ConfigType::U16 => "_int".into(), + ConfigType::Vec => "_strings".into(), + ConfigType::String => "str".into(), + } + } +} diff --git a/dev/templates/python.py.jinja2 b/dev/templates/python.py.jinja2 new file mode 100644 index 000000000000..7c8b2732efc2 --- /dev/null +++ b/dev/templates/python.py.jinja2 @@ -0,0 +1,62 @@ +""" +this file is generated by opendal/dev/generate/binding_python.rs, and opendal.__base doesn't exists. + +DO NOT EDIT IT Manually +""" + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from typing import overload, Literal, TypeAlias + +# `true`/`false`` in any case, for example, `true`/`True`/`TRUE` `false`/`False`/`FALSE` +_bool: TypeAlias = str +# a str represent a int, for example, `"10"`/`"0"` +_int: TypeAlias = str + +# a human readable duration string +# see https://docs.rs/humantime/latest/humantime/fn.parse_duration.html +# for more details +_duration: TypeAlias = str + + +# A "," separated string, for example `"127.0.0.1:1,127.0.0.1:2"` +_strings: TypeAlias = str + +class _Base: + """this is not a real base class but typing mixin, + + The services list here is support by opendal pypi wheel. + """ + +{% for srv in services %} + @overload + def __init__(self, + scheme: Literal["{{srv.0}}"], + /, + *, + {% for field in srv.1.config.clone().into_iter() %} + {% if field.deprecated.is_some() %} + # deprecated: {{ field.deprecated.unwrap().note }} + {% endif %} + {{field.name}}: {{field.value.python_type()}} {% if field.optional %} = ... {% endif %}, + {% endfor %} + ) -> None: ... +{% endfor %} + + @overload + def __init__(self, scheme:str, /, **kwargs: str) -> None: ...