From 631f2fe843d7fae9403e4f3482ba2748beef5394 Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 01:09:55 +0800 Subject: [PATCH 1/3] parse deprecated note from config --- dev/Cargo.lock | 47 +++++++++++++++++++++ dev/Cargo.toml | 5 ++- dev/src/generate/binding_python.rs | 3 +- dev/src/generate/mod.rs | 3 +- dev/src/generate/parser.rs | 68 +++++++++++++++++++++++++++--- 5 files changed, 117 insertions(+), 9 deletions(-) diff --git a/dev/Cargo.lock b/dev/Cargo.lock index bbc6ca74ea1e..d1a8e49471ac 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -118,6 +118,21 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "enquote" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" +dependencies = [ + "thiserror", +] + [[package]] name = "env_filter" version = "0.1.3" @@ -159,6 +174,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "log" version = "0.4.22" @@ -177,9 +201,12 @@ version = "0.0.1" dependencies = [ "anyhow", "clap", + "enquote", "env_logger", + "itertools", "log", "pretty_assertions", + "proc-macro2", "syn", ] @@ -257,6 +284,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "unicode-ident" version = "1.0.14" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 7674646a94a8..31120d77f674 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -30,9 +30,12 @@ version = "0.0.1" [dependencies] anyhow = "1.0.95" clap = { version = "4.5.23", features = ["derive"] } +enquote = "1.1.0" env_logger = "0.11.6" +itertools = "0.13.0" log = "0.4.22" -syn = { version = "2.0.91", features = ["visit","full","extra-traits"] } +proc-macro2 = { version = "1.0.91", features = ["span-locations"] } +syn = { version = "2.0.91", features = ['parsing', 'full', 'derive', 'visit', 'extra-traits'] } [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index 7f3e3bdef83f..c6bef062034e 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -17,8 +17,9 @@ use crate::generate::parser::Services; use anyhow::Result; +use std::path::PathBuf; -pub fn generate(services: &Services) -> Result<()> { +pub fn generate(_project_root: PathBuf, services: &Services) -> Result<()> { println!("{:?}", services); Ok(()) diff --git a/dev/src/generate/mod.rs b/dev/src/generate/mod.rs index 743aa3b1c483..71ec44429998 100644 --- a/dev/src/generate/mod.rs +++ b/dev/src/generate/mod.rs @@ -25,10 +25,11 @@ use std::path::PathBuf; pub fn run(language: &str) -> Result<()> { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let services_path = manifest_dir.join("../core/src/services").canonicalize()?; + let project_root = manifest_dir.join("..").canonicalize()?; let services = parser::parse(&services_path.to_string_lossy())?; match language { - "python" | "py" => binding_python::generate(&services), + "python" | "py" => binding_python::generate(project_root, &services), _ => Err(anyhow::anyhow!("Unsupported language: {}", language)), } } diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index e8c3b6f03551..be0085831852 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -17,23 +17,23 @@ use anyhow::Result; use anyhow::{anyhow, Context}; +use itertools::Itertools; use log::debug; -use std::collections::hash_map; use std::collections::HashMap; -use std::fs; use std::fs::read_dir; use std::str::FromStr; -use syn::{Field, GenericArgument, Item, ItemStruct, PathArguments, Type, TypePath}; +use std::{fs, vec}; +use syn::{Field, GenericArgument, Item, PathArguments, Type, TypePath}; #[derive(Debug, Clone)] pub struct Services(HashMap); impl IntoIterator for Services { type Item = (String, Service); - type IntoIter = hash_map::IntoIter; + type IntoIter = vec::IntoIter<(String, Service)>; fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() + self.0.into_iter().sorted_by_key(|x| x.0.clone()) } } @@ -53,6 +53,8 @@ pub struct Config { pub value: ConfigType, /// If given config is optional or not. pub optional: bool, + /// if this field is deprecated, a deprecated message will be provided. + pub deprecated: Option, /// The comments for this config. /// /// All white spaces and extra new lines will be trimmed. @@ -196,6 +198,8 @@ impl ServiceParser { .clone() .ok_or_else(|| anyhow!("field name is missing for {:?}", &field))?; + let deprecated = Self::deprecated_note(&field); + let (cfg_type, optional) = match &field.ty { Type::Path(TypePath { path, .. }) => { let segment = path @@ -226,6 +230,7 @@ impl ServiceParser { }; let typ = type_name.as_str().parse()?; + let optional = optional || typ == ConfigType::Bool; (typ, optional) } @@ -236,15 +241,51 @@ impl ServiceParser { name: name.to_string(), value: cfg_type, optional, + deprecated, comments: "".to_string(), }) } + + fn deprecated_note(field: &Field) -> Option { + for attr in &field.attrs { + if !attr.path().is_ident("deprecated") { + continue; + } + + let meta_list = match &attr.meta { + syn::Meta::List(meta_list) => meta_list, + _ => continue, + }; + + let tokens = Vec::from_iter(meta_list.tokens.clone().into_iter()); + for (index, token) in tokens.iter().enumerate() { + let ident = match token { + proc_macro2::TokenTree::Ident(ident) => ident, + _ => { + continue; + } + }; + + if ident == "note" { + return tokens + .get(index + 2) + .expect("deprecated attribute missing note") + .span() + .source_text() + .map(|s| enquote::unquote(s.as_str()).expect("should unquote string")); + } + } + } + + return None; + } } #[cfg(test)] mod tests { use super::*; use pretty_assertions::assert_eq; + use syn::ItemStruct; use std::path::PathBuf; #[test] @@ -256,6 +297,7 @@ mod tests { name: "root".to_string(), value: ConfigType::String, optional: true, + deprecated: None, comments: "".to_string(), }, ), @@ -265,6 +307,7 @@ mod tests { name: "root".to_string(), value: ConfigType::String, optional: false, + deprecated: None, comments: "".to_string(), }, ), @@ -495,12 +538,25 @@ impl Debug for S3Config { let service = parser.parse().unwrap(); assert_eq!(service.config.len(), 26); + assert_eq!( + service.config[21], + Config { + name: "batch_max_operations".to_string(), + value: ConfigType::Usize, + optional: true, + deprecated: Some( + "Please use `delete_max_size` instead of `batch_max_operations`".into() + ), + comments: "".to_string(), + }, + ); assert_eq!( service.config[25], Config { name: "disable_write_with_if_match".to_string(), value: ConfigType::Bool, - optional: false, + optional: true, + deprecated: None, comments: "".to_string(), }, ); From 7413f02048fa18d29933a2db288198c5938f65cb Mon Sep 17 00:00:00 2001 From: Trim21 Date: Fri, 27 Dec 2024 03:20:33 +0800 Subject: [PATCH 2/3] run test in ci --- .github/workflows/ci_odev.yml | 62 +++++++++++++++++++++++++++++++++++ dev/src/generate/parser.rs | 8 ++--- 2 files changed, 65 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/ci_odev.yml diff --git a/.github/workflows/ci_odev.yml b/.github/workflows/ci_odev.yml new file mode 100644 index 000000000000..fcf2c39ca12a --- /dev/null +++ b/.github/workflows/ci_odev.yml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: ODev CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + paths: + - "**/*.rs" + - ".github/workflows/ci_odev.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + check_clippy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: ./.github/actions/setup + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Cargo clippy + working-directory: dev + run: cargo clippy --all-targets --all-features -- -D warnings + + test_dev: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: ./.github/actions/setup + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Cargo Test + working-directory: dev + run: cargo test diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index be0085831852..2f721d88e196 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -261,9 +261,7 @@ impl ServiceParser { for (index, token) in tokens.iter().enumerate() { let ident = match token { proc_macro2::TokenTree::Ident(ident) => ident, - _ => { - continue; - } + _ => continue, }; if ident == "note" { @@ -277,7 +275,7 @@ impl ServiceParser { } } - return None; + None } } @@ -285,8 +283,8 @@ impl ServiceParser { mod tests { use super::*; use pretty_assertions::assert_eq; - use syn::ItemStruct; use std::path::PathBuf; + use syn::ItemStruct; #[test] fn test_parse_field() { From 82526bf553a787791756a6de58870db658fa5e7b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Fri, 27 Dec 2024 13:11:55 +0800 Subject: [PATCH 3/3] Parse deprecated attrs Signed-off-by: Xuanwo --- dev/Cargo.lock | 47 -------------- dev/Cargo.toml | 3 - dev/src/generate/parser.rs | 123 ++++++++++++++++++++++++++----------- 3 files changed, 87 insertions(+), 86 deletions(-) diff --git a/dev/Cargo.lock b/dev/Cargo.lock index d1a8e49471ac..bbc6ca74ea1e 100644 --- a/dev/Cargo.lock +++ b/dev/Cargo.lock @@ -118,21 +118,6 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" -[[package]] -name = "either" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" - -[[package]] -name = "enquote" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" -dependencies = [ - "thiserror", -] - [[package]] name = "env_filter" version = "0.1.3" @@ -174,15 +159,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "log" version = "0.4.22" @@ -201,12 +177,9 @@ version = "0.0.1" dependencies = [ "anyhow", "clap", - "enquote", "env_logger", - "itertools", "log", "pretty_assertions", - "proc-macro2", "syn", ] @@ -284,26 +257,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "unicode-ident" version = "1.0.14" diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 31120d77f674..9cd342176c65 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -30,11 +30,8 @@ version = "0.0.1" [dependencies] anyhow = "1.0.95" clap = { version = "4.5.23", features = ["derive"] } -enquote = "1.1.0" env_logger = "0.11.6" -itertools = "0.13.0" log = "0.4.22" -proc-macro2 = { version = "1.0.91", features = ["span-locations"] } syn = { version = "2.0.91", features = ['parsing', 'full', 'derive', 'visit', 'extra-traits'] } [dev-dependencies] diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index 2f721d88e196..b977e7bfa3d5 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -17,13 +17,12 @@ use anyhow::Result; use anyhow::{anyhow, Context}; -use itertools::Itertools; use log::debug; use std::collections::HashMap; use std::fs::read_dir; use std::str::FromStr; use std::{fs, vec}; -use syn::{Field, GenericArgument, Item, PathArguments, Type, TypePath}; +use syn::{Field, GenericArgument, Item, LitStr, PathArguments, Type, TypePath}; #[derive(Debug, Clone)] pub struct Services(HashMap); @@ -33,19 +32,21 @@ impl IntoIterator for Services { type IntoIter = vec::IntoIter<(String, Service)>; fn into_iter(self) -> Self::IntoIter { - self.0.into_iter().sorted_by_key(|x| x.0.clone()) + let mut v = Vec::from_iter(self.0); + v.sort(); + v.into_iter() } } /// Service represents a service supported by opendal core, like `s3` and `fs` -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] pub struct Service { /// All configurations for this service. pub config: Vec, } /// Config represents a configuration item for a service. -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] pub struct Config { /// The name of this config, for example, `access_key_id` and `secret_access_key` pub name: String, @@ -54,14 +55,14 @@ pub struct Config { /// If given config is optional or not. pub optional: bool, /// if this field is deprecated, a deprecated message will be provided. - pub deprecated: Option, + pub deprecated: Option, /// The comments for this config. /// /// All white spaces and extra new lines will be trimmed. pub comments: String, } -#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)] pub enum ConfigType { /// Mapping to rust's `bool` Bool, @@ -110,6 +111,36 @@ impl FromStr for ConfigType { } } +/// The deprecated attribute for a field. +/// +/// For given field: +/// +/// ```text +/// #[deprecated( +/// since = "0.52.0", +/// note = "Please use `delete_max_size` instead of `batch_max_operations`" +/// )] +/// pub batch_max_operations: Option, +/// ``` +/// +/// We will have: +/// +/// ```text +/// AttrDeprecated { +/// since: "0.52.0", +/// note: "Please use `delete_max_size` instead of `batch_max_operations`" +/// } +/// ``` +/// +/// - since = "0.52.0" +#[derive(Debug, Default, Clone, Eq, PartialEq, Ord, PartialOrd)] +pub struct AttrDeprecated { + /// The since of this deprecated field. + pub since: String, + /// The note for this deprecated field. + pub note: String, +} + /// List and parse given path to a `Services` struct. pub fn parse(path: &str) -> Result { let mut map = HashMap::default(); @@ -198,7 +229,7 @@ impl ServiceParser { .clone() .ok_or_else(|| anyhow!("field name is missing for {:?}", &field))?; - let deprecated = Self::deprecated_note(&field); + let deprecated = Self::parse_attr_deprecated(&field)?; let (cfg_type, optional) = match &field.ty { Type::Path(TypePath { path, .. }) => { @@ -246,36 +277,55 @@ impl ServiceParser { }) } - fn deprecated_note(field: &Field) -> Option { - for attr in &field.attrs { - if !attr.path().is_ident("deprecated") { - continue; - } + /// Parse the deprecated attr from the field. + /// + /// ```text + /// #[deprecated( + /// since = "0.52.0", + /// note = "Please use `delete_max_size` instead of `batch_max_operations`" + /// )] + /// pub batch_max_operations: Option, + /// ``` + fn parse_attr_deprecated(field: &Field) -> Result> { + let deprecated: Vec<_> = field + .attrs + .iter() + .filter(|attr| attr.path().is_ident("deprecated")) + .collect(); - let meta_list = match &attr.meta { - syn::Meta::List(meta_list) => meta_list, - _ => continue, - }; + if deprecated.len() > 1 { + return Err(anyhow!("only one deprecated attribute is allowed")); + } - let tokens = Vec::from_iter(meta_list.tokens.clone().into_iter()); - for (index, token) in tokens.iter().enumerate() { - let ident = match token { - proc_macro2::TokenTree::Ident(ident) => ident, - _ => continue, - }; + let Some(attr) = deprecated.first() else { + return Ok(None); + }; - if ident == "note" { - return tokens - .get(index + 2) - .expect("deprecated attribute missing note") - .span() - .source_text() - .map(|s| enquote::unquote(s.as_str()).expect("should unquote string")); - } + let mut result = AttrDeprecated::default(); + + attr.parse_nested_meta(|meta| { + // this parses the `since` + if meta.path.is_ident("since") { + // this parses the `=` + let value = meta.value()?; + // this parses the value + let s: LitStr = value.parse()?; + result.since = s.value(); } - } - None + // this parses the `note` + if meta.path.is_ident("note") { + // this parses the `=` + let value = meta.value()?; + // this parses the value + let s: LitStr = value.parse()?; + result.note = s.value(); + } + + Ok(()) + })?; + + Ok(Some(result)) } } @@ -542,9 +592,10 @@ impl Debug for S3Config { name: "batch_max_operations".to_string(), value: ConfigType::Usize, optional: true, - deprecated: Some( - "Please use `delete_max_size` instead of `batch_max_operations`".into() - ), + deprecated: Some(AttrDeprecated { + since: "0.52.0".to_string(), + note: "Please use `delete_max_size` instead of `batch_max_operations`".into(), + }), comments: "".to_string(), }, );