diff --git a/.github/workflows/ci_odev.yml b/.github/workflows/ci_odev.yml new file mode 100644 index 000000000000..fcf2c39ca12a --- /dev/null +++ b/.github/workflows/ci_odev.yml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: ODev CI + +on: + push: + branches: + - main + pull_request: + branches: + - main + paths: + - "**/*.rs" + - ".github/workflows/ci_odev.yml" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + check_clippy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: ./.github/actions/setup + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Cargo clippy + working-directory: dev + run: cargo clippy --all-targets --all-features -- -D warnings + + test_dev: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Setup Rust toolchain + uses: ./.github/actions/setup + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Cargo Test + working-directory: dev + run: cargo test diff --git a/dev/Cargo.toml b/dev/Cargo.toml index 7674646a94a8..9cd342176c65 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -32,7 +32,7 @@ anyhow = "1.0.95" clap = { version = "4.5.23", features = ["derive"] } env_logger = "0.11.6" log = "0.4.22" -syn = { version = "2.0.91", features = ["visit","full","extra-traits"] } +syn = { version = "2.0.91", features = ['parsing', 'full', 'derive', 'visit', 'extra-traits'] } [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/dev/src/generate/binding_python.rs b/dev/src/generate/binding_python.rs index 7f3e3bdef83f..c6bef062034e 100644 --- a/dev/src/generate/binding_python.rs +++ b/dev/src/generate/binding_python.rs @@ -17,8 +17,9 @@ use crate::generate::parser::Services; use anyhow::Result; +use std::path::PathBuf; -pub fn generate(services: &Services) -> Result<()> { +pub fn generate(_project_root: PathBuf, services: &Services) -> Result<()> { println!("{:?}", services); Ok(()) diff --git a/dev/src/generate/mod.rs b/dev/src/generate/mod.rs index 743aa3b1c483..71ec44429998 100644 --- a/dev/src/generate/mod.rs +++ b/dev/src/generate/mod.rs @@ -25,10 +25,11 @@ use std::path::PathBuf; pub fn run(language: &str) -> Result<()> { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let services_path = manifest_dir.join("../core/src/services").canonicalize()?; + let project_root = manifest_dir.join("..").canonicalize()?; let services = parser::parse(&services_path.to_string_lossy())?; match language { - "python" | "py" => binding_python::generate(&services), + "python" | "py" => binding_python::generate(project_root, &services), _ => Err(anyhow::anyhow!("Unsupported language: {}", language)), } } diff --git a/dev/src/generate/parser.rs b/dev/src/generate/parser.rs index e8c3b6f03551..b977e7bfa3d5 100644 --- a/dev/src/generate/parser.rs +++ b/dev/src/generate/parser.rs @@ -18,34 +18,35 @@ use anyhow::Result; use anyhow::{anyhow, Context}; use log::debug; -use std::collections::hash_map; use std::collections::HashMap; -use std::fs; use std::fs::read_dir; use std::str::FromStr; -use syn::{Field, GenericArgument, Item, ItemStruct, PathArguments, Type, TypePath}; +use std::{fs, vec}; +use syn::{Field, GenericArgument, Item, LitStr, PathArguments, Type, TypePath}; #[derive(Debug, Clone)] pub struct Services(HashMap); impl IntoIterator for Services { type Item = (String, Service); - type IntoIter = hash_map::IntoIter; + type IntoIter = vec::IntoIter<(String, Service)>; fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() + let mut v = Vec::from_iter(self.0); + v.sort(); + v.into_iter() } } /// Service represents a service supported by opendal core, like `s3` and `fs` -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] pub struct Service { /// All configurations for this service. pub config: Vec, } /// Config represents a configuration item for a service. -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)] pub struct Config { /// The name of this config, for example, `access_key_id` and `secret_access_key` pub name: String, @@ -53,13 +54,15 @@ pub struct Config { pub value: ConfigType, /// If given config is optional or not. pub optional: bool, + /// if this field is deprecated, a deprecated message will be provided. + pub deprecated: Option, /// The comments for this config. /// /// All white spaces and extra new lines will be trimmed. pub comments: String, } -#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)] pub enum ConfigType { /// Mapping to rust's `bool` Bool, @@ -108,6 +111,36 @@ impl FromStr for ConfigType { } } +/// The deprecated attribute for a field. +/// +/// For given field: +/// +/// ```text +/// #[deprecated( +/// since = "0.52.0", +/// note = "Please use `delete_max_size` instead of `batch_max_operations`" +/// )] +/// pub batch_max_operations: Option, +/// ``` +/// +/// We will have: +/// +/// ```text +/// AttrDeprecated { +/// since: "0.52.0", +/// note: "Please use `delete_max_size` instead of `batch_max_operations`" +/// } +/// ``` +/// +/// - since = "0.52.0" +#[derive(Debug, Default, Clone, Eq, PartialEq, Ord, PartialOrd)] +pub struct AttrDeprecated { + /// The since of this deprecated field. + pub since: String, + /// The note for this deprecated field. + pub note: String, +} + /// List and parse given path to a `Services` struct. pub fn parse(path: &str) -> Result { let mut map = HashMap::default(); @@ -196,6 +229,8 @@ impl ServiceParser { .clone() .ok_or_else(|| anyhow!("field name is missing for {:?}", &field))?; + let deprecated = Self::parse_attr_deprecated(&field)?; + let (cfg_type, optional) = match &field.ty { Type::Path(TypePath { path, .. }) => { let segment = path @@ -226,6 +261,7 @@ impl ServiceParser { }; let typ = type_name.as_str().parse()?; + let optional = optional || typ == ConfigType::Bool; (typ, optional) } @@ -236,9 +272,61 @@ impl ServiceParser { name: name.to_string(), value: cfg_type, optional, + deprecated, comments: "".to_string(), }) } + + /// Parse the deprecated attr from the field. + /// + /// ```text + /// #[deprecated( + /// since = "0.52.0", + /// note = "Please use `delete_max_size` instead of `batch_max_operations`" + /// )] + /// pub batch_max_operations: Option, + /// ``` + fn parse_attr_deprecated(field: &Field) -> Result> { + let deprecated: Vec<_> = field + .attrs + .iter() + .filter(|attr| attr.path().is_ident("deprecated")) + .collect(); + + if deprecated.len() > 1 { + return Err(anyhow!("only one deprecated attribute is allowed")); + } + + let Some(attr) = deprecated.first() else { + return Ok(None); + }; + + let mut result = AttrDeprecated::default(); + + attr.parse_nested_meta(|meta| { + // this parses the `since` + if meta.path.is_ident("since") { + // this parses the `=` + let value = meta.value()?; + // this parses the value + let s: LitStr = value.parse()?; + result.since = s.value(); + } + + // this parses the `note` + if meta.path.is_ident("note") { + // this parses the `=` + let value = meta.value()?; + // this parses the value + let s: LitStr = value.parse()?; + result.note = s.value(); + } + + Ok(()) + })?; + + Ok(Some(result)) + } } #[cfg(test)] @@ -246,6 +334,7 @@ mod tests { use super::*; use pretty_assertions::assert_eq; use std::path::PathBuf; + use syn::ItemStruct; #[test] fn test_parse_field() { @@ -256,6 +345,7 @@ mod tests { name: "root".to_string(), value: ConfigType::String, optional: true, + deprecated: None, comments: "".to_string(), }, ), @@ -265,6 +355,7 @@ mod tests { name: "root".to_string(), value: ConfigType::String, optional: false, + deprecated: None, comments: "".to_string(), }, ), @@ -495,12 +586,26 @@ impl Debug for S3Config { let service = parser.parse().unwrap(); assert_eq!(service.config.len(), 26); + assert_eq!( + service.config[21], + Config { + name: "batch_max_operations".to_string(), + value: ConfigType::Usize, + optional: true, + deprecated: Some(AttrDeprecated { + since: "0.52.0".to_string(), + note: "Please use `delete_max_size` instead of `batch_max_operations`".into(), + }), + comments: "".to_string(), + }, + ); assert_eq!( service.config[25], Config { name: "disable_write_with_if_match".to_string(), value: ConfigType::Bool, - optional: false, + optional: true, + deprecated: None, comments: "".to_string(), }, );