dev: parse deprecated note from config and make bool optional (#5463)

Co-authored-by: Xuanwo <[email protected]>

.github/workflows/ci_odev.yml

@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: ODev CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "**/*.rs"
+      - ".github/workflows/ci_odev.yml"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+jobs:
+  check_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Cargo clippy
+        working-directory: dev
+        run: cargo clippy --all-targets --all-features -- -D warnings
+
+  test_dev:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Cargo Test
+        working-directory: dev
+        run: cargo test

dev/Cargo.toml

@@ -32,7 +32,7 @@ anyhow = "1.0.95"
 clap = { version = "4.5.23", features = ["derive"] }
 env_logger = "0.11.6"
 log = "0.4.22"
-syn = { version = "2.0.91", features = ["visit","full","extra-traits"] }
+syn = { version = "2.0.91", features = ['parsing', 'full', 'derive', 'visit', 'extra-traits'] }
 
 [dev-dependencies]
 pretty_assertions = "1.4.1"

dev/src/generate/binding_python.rs

@@ -17,8 +17,9 @@
 
 use crate::generate::parser::Services;
 use anyhow::Result;
+use std::path::PathBuf;
 
-pub fn generate(services: &Services) -> Result<()> {
+pub fn generate(_project_root: PathBuf, services: &Services) -> Result<()> {
     println!("{:?}", services);
 
     Ok(())

dev/src/generate/mod.rs

@@ -25,10 +25,11 @@ use std::path::PathBuf;
 pub fn run(language: &str) -> Result<()> {
     let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
     let services_path = manifest_dir.join("../core/src/services").canonicalize()?;
+    let project_root = manifest_dir.join("..").canonicalize()?;
     let services = parser::parse(&services_path.to_string_lossy())?;
 
     match language {
-        "python" | "py" => binding_python::generate(&services),
+        "python" | "py" => binding_python::generate(project_root, &services),
         _ => Err(anyhow::anyhow!("Unsupported language: {}", language)),
     }
 }

dev/src/generate/parser.rs

@@ -18,48 +18,51 @@
 use anyhow::Result;
 use anyhow::{anyhow, Context};
 use log::debug;
-use std::collections::hash_map;
 use std::collections::HashMap;
-use std::fs;
 use std::fs::read_dir;
 use std::str::FromStr;
-use syn::{Field, GenericArgument, Item, ItemStruct, PathArguments, Type, TypePath};
+use std::{fs, vec};
+use syn::{Field, GenericArgument, Item, LitStr, PathArguments, Type, TypePath};
 
 #[derive(Debug, Clone)]
 pub struct Services(HashMap<String, Service>);
 
 impl IntoIterator for Services {
     type Item = (String, Service);
-    type IntoIter = hash_map::IntoIter<String, Service>;
+    type IntoIter = vec::IntoIter<(String, Service)>;
 
     fn into_iter(self) -> Self::IntoIter {
-        self.0.into_iter()
+        let mut v = Vec::from_iter(self.0);
+        v.sort();
+        v.into_iter()
     }
 }
 
 /// Service represents a service supported by opendal core, like `s3` and `fs`
-#[derive(Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
 pub struct Service {
     /// All configurations for this service.
     pub config: Vec<Config>,
 }
 
 /// Config represents a configuration item for a service.
-#[derive(Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
 pub struct Config {
     /// The name of this config, for example, `access_key_id` and `secret_access_key`
     pub name: String,
     /// The value type this config.
     pub value: ConfigType,
     /// If given config is optional or not.
     pub optional: bool,
+    /// if this field is deprecated, a deprecated message will be provided.
+    pub deprecated: Option<AttrDeprecated>,
     /// The comments for this config.
     ///
     /// All white spaces and extra new lines will be trimmed.
     pub comments: String,
 }
 
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)]
 pub enum ConfigType {
     /// Mapping to rust's `bool`
     Bool,
@@ -108,6 +111,36 @@ impl FromStr for ConfigType {
     }
 }
 
+/// The deprecated attribute for a field.
+///
+/// For given field:
+///
+/// ```text
+/// #[deprecated(
+///     since = "0.52.0",
+///     note = "Please use `delete_max_size` instead of `batch_max_operations`"
+/// )]
+/// pub batch_max_operations: Option<usize>,
+/// ```
+///
+/// We will have:
+///
+/// ```text
+/// AttrDeprecated {
+///    since: "0.52.0",
+///    note: "Please use `delete_max_size` instead of `batch_max_operations`"
+/// }
+/// ```
+///
+/// - since = "0.52.0"
+#[derive(Debug, Default, Clone, Eq, PartialEq, Ord, PartialOrd)]
+pub struct AttrDeprecated {
+    /// The since of this deprecated field.
+    pub since: String,
+    /// The note for this deprecated field.
+    pub note: String,
+}
+
 /// List and parse given path to a `Services` struct.
 pub fn parse(path: &str) -> Result<Services> {
     let mut map = HashMap::default();
@@ -196,6 +229,8 @@ impl ServiceParser {
             .clone()
             .ok_or_else(|| anyhow!("field name is missing for {:?}", &field))?;
 
+        let deprecated = Self::parse_attr_deprecated(&field)?;
+
         let (cfg_type, optional) = match &field.ty {
             Type::Path(TypePath { path, .. }) => {
                 let segment = path
@@ -226,6 +261,7 @@ impl ServiceParser {
                 };
 
                 let typ = type_name.as_str().parse()?;
+                let optional = optional || typ == ConfigType::Bool;
 
                 (typ, optional)
             }
@@ -236,16 +272,69 @@ impl ServiceParser {
             name: name.to_string(),
             value: cfg_type,
             optional,
+            deprecated,
             comments: "".to_string(),
         })
     }
+
+    /// Parse the deprecated attr from the field.
+    ///
+    /// ```text
+    /// #[deprecated(
+    ///     since = "0.52.0",
+    ///     note = "Please use `delete_max_size` instead of `batch_max_operations`"
+    /// )]
+    /// pub batch_max_operations: Option<usize>,
+    /// ```
+    fn parse_attr_deprecated(field: &Field) -> Result<Option<AttrDeprecated>> {
+        let deprecated: Vec<_> = field
+            .attrs
+            .iter()
+            .filter(|attr| attr.path().is_ident("deprecated"))
+            .collect();
+
+        if deprecated.len() > 1 {
+            return Err(anyhow!("only one deprecated attribute is allowed"));
+        }
+
+        let Some(attr) = deprecated.first() else {
+            return Ok(None);
+        };
+
+        let mut result = AttrDeprecated::default();
+
+        attr.parse_nested_meta(|meta| {
+            // this parses the `since`
+            if meta.path.is_ident("since") {
+                // this parses the `=`
+                let value = meta.value()?;
+                // this parses the value
+                let s: LitStr = value.parse()?;
+                result.since = s.value();
+            }
+
+            // this parses the `note`
+            if meta.path.is_ident("note") {
+                // this parses the `=`
+                let value = meta.value()?;
+                // this parses the value
+                let s: LitStr = value.parse()?;
+                result.note = s.value();
+            }
+
+            Ok(())
+        })?;
+
+        Ok(Some(result))
+    }
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use pretty_assertions::assert_eq;
     use std::path::PathBuf;
+    use syn::ItemStruct;
 
     #[test]
     fn test_parse_field() {
@@ -256,6 +345,7 @@ mod tests {
                     name: "root".to_string(),
                     value: ConfigType::String,
                     optional: true,
+                    deprecated: None,
                     comments: "".to_string(),
                 },
             ),
@@ -265,6 +355,7 @@ mod tests {
                     name: "root".to_string(),
                     value: ConfigType::String,
                     optional: false,
+                    deprecated: None,
                     comments: "".to_string(),
                 },
             ),
@@ -495,12 +586,26 @@ impl Debug for S3Config {
 
         let service = parser.parse().unwrap();
         assert_eq!(service.config.len(), 26);
+        assert_eq!(
+            service.config[21],
+            Config {
+                name: "batch_max_operations".to_string(),
+                value: ConfigType::Usize,
+                optional: true,
+                deprecated: Some(AttrDeprecated {
+                    since: "0.52.0".to_string(),
+                    note: "Please use `delete_max_size` instead of `batch_max_operations`".into(),
+                }),
+                comments: "".to_string(),
+            },
+        );
         assert_eq!(
             service.config[25],
             Config {
                 name: "disable_write_with_if_match".to_string(),
                 value: ConfigType::Bool,
-                optional: false,
+                optional: true,
+                deprecated: None,
                 comments: "".to_string(),
             },
         );