From 9efbaa9f0e1d36ab27bb0d48d0030a0f48124f73 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 6 Feb 2024 18:21:27 +0800 Subject: [PATCH 1/9] Save work Signed-off-by: Xuanwo --- scripts/constants.py | 43 ++++++++++----------- scripts/release.py | 69 +++++++++++++++++++++++++++++++++ scripts/release.sh | 91 -------------------------------------------- 3 files changed, 90 insertions(+), 113 deletions(-) mode change 100644 => 100755 scripts/constants.py create mode 100755 scripts/release.py delete mode 100755 scripts/release.sh diff --git a/scripts/constants.py b/scripts/constants.py old mode 100644 new mode 100755 index 7081a62f5cce..8e83cc257278 --- a/scripts/constants.py +++ b/scripts/constants.py @@ -16,25 +16,24 @@ # specific language governing permissions and limitations # under the License. -PACKAGES = [ - "core", - - "bin/oli", - "bin/oay", - "bin/ofs", - - "bindings/c", - "bindings/cpp", - "bindings/dotnet", - "bindings/haskell", - "bindings/java", - "bindings/lua", - "bindings/nodejs", - "bindings/ocaml", - "bindings/php", - "bindings/python", - "bindings/ruby", - - "integrations/dav-server", - "integrations/object_store", -] +from pathlib import Path + + +ROOT_DIR = Path(__file__).parent.parent + + +def list_packages(): + packages = ["core"] + + for dir in ["bin", "bindings", "integrations"]: + for path in (ROOT_DIR / dir).iterdir(): + if path.is_dir(): + packages.append(path.relative_to(ROOT_DIR)) + return packages + + +PACKAGES = list_packages() + +if __name__ == "__main__": + for project in PACKAGES: + print(project) diff --git a/scripts/release.py b/scripts/release.py new file mode 100755 index 000000000000..9bf1e869540a --- /dev/null +++ b/scripts/release.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import subprocess +from pathlib import Path +import tomllib + +OPENDAL_VERSION = os.getenv('OPENDAL_VERSION') +OPENDAL_VERSION_RC = os.getenv('OPENDAL_VERSION_RC', 'rc.1') + +if not OPENDAL_VERSION: + print("OPENDAL_VERSION is unset") + exit(1) +else: + print(f"var is set to '{OPENDAL_VERSION}'") + +# tar source code +release_version = OPENDAL_VERSION +# rc versions +rc_version = OPENDAL_VERSION_RC +# Corresponding git repository branch +git_branch = f"release-{release_version}-{rc_version}" + +dist_path = Path('dist') +if not dist_path.exists(): + dist_path.mkdir() + +print("> Checkout version branch") +subprocess.run(['git', 'checkout', '-B', git_branch], check=True) + +print("> Start package") +subprocess.run(['git', 'archive', '--format=tar.gz', '--output', f"dist/apache-opendal-{release_version}-src.tar.gz", '--prefix', f"apache-opendal-{release_version}-src/", '--add-file=Cargo.toml', git_branch], check=True) + +os.chdir('dist') +print("> Generate signature") +for i in Path('.').glob('*.tar.gz'): + print(i) + subprocess.run(['gpg', '--armor', '--output', f"{i}.asc", '--detach-sig', str(i)], check=True) + +print("> Check signature") +for i in Path('.').glob('*.tar.gz'): + print(i) + subprocess.run(['gpg', '--verify', f"{i}.asc", str(i)], check=True) + +print("> Generate sha512sum") +for i in Path('.').glob('*.tar.gz'): + print(i) + subprocess.run(['sha512sum', str(i)], stdout=open(f"{i}.sha512", "w")) + +print("> Check sha512sum") +for i in Path('.').glob('*.tar.gz.sha512'): + print(i) + subprocess.run(['sha512sum', '--check', str(i)], check=True) diff --git a/scripts/release.sh b/scripts/release.sh deleted file mode 100755 index 95c25e55648e..000000000000 --- a/scripts/release.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e - -if [ -z "${OPENDAL_VERSION}" ]; then - echo "OPENDAL_VERSION is unset" - exit 1 -else - echo "var is set to '$OPENDAL_VERSION'" -fi - -# tar source code -release_version=${OPENDAL_VERSION} -# rc versions -rc_version="${OPENDAL_VERSION_RC:-rc.1}" -# Corresponding git repository branch -git_branch=release-${release_version}-${rc_version} - -rm -rf dist -mkdir -p dist/ - -echo "> Checkout version branch" -git checkout -B "${git_branch}" - -# Please update this part while package list changed. -echo "> Update Cargo.toml to remove not released members" -PACKAGES=( - "bindings\/c" - "bindings\/ruby" - "bindings\/haskell" - "bindings\/lua" - "bindings\/dotnet" - "bindings\/ocaml" - "bindings\/php" - "bindings\/cpp" - "bin\/" - "integrations\/" -) -for package in "${PACKAGES[@]}"; do - if [[ "$OSTYPE" == "darwin"* ]]; then - # macOS system sed usage is different from others - sed -i '' "/${package}/d" Cargo.toml - else - sed -i "/${package}/d" Cargo.toml - fi -done - -echo "> Start package" -git archive --format=tar.gz --output="dist/apache-opendal-$release_version-src.tar.gz" --prefix="apache-opendal-$release_version-src/" --add-file=Cargo.toml "$git_branch" - -cd dist -echo "> Generate signature" -for i in *.tar.gz; do - echo "$i" - gpg --armor --output "$i.asc" --detach-sig "$i" -done -echo "> Check signature" -for i in *.tar.gz; do - echo "$i" - gpg --verify "$i.asc" "$i" -done -echo "> Generate sha512sum" -for i in *.tar.gz; do - echo "$i" - sha512sum "$i" >"$i.sha512" -done -echo "> Check sha512sum" -for i in *.tar.gz; do - echo "$i" - sha512sum --check "$i.sha512" -done - -cd .. -echo "> Check license" -docker run -it --rm -v "$(pwd):/github/workspace" -u "$(id -u):$(id -g)" ghcr.io/korandoru/hawkeye-native check From d66eb74060263bc97ad4d393261f459030c9c80b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sun, 18 Feb 2024 18:24:26 +0800 Subject: [PATCH 2/9] Fix build Signed-off-by: Xuanwo --- scripts/README.md | 2 +- scripts/constants.py | 57 ++++++++++++++++++- scripts/release.py | 133 +++++++++++++++++++++++++++---------------- scripts/verify.py | 4 +- 4 files changed, 141 insertions(+), 55 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 0d721e64880f..cb3f1339ce8d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,7 +8,7 @@ NOTES: all scripts must be running at root folder of OpenDAL project. ## Release ```shell -OPENDAL_VERSION=0.30.2 OPENDAL_VERSION_RC=rc1 ./scripts/release.sh +./scripts/release.py ``` > Before running release, please make sure you have bump all versions. diff --git a/scripts/constants.py b/scripts/constants.py index 8e83cc257278..b952a379b7e4 100755 --- a/scripts/constants.py +++ b/scripts/constants.py @@ -17,7 +17,7 @@ # under the License. from pathlib import Path - +import tomllib ROOT_DIR = Path(__file__).parent.parent @@ -34,6 +34,57 @@ def list_packages(): PACKAGES = list_packages() +# package dependencies is used to maintain the dependencies between packages. +# +# All packages are depend on `core` by default, so we should only maintain the exceptions in list. +PACKAGE_DEPENDENCIES = { + Path("bindings/go"): "bindings/c", + Path("bindings/swift"): "bindings/c", + Path("bindings/zig"): "bindings/c", +} + + +# fetch the package dependence, return `core` if not listed in `PACAKGE_DEPENDENCIES`. +def get_package_dependence(package: Path) -> str: + return PACKAGE_DEPENDENCIES.get(package, "core") + + +# input: Path to a Rust package like `core` and `bindings/python`. +def get_rust_package_version(path): + with open(ROOT_DIR / path / "Cargo.toml", "rb") as f: + data = tomllib.load(f) + version = data["package"]["version"] + return version + + +# get the package version by package name. +# +# For examples: +# core: `0.45.0` +# packages depends on core: `0.1.0+core.0.45.0` +# packages depends on bindings/c: `0.1.0+bindings_c.0.1.0` +def get_package_version(package): + if package == "core": + return get_rust_package_version("core") + + # NOTE: for now, all dependence package must be rust package. + dependence = get_package_dependence(package) + dependence_version = get_rust_package_version(dependence) + + cargo_toml = ROOT_DIR / package / "Cargo.toml" + # cargo_toml exists, we can get the version from Cargo.toml. + if cargo_toml.exists(): + package_version = get_rust_package_version(package) + return f"{package_version}+{dependence.replace('/', '_')}.{dependence_version}" + + # cargo_toml not exists, we should handle case by case ideally. + # + # However, those packages are not mature enough, it's much easier for us to always return `0.0.0` instead. + return f"0.0.0+{dependence.replace('/', '_')}.{dependence_version}" + + if __name__ == "__main__": - for project in PACKAGES: - print(project) + for v in PACKAGES: + print( + f"{v}: depends={get_package_dependence(v)}, version={get_package_version(v)}" + ) diff --git a/scripts/release.py b/scripts/release.py index 9bf1e869540a..df02ea41b088 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -16,54 +16,89 @@ # specific language governing permissions and limitations # under the License. -import os import subprocess from pathlib import Path -import tomllib - -OPENDAL_VERSION = os.getenv('OPENDAL_VERSION') -OPENDAL_VERSION_RC = os.getenv('OPENDAL_VERSION_RC', 'rc.1') - -if not OPENDAL_VERSION: - print("OPENDAL_VERSION is unset") - exit(1) -else: - print(f"var is set to '{OPENDAL_VERSION}'") - -# tar source code -release_version = OPENDAL_VERSION -# rc versions -rc_version = OPENDAL_VERSION_RC -# Corresponding git repository branch -git_branch = f"release-{release_version}-{rc_version}" - -dist_path = Path('dist') -if not dist_path.exists(): - dist_path.mkdir() - -print("> Checkout version branch") -subprocess.run(['git', 'checkout', '-B', git_branch], check=True) - -print("> Start package") -subprocess.run(['git', 'archive', '--format=tar.gz', '--output', f"dist/apache-opendal-{release_version}-src.tar.gz", '--prefix', f"apache-opendal-{release_version}-src/", '--add-file=Cargo.toml', git_branch], check=True) - -os.chdir('dist') -print("> Generate signature") -for i in Path('.').glob('*.tar.gz'): - print(i) - subprocess.run(['gpg', '--armor', '--output', f"{i}.asc", '--detach-sig', str(i)], check=True) - -print("> Check signature") -for i in Path('.').glob('*.tar.gz'): - print(i) - subprocess.run(['gpg', '--verify', f"{i}.asc", str(i)], check=True) - -print("> Generate sha512sum") -for i in Path('.').glob('*.tar.gz'): - print(i) - subprocess.run(['sha512sum', str(i)], stdout=open(f"{i}.sha512", "w")) - -print("> Check sha512sum") -for i in Path('.').glob('*.tar.gz.sha512'): - print(i) - subprocess.run(['sha512sum', '--check', str(i)], check=True) +from constants import get_package_version, get_package_dependence, PACKAGES + +ROOT_DIR = Path(__file__).parent.parent + +# If this package depends on `core`, we need to update it's core version in `Cargo.toml` file. +def update_package(path): + # skip core package. + if path == "core": + return + if get_package_dependence(path) != "core": + return + + core_version = get_package_version("core") + + +def archive_package(path): + print(f"Archive package {path} started") + + version = get_package_version(path) + name = f"apache-opendal-{str(path).replace('/', '-')}-{version}-src" + + ls_command = ["git", "ls-files", "."] + ls_result = subprocess.run( + ls_command, cwd=ROOT_DIR / path, capture_output=True, check=True, text=True + ) + + tar_command = [ + "tar", + "-zcf", + f"{ROOT_DIR}/dist/{name}.tar.gz", + "--transform", + f"s,^,{name}/,", + "-T", + "-", + ] + subprocess.run( + tar_command, + cwd=ROOT_DIR / path, + input=ls_result.stdout.encode("utf-8"), + check=True, + ) + + print(f"Archive package {path} to dist/{name}.tar.gz") + +def generate_signature(): + for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"): + print(f"Generate signature for {i}") + subprocess.run( + ["gpg", "--yes", "--armor", "--output", f"{i}.asc", "--detach-sig", str(i)], + cwd=ROOT_DIR / "dist", + check=True, + ) + + for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"): + print(f"Check signature for {i}") + subprocess.run( + ["gpg", "--verify", f"{i}.asc", str(i)], cwd=ROOT_DIR / "dist", check=True + ) + + +def generate_checksum(): + for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"): + print(f"Generate checksum for {i}") + subprocess.run( + ["sha512sum", str(i.relative_to(ROOT_DIR / "dist"))], + stdout=open(f"{i}.sha512", "w"), + cwd=ROOT_DIR / "dist", + check=True, + ) + + for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"): + print(f"Check checksum for {i}") + subprocess.run( + ["sha512sum", "--check", f"{str(i.relative_to(ROOT_DIR / 'dist'))}.sha512"], + cwd=ROOT_DIR / "dist", + check=True, + ) + + +if __name__ == "__main__": + for v in PACKAGES: + archive_package(v) + generate_signature() + generate_checksum() diff --git a/scripts/verify.py b/scripts/verify.py index 81d6a41ba413..bfca20a52035 100755 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -47,7 +47,7 @@ def check_java(): def build_core(): print("Start building opendal core") - subprocess.run(["cargo", "build", "--release"], check=True) + subprocess.run(["cargo", "build", "--release"], cwd="core", check=True) def build_java_binding(): @@ -62,7 +62,7 @@ def build_java_binding(): "-Dcargo-build.profile=release", ], check=True, - cwd="bindings/java" + cwd="bindings/java", ) From 8197abffdf6da7e2fb0fa4cc51306d147632a3a6 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sun, 18 Feb 2024 19:09:39 +0800 Subject: [PATCH 3/9] Refactor Signed-off-by: Xuanwo --- scripts/release.py | 27 ++++++------- scripts/verify.py | 53 +++++++++++++++---------- website/community/committers/release.md | 32 +++++++++------ 3 files changed, 64 insertions(+), 48 deletions(-) diff --git a/scripts/release.py b/scripts/release.py index df02ea41b088..e549d9a5025d 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -22,26 +22,24 @@ ROOT_DIR = Path(__file__).parent.parent -# If this package depends on `core`, we need to update it's core version in `Cargo.toml` file. -def update_package(path): - # skip core package. - if path == "core": - return - if get_package_dependence(path) != "core": - return - - core_version = get_package_version("core") - def archive_package(path): print(f"Archive package {path} started") + core_version = get_package_version("core") version = get_package_version(path) name = f"apache-opendal-{str(path).replace('/', '-')}-{version}-src" - ls_command = ["git", "ls-files", "."] + # `git ls-files` handles duplicated path correctly, so we don't need to worry about it + ls_command = [ + "git", + "ls-files", + "core", + f"{path}", + f"{get_package_dependence(path)}", + ] ls_result = subprocess.run( - ls_command, cwd=ROOT_DIR / path, capture_output=True, check=True, text=True + ls_command, cwd=ROOT_DIR, capture_output=True, check=True, text=True ) tar_command = [ @@ -49,19 +47,20 @@ def archive_package(path): "-zcf", f"{ROOT_DIR}/dist/{name}.tar.gz", "--transform", - f"s,^,{name}/,", + f"s,^,apache-opendal-{core_version}-src/,", "-T", "-", ] subprocess.run( tar_command, - cwd=ROOT_DIR / path, + cwd=ROOT_DIR, input=ls_result.stdout.encode("utf-8"), check=True, ) print(f"Archive package {path} to dist/{name}.tar.gz") + def generate_signature(): for i in Path(ROOT_DIR / "dist").glob("*.tar.gz"): print(f"Generate signature for {i}") diff --git a/scripts/verify.py b/scripts/verify.py index bfca20a52035..f778bc4ce475 100755 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -20,8 +20,16 @@ import subprocess import sys import os +from pathlib import Path -BASE_DIR = os.getcwd() +BASE_DIR = Path(os.getcwd()) + + +def extract_packages(): + print("Start extracting packages") + + for file in BASE_DIR.glob("*.tar.gz"): + subprocess.run(["tar", "-xzf", file], check=True) def check_rust(): @@ -44,13 +52,13 @@ def check_java(): raise Exception("Check java met unexpected error", e) -def build_core(): +def build_core(dir): print("Start building opendal core") - subprocess.run(["cargo", "build", "--release"], cwd="core", check=True) + subprocess.run(["cargo", "build", "--release"], cwd=dir/"core", check=True) -def build_java_binding(): +def build_java_binding(dir): print("Start building opendal java binding") subprocess.run( @@ -62,25 +70,26 @@ def build_java_binding(): "-Dcargo-build.profile=release", ], check=True, - cwd="bindings/java", + cwd=dir/"bindings/java", ) -def main(): - if not check_rust(): - print( - "Cargo is not found, please check if rust development has been setup correctly" - ) - print("Visit https://www.rust-lang.org/tools/install for more information") - sys.exit(1) - - build_core() - - if check_java(): - build_java_binding() - else: - print("Java is not found, skipped building java binding") - - if __name__ == "__main__": - main() + extract_packages() + + for dir in BASE_DIR.glob("apache-opendal-*-src/"): + if check_rust(): + build_core(dir) + else: + print( + "Cargo is not found, please check if rust development has been setup correctly" + ) + print("Visit https://www.rust-lang.org/tools/install for more information") + sys.exit(1) + + build_core(dir) + + if check_java(): + build_java_binding(dir) + else: + print("Java is not found, skipped building java binding") diff --git a/website/community/committers/release.md b/website/community/committers/release.md index 3a7d83b364b7..f3d6a7518ea0 100644 --- a/website/community/committers/release.md +++ b/website/community/committers/release.md @@ -188,14 +188,11 @@ Additionally, we should also drop the staging Maven artifacts on https://reposit After GitHub Release has been created, we can start to create ASF Release. - Checkout to released tag. (e.g. `git checkout v0.36.0-rc.1`, tag is created in the previous step) -- Use the release script to create a new release: `OPENDAL_VERSION= OPENDAL_VERSION_RC= ./scripts/release.sh`(e.g. `OPENDAL_VERSION=0.36.0 OPENDAL_VERSION_RC=rc.1 ./scripts/release.sh`) - - This script will do the following things: - - Create a new branch named by `release-${release_version}` from the tag - - Generate the release candidate artifacts under `dist`, including: - - `apache-opendal-${release_version}-src.tar.gz` - - `apache-opendal-${release_version}-src.tar.gz.asc` - - `apache-opendal-${release_version}-src.tar.gz.sha512` - - Check the header of the source code. This step needs docker to run. +- Use the release script to create a new release: `python ./scripts/release.py` + - This script will generate the release candidate artifacts under `dist`, including: + - `apache-opendal-{package}-{version}-src.tar.gz` + - `apache-opendal-{package}-{version}-src.tar.gz.asc` + - `apache-opendal-{package}-{version}-src.tar.gz.sha512` - Push the newly created branch to GitHub This script will create a new release under `dist`. @@ -203,11 +200,22 @@ This script will create a new release under `dist`. For example: ```shell -> tree dist dist -├── apache-opendal-0.36.0-src.tar.gz -├── apache-opendal-0.36.0-src.tar.gz.asc -└── apache-opendal-0.36.0-src.tar.gz.sha512 +├── apache-opendal-bindings-c-0.44.2+core.0.45.0-src.tar.gz +├── apache-opendal-bindings-c-0.44.2+core.0.45.0-src.tar.gz.asc +├── apache-opendal-bindings-c-0.44.2+core.0.45.0-src.tar.gz.sha512 +... +├── apache-opendal-core-0.45.0-src.tar.gz +├── apache-opendal-core-0.45.0-src.tar.gz.asc +├── apache-opendal-core-0.45.0-src.tar.gz.sha512 +├── apache-opendal-integrations-dav-server-0.0.0+core.0.45.0-src.tar.gz +├── apache-opendal-integrations-dav-server-0.0.0+core.0.45.0-src.tar.gz.asc +├── apache-opendal-integrations-dav-server-0.0.0+core.0.45.0-src.tar.gz.sha512 +├── apache-opendal-integrations-object_store-0.42.0+core.0.45.0-src.tar.gz +├── apache-opendal-integrations-object_store-0.42.0+core.0.45.0-src.tar.gz.asc +└── apache-opendal-integrations-object_store-0.42.0+core.0.45.0-src.tar.gz.sha512 + +1 directory, 60 files ``` ### Upload artifacts to the SVN dist repo From 5edb465cb87be68ac8f33b8c495fd6078761acc2 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sun, 18 Feb 2024 19:11:00 +0800 Subject: [PATCH 4/9] Format cargo Signed-off-by: Xuanwo --- bin/oay/Cargo.toml | 2 +- bin/ofs/Cargo.toml | 2 +- bin/oli/Cargo.toml | 2 +- bindings/java/Cargo.toml | 2 +- bindings/python/Cargo.toml | 2 +- bindings/python/pyproject.toml | 2 +- integrations/dav-server/Cargo.toml | 2 +- integrations/object_store/Cargo.toml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/bin/oay/Cargo.toml b/bin/oay/Cargo.toml index 1668ba9eebae..276bd247da0a 100644 --- a/bin/oay/Cargo.toml +++ b/bin/oay/Cargo.toml @@ -51,7 +51,7 @@ dav-server-opendalfs = { path = "../../integrations/dav-server", optional = true dirs = "5.0.1" futures = "0.3" futures-util = { version = "0.3.29", optional = true } -opendal = { path="../../core" } +opendal = { version = "0.45.0", path = "../../core" } quick-xml = { version = "0.31", features = ["serialize", "overlapped-lists"] } serde = { version = "1", features = ["derive"] } tokio = { version = "1.34", features = [ diff --git a/bin/ofs/Cargo.toml b/bin/ofs/Cargo.toml index f483d498a56a..83d1b62e1a49 100644 --- a/bin/ofs/Cargo.toml +++ b/bin/ofs/Cargo.toml @@ -39,7 +39,7 @@ futures-util = "0.3.30" libc = "0.2.151" log = "0.4.20" nix = { version = "0.27.1", features = ["user"] } -opendal = {path="../../core"} +opendal = { version = "0.45.0", path = "../../core" } tokio = { version = "1.34", features = [ "fs", "macros", diff --git a/bin/oli/Cargo.toml b/bin/oli/Cargo.toml index 90f09095203e..54934b77287f 100644 --- a/bin/oli/Cargo.toml +++ b/bin/oli/Cargo.toml @@ -60,7 +60,7 @@ dirs = "5.0.1" env_logger = "0.10" futures = "0.3" log = "0.4" -opendal = {path="../../core"} +opendal = { version = "0.45.0", path = "../../core" } serde = { version = "1", features = ["derive"] } tokio = { version = "1.34", features = [ "fs", diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index 688a43516c68..0d40d9ac6925 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -162,8 +162,8 @@ tokio = { version = "1.28.1", features = ["full"] } # This is not optimal. See also the Cargo issue: # https://github.com/rust-lang/cargo/issues/1197#issuecomment-1641086954 [target.'cfg(unix)'.dependencies.opendal] -path = "../../core" features = [ # Depend on "openssh" which depends on "tokio-pipe" that is unavailable on Windows. "services-sftp", ] +path = "../../core" diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index bb944f4aaa47..be91112310f2 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -160,10 +160,10 @@ pyo3-asyncio = { version = "0.20", features = ["tokio-runtime"] } tokio = "1" [target.'cfg(unix)'.dependencies.opendal] -path = "../../core" features = [ "layers-blocking", # Depend on "openssh" which depends on "tokio-pipe" that is unavailable on Windows. "services-sftp", ] +path = "../../core" diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 39a326c08179..209a118eafd6 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -41,8 +41,8 @@ benchmark = [ "boto3-stubs[essential]", ] docs = ["pdoc"] -test = ["pytest", "python-dotenv", "pytest-asyncio"] lint = ["ruff"] +test = ["pytest", "python-dotenv", "pytest-asyncio"] [project.urls] Documentation = "https://opendal.apache.org/docs/python/opendal.html" diff --git a/integrations/dav-server/Cargo.toml b/integrations/dav-server/Cargo.toml index 614e6e620bc1..535f20ec78ac 100644 --- a/integrations/dav-server/Cargo.toml +++ b/integrations/dav-server/Cargo.toml @@ -35,7 +35,7 @@ dav-server = { version = "0.5.8" } dirs = "5.0.0" futures = "0.3" futures-util = { version = "0.3.16" } -opendal = { path = "../../core"} +opendal = { version = "0.45.0", path = "../../core" } quick-xml = { version = "0.31", features = ["serialize", "overlapped-lists"] } serde = { version = "1", features = ["derive"] } tokio = { version = "1.27", features = [ diff --git a/integrations/object_store/Cargo.toml b/integrations/object_store/Cargo.toml index f25843d16c60..9d3facf18e58 100644 --- a/integrations/object_store/Cargo.toml +++ b/integrations/object_store/Cargo.toml @@ -32,7 +32,7 @@ async-trait = "0.1" bytes = "1" futures = "0.3" object_store = "0.7" -opendal = {path = "../../core"} +opendal = { version = "0.45.0", path = "../../core" } tokio = "1" [dev-dependencies] From 66af29a235cbe9878f5639db5a286218cc174051 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sun, 18 Feb 2024 19:13:26 +0800 Subject: [PATCH 5/9] Fix typo Signed-off-by: Xuanwo --- scripts/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/constants.py b/scripts/constants.py index b952a379b7e4..261b20b506e7 100755 --- a/scripts/constants.py +++ b/scripts/constants.py @@ -44,7 +44,7 @@ def list_packages(): } -# fetch the package dependence, return `core` if not listed in `PACAKGE_DEPENDENCIES`. +# fetch the package dependence, return `core` if not listed in `PACKAGE_DEPENDENCIES`. def get_package_dependence(package: Path) -> str: return PACKAGE_DEPENDENCIES.get(package, "core") From c95acafc0d8d0621081fea51dddbdddc30298702 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sun, 18 Feb 2024 19:16:59 +0800 Subject: [PATCH 6/9] Fix python Signed-off-by: Xuanwo --- .github/workflows/ci_check.yml | 7 ++++++- .github/workflows/ci_core.yml | 6 +++--- .github/workflows/release_core.yml | 2 +- .github/workflows/release_java.yml | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_check.yml b/.github/workflows/ci_check.yml index 7a939b082e30..10822542a107 100644 --- a/.github/workflows/ci_check.yml +++ b/.github/workflows/ci_check.yml @@ -63,7 +63,9 @@ jobs: FORCE_COLOR: 1 steps: - uses: actions/checkout@v4 - + - uses: actions/setup-python@v5 + with: + python-version: "3.11" - name: Rust Code Format run: ./scripts/workspace.py cargo fmt -- --check @@ -79,6 +81,9 @@ jobs: uses: ./.github/actions/setup with: need-deny: true + - uses: actions/setup-python@v5 + with: + python-version: "3.11" - name: Check dependencies run: python3 ./scripts/dependencies.py check diff --git a/.github/workflows/ci_core.yml b/.github/workflows/ci_core.yml index d276536f168f..1402f0c8a4e6 100644 --- a/.github/workflows/ci_core.yml +++ b/.github/workflows/ci_core.yml @@ -117,7 +117,7 @@ jobs: - name: Checkout python env uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.11" - name: Checkout java env uses: actions/setup-java@v4 with: @@ -148,7 +148,7 @@ jobs: - name: Checkout python env uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.11" - name: Checkout java env uses: actions/setup-java@v4 with: @@ -243,7 +243,7 @@ jobs: - name: Checkout python env uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.11" - name: Checkout java env uses: actions/setup-java@v4 with: diff --git a/.github/workflows/release_core.yml b/.github/workflows/release_core.yml index e12b6a38242e..e87c6c8144ef 100644 --- a/.github/workflows/release_core.yml +++ b/.github/workflows/release_core.yml @@ -37,7 +37,7 @@ jobs: - name: Checkout python env uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.11" - name: Checkout java env uses: actions/setup-java@v4 with: diff --git a/.github/workflows/release_java.yml b/.github/workflows/release_java.yml index 08b8b0f8703a..0baa1eac03d5 100644 --- a/.github/workflows/release_java.yml +++ b/.github/workflows/release_java.yml @@ -62,7 +62,7 @@ jobs: gpg-passphrase: MAVEN_GPG_PASSPHRASE - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - name: Install Protoc uses: arduino/setup-protoc@v2 with: From 13fc10b278eb362b0f65d7f7c49198675afc4054 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sun, 18 Feb 2024 19:20:23 +0800 Subject: [PATCH 7/9] Fix workspace Signed-off-by: Xuanwo --- scripts/workspace.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/workspace.py b/scripts/workspace.py index b490a45bcf8e..c9d6fee2dd30 100755 --- a/scripts/workspace.py +++ b/scripts/workspace.py @@ -18,6 +18,7 @@ import subprocess import sys +from pathlib import Path from constants import PACKAGES @@ -28,6 +29,14 @@ for directory in PACKAGES: print(f"Executing '{command}' in {directory}") + # Make cargo happy if `Cargo.toml` not exist + if ( + command.startswith("cargo") + and not (Path(directory) / "Cargo.toml").exists() + ): + print(f"Skip {directory} because `Cargo.toml` not exist") + continue + subprocess.run( command, shell=True, From a0731133feafb01da2c3a4f28ae9f55b6373b08b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sun, 18 Feb 2024 19:51:02 +0800 Subject: [PATCH 8/9] Version should inside Signed-off-by: Xuanwo --- bin/oay/Cargo.toml | 2 +- bin/ofs/Cargo.toml | 2 +- bin/oli/Cargo.toml | 2 +- bindings/c/Cargo.toml | 6 ++++-- bindings/cpp/Cargo.toml | 4 ++-- bindings/dotnet/Cargo.toml | 4 ++-- bindings/haskell/Cargo.toml | 2 +- bindings/java/Cargo.toml | 6 ++++-- bindings/lua/Cargo.toml | 4 ++-- bindings/nodejs/Cargo.toml | 6 ++++-- bindings/ocaml/Cargo.toml | 2 +- bindings/php/Cargo.toml | 4 ++-- bindings/python/Cargo.toml | 7 ++++--- bindings/ruby/Cargo.toml | 2 +- integrations/dav-server/Cargo.toml | 2 +- integrations/object_store/Cargo.toml | 2 +- scripts/constants.py | 11 ++++------- 17 files changed, 36 insertions(+), 32 deletions(-) diff --git a/bin/oay/Cargo.toml b/bin/oay/Cargo.toml index 276bd247da0a..f95b78e56069 100644 --- a/bin/oay/Cargo.toml +++ b/bin/oay/Cargo.toml @@ -27,7 +27,7 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.67" -version = "0.41.0" +version = "0.41.0+core.0.45.0" [features] default = ["frontends-webdav", "frontends-s3"] diff --git a/bin/ofs/Cargo.toml b/bin/ofs/Cargo.toml index 83d1b62e1a49..ad45a6617916 100644 --- a/bin/ofs/Cargo.toml +++ b/bin/ofs/Cargo.toml @@ -20,7 +20,7 @@ categories = ["filesystem"] description = "OpenDAL File System" keywords = ["storage", "data", "s3", "fs", "azblob"] name = "ofs" -version = "0.0.1" +version = "0.0.1+core.0.45.0" authors = ["Apache OpenDAL "] edition = "2021" diff --git a/bin/oli/Cargo.toml b/bin/oli/Cargo.toml index 54934b77287f..75cecc0bbdbe 100644 --- a/bin/oli/Cargo.toml +++ b/bin/oli/Cargo.toml @@ -27,7 +27,7 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.67" -version = "0.41.0" +version = "0.41.0+core.0.45.0" [features] # Enable services dashmap support diff --git a/bindings/c/Cargo.toml b/bindings/c/Cargo.toml index ab30cfd2b011..16c5027cc3cb 100644 --- a/bindings/c/Cargo.toml +++ b/bindings/c/Cargo.toml @@ -25,7 +25,7 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.67" -version = "0.44.2" +version = "0.44.2+core.0.45.0" [lib] crate-type = ["cdylib", "staticlib"] @@ -37,5 +37,7 @@ cbindgen = "0.26.0" [dependencies] bytes = "1.4.0" once_cell = "1.17.1" -opendal = { path = "../../core", features = ["layers-blocking"] } +opendal = { version = "0.45", path = "../../core", features = [ + "layers-blocking", +] } tokio = { version = "1.27", features = ["fs", "macros", "rt-multi-thread"] } diff --git a/bindings/cpp/Cargo.toml b/bindings/cpp/Cargo.toml index 81a98e79c710..6f9bc113dbdf 100644 --- a/bindings/cpp/Cargo.toml +++ b/bindings/cpp/Cargo.toml @@ -25,7 +25,7 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.67" -version = "0.44.2" +version = "0.44.2+core.0.45.0" [lib] crate-type = ["staticlib"] @@ -34,7 +34,7 @@ crate-type = ["staticlib"] anyhow = "1.0" chrono = "0.4" cxx = "1.0" -opendal = { path = "../../core" } +opendal = { version = "0.45.0", path = "../../core" } [build-dependencies] cxx-build = "1.0" diff --git a/bindings/dotnet/Cargo.toml b/bindings/dotnet/Cargo.toml index 70068031ed72..499b64c8090b 100644 --- a/bindings/dotnet/Cargo.toml +++ b/bindings/dotnet/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "opendal-dotnet" publish = false -version = "0.1.0" +version = "0.1.0+core.0.45.0" authors = ["Apache OpenDAL "] edition = "2021" @@ -32,4 +32,4 @@ crate-type = ["cdylib"] doc = false [dependencies] -opendal = { path = "../../core" } +opendal = { version = "0.45.0", path = "../../core" } diff --git a/bindings/haskell/Cargo.toml b/bindings/haskell/Cargo.toml index cc2cd2a2bed7..de1fe0477069 100644 --- a/bindings/haskell/Cargo.toml +++ b/bindings/haskell/Cargo.toml @@ -25,7 +25,7 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.67" -version = "0.44.2" +version = "0.44.2+core.0.44.2" [lib] crate-type = ["cdylib"] diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index 0d40d9ac6925..b9f9d3a3c503 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -25,7 +25,7 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.67" -version = "0.45.0" +version = "0.45.0+core.0.45.0" [lib] crate-type = ["cdylib"] @@ -156,7 +156,9 @@ anyhow = "1.0.71" jni = "0.21.1" num_cpus = "1.15.0" once_cell = "1.19.0" -opendal = { path = "../../core", features = ["layers-blocking"] } +opendal = { version = "0.45.0", path = "../../core", features = [ + "layers-blocking", +] } tokio = { version = "1.28.1", features = ["full"] } # This is not optimal. See also the Cargo issue: diff --git a/bindings/lua/Cargo.toml b/bindings/lua/Cargo.toml index f95b9165bf18..30df2543bd53 100644 --- a/bindings/lua/Cargo.toml +++ b/bindings/lua/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "opendal-lua" publish = false -version = "0.1.0" +version = "0.1.0+core.0.45.0" authors = ["Apache OpenDAL "] edition = "2021" @@ -39,4 +39,4 @@ mlua = { version = "0.9", features = [ "module", "macros", ], default-features = false, optional = true } -opendal = { path = "../../core" } +opendal = { version = "0.45.0", path = "../../core" } diff --git a/bindings/nodejs/Cargo.toml b/bindings/nodejs/Cargo.toml index fa62a54fa88e..3524ddfd3ff5 100644 --- a/bindings/nodejs/Cargo.toml +++ b/bindings/nodejs/Cargo.toml @@ -25,7 +25,7 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.67" -version = "0.45.0" +version = "0.45.0+core.0.45.0" [features] default = [ @@ -157,7 +157,9 @@ napi = { version = "2.11.3", default-features = false, features = [ "async", ] } napi-derive = "2.14.6" -opendal = { path = "../../core", features = ["layers-blocking"] } +opendal = { version = "0.45.0", path = "../../core", features = [ + "layers-blocking", +] } tokio = "1" [build-dependencies] diff --git a/bindings/ocaml/Cargo.toml b/bindings/ocaml/Cargo.toml index f7529ca575e4..f3505d2e41c0 100644 --- a/bindings/ocaml/Cargo.toml +++ b/bindings/ocaml/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "opendal-ocaml" publish = false -version = "0.0.0" +version = "0.0.0+core.0.44.2" authors = ["Apache OpenDAL "] edition = "2021" diff --git a/bindings/php/Cargo.toml b/bindings/php/Cargo.toml index 3ad2a50849ee..ab7bf15f65fa 100644 --- a/bindings/php/Cargo.toml +++ b/bindings/php/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "opendal-php" publish = false -version = "0.1.0" +version = "0.1.0+core.0.45.0" authors = ["Apache OpenDAL "] edition = "2021" @@ -32,4 +32,4 @@ crate-type = ["cdylib"] [dependencies] ext-php-rs = "0.11.2" -opendal = { path = "../../core" } +opendal = { version = "0.45.0", path = "../../core" } diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index be91112310f2..1bd2cf8280bd 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -25,7 +25,7 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.67" -version = "0.45.0" +version = "0.45.0+core.0.45.0" [features] default = [ @@ -154,7 +154,9 @@ doc = false [dependencies] futures = "0.3.28" -opendal = { path = "../../core", features = ["layers-blocking"] } +opendal = { version = "0.45.0", path = "../../core", features = [ + "layers-blocking", +] } pyo3 = "0.20.1" pyo3-asyncio = { version = "0.20", features = ["tokio-runtime"] } tokio = "1" @@ -166,4 +168,3 @@ features = [ # Depend on "openssh" which depends on "tokio-pipe" that is unavailable on Windows. "services-sftp", ] -path = "../../core" diff --git a/bindings/ruby/Cargo.toml b/bindings/ruby/Cargo.toml index d28d12039901..5572f3f86717 100644 --- a/bindings/ruby/Cargo.toml +++ b/bindings/ruby/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "opendal-ruby" publish = false -version = "0.1.0" +version = "0.1.0+core.0.43.0" authors = ["Apache OpenDAL "] edition = "2021" diff --git a/integrations/dav-server/Cargo.toml b/integrations/dav-server/Cargo.toml index 535f20ec78ac..1c8623c20adb 100644 --- a/integrations/dav-server/Cargo.toml +++ b/integrations/dav-server/Cargo.toml @@ -18,7 +18,7 @@ [package] description = "Use OpenDAL as a backend to access data in various service with WebDAV protocol" name = "dav-server-opendalfs" -version = "0.0.0" +version = "0.0.0+core.0.45.0" authors = ["Apache OpenDAL "] edition = "2021" diff --git a/integrations/object_store/Cargo.toml b/integrations/object_store/Cargo.toml index 9d3facf18e58..5ad892ee53d9 100644 --- a/integrations/object_store/Cargo.toml +++ b/integrations/object_store/Cargo.toml @@ -25,7 +25,7 @@ homepage = "https://opendal.apache.org/" license = "Apache-2.0" repository = "https://github.com/apache/opendal" rust-version = "1.67" -version = "0.42.0" +version = "0.42.0+core.0.45.0" [dependencies] async-trait = "0.1" diff --git a/scripts/constants.py b/scripts/constants.py index 261b20b506e7..11818229b273 100755 --- a/scripts/constants.py +++ b/scripts/constants.py @@ -62,29 +62,26 @@ def get_rust_package_version(path): # For examples: # core: `0.45.0` # packages depends on core: `0.1.0+core.0.45.0` -# packages depends on bindings/c: `0.1.0+bindings_c.0.1.0` def get_package_version(package): if package == "core": return get_rust_package_version("core") # NOTE: for now, all dependence package must be rust package. - dependence = get_package_dependence(package) - dependence_version = get_rust_package_version(dependence) + core_version = get_rust_package_version("core") cargo_toml = ROOT_DIR / package / "Cargo.toml" # cargo_toml exists, we can get the version from Cargo.toml. if cargo_toml.exists(): - package_version = get_rust_package_version(package) - return f"{package_version}+{dependence.replace('/', '_')}.{dependence_version}" + return get_rust_package_version(package) # cargo_toml not exists, we should handle case by case ideally. # # However, those packages are not mature enough, it's much easier for us to always return `0.0.0` instead. - return f"0.0.0+{dependence.replace('/', '_')}.{dependence_version}" + return f"0.0.0+core.{core_version}" if __name__ == "__main__": for v in PACKAGES: print( - f"{v}: depends={get_package_dependence(v)}, version={get_package_version(v)}" + f"{v}: version={get_package_version(v)}" ) From 2849ae87a3252de84ff54f55fe4fe1722c160246 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Sun, 18 Feb 2024 19:53:03 +0800 Subject: [PATCH 9/9] Fix build Signed-off-by: Xuanwo --- bindings/python/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 1bd2cf8280bd..a5921e755a3e 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -162,6 +162,8 @@ pyo3-asyncio = { version = "0.20", features = ["tokio-runtime"] } tokio = "1" [target.'cfg(unix)'.dependencies.opendal] +version = "0.45.0" +path = "../../core" features = [ "layers-blocking",