From e7538bd5712cb100b0f6180d1f63733088f27492 Mon Sep 17 00:00:00 2001 From: aceforeverd Date: Fri, 8 Dec 2023 10:33:54 +0800 Subject: [PATCH] build(proto): improve protobuf generation (#3636) 1. tracking all generated files (.cpp, .h, .java), instead of .cpp file only 2. auto clean all generated files on clean target, fix #891 for proto 3. simplify dependency in java modules, fix #1090 - proto generated files to openmldb-common only, instead of multiple projects - openmldb-native now depends on openmldb-common, no direct dependency to openmldb-common from other modules --- hybridse/src/proto/CMakeLists.txt | 117 ++++++++++-------------------- hybridse/src/sdk/CMakeLists.txt | 2 +- java/openmldb-batch/pom.xml | 6 -- java/openmldb-jdbc/pom.xml | 5 -- java/openmldb-native/pom.xml | 7 +- java/openmldb-synctool/pom.xml | 5 -- java/openmldb-taskmanager/pom.xml | 6 -- src/CMakeLists.txt | 58 +++++++++------ 8 files changed, 83 insertions(+), 123 deletions(-) diff --git a/hybridse/src/proto/CMakeLists.txt b/hybridse/src/proto/CMakeLists.txt index e8ea3264189..b3917c040e5 100644 --- a/hybridse/src/proto/CMakeLists.txt +++ b/hybridse/src/proto/CMakeLists.txt @@ -12,80 +12,43 @@ # See the License for the specific language governing permissions and # limitations under the License. -include_directories(${INCLUDE_DIRECTORIES} - ${PROJECT_SOURCE_DIR}/src - ${PROJECT_BINARY_DIR}/src) - -set(TYPE_PB_CC - ${CMAKE_CURRENT_BINARY_DIR}/fe_type.pb.cc) - -set(COMMON_PB_CC - ${CMAKE_CURRENT_BINARY_DIR}/fe_common.pb.cc - ) - -set(DBMS_PB_CC - ${CMAKE_CURRENT_BINARY_DIR}/dbms.pb.cc) - -set(TABLET_PB_CC - ${CMAKE_CURRENT_BINARY_DIR}/fe_tablet.pb.cc) - -set(BATCH_PB_CC - ${CMAKE_CURRENT_BINARY_DIR}/batch.pb.cc) - -set(PLAN_PB_CC - ${CMAKE_CURRENT_BINARY_DIR}/plan.pb.cc) - -add_custom_command(OUTPUT ${COMMON_PB_CC} - COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto - --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_common.proto - DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_common.proto - ) - -add_custom_command(OUTPUT ${TYPE_PB_CC} - COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_type.proto - DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_type.proto - ) - -add_custom_command(OUTPUT ${DBMS_PB_CC} - COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/dbms.proto - DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/dbms.proto - ) - -add_custom_command(OUTPUT ${TABLET_PB_CC} - COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_tablet.proto - DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_tablet.proto - ) - -add_custom_command(OUTPUT ${BATCH_PB_CC} - COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto - --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/batch.proto - DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/batch.proto - ) - -add_custom_command(OUTPUT ${PLAN_PB_CC} - COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/plan.proto - DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/plan.proto - ) - -set(HYBRIDSE_PROTO_SRC_FILES - ${TABLET_PB_CC} - ${DBMS_PB_CC} - ${COMMON_PB_CC} - ${TYPE_PB_CC} - ${BATCH_PB_CC} - ${PLAN_PB_CC}) - -ADD_CUSTOM_TARGET(run_gen_proto DEPENDS ${HYBRIDSE_PROTO_SRC_FILES}) - -add_library(hybridse_proto OBJECT ${HYBRIDSE_PROTO_SRC_FILES}) -add_dependencies(hybridse_proto run_gen_proto) - -if (SQL_JAVASDK_ENABLE) - add_custom_target(hybridse_proto_java ALL - COMMAND ${Protobuf_PROTOC_EXECUTABLE} - -I ${PROJECT_SOURCE_DIR}/src/proto - --java_out=${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java - ${PROJECT_SOURCE_DIR}/src/proto/*.proto - COMMENT "Generating protobuf java class into ${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java" - ) -endif() +include_directories(${INCLUDE_DIRECTORIES} ${PROJECT_SOURCE_DIR}/src + ${PROJECT_BINARY_DIR}/src) + +set(PROTO_CPP_FILES "") +set(PROTO_FILES "") +function(compile_proto proto_name java_file_suffix_path) + add_custom_command( + OUTPUT + ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc + ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.h + ${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java/com/_4paradigm/hybridse/${java_file_suffix_path} + COMMAND ${Protobuf_PROTOC_EXECUTABLE} + -I ${PROJECT_SOURCE_DIR}/src/proto + --cpp_out=${CMAKE_CURRENT_BINARY_DIR} + --java_out=${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java + ${PROJECT_SOURCE_DIR}/src/proto/${proto_name}.proto + DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/${proto_name}.proto) + list(APPEND PROTO_CPP_FILES ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc) + list(APPEND PROTO_FILES + ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc + ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.h + ${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java/com/_4paradigm/hybridse/${java_file_suffix_path} + ) + set(PROTO_CPP_FILES ${PROTO_CPP_FILES} PARENT_SCOPE) + set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE) +endfunction(compile_proto) + +compile_proto(fe_type type/TypeOuterClass.java) +compile_proto(fe_common common/Common.java) +compile_proto(dbms dbms/DBMS.java) +compile_proto(fe_tablet tablet/Tablet.java) +compile_proto(batch batch/Batch.java) +compile_proto(plan batch/Plan.java) + +add_library(hybridse_proto OBJECT ${PROTO_CPP_FILES}) +set_property( + TARGET hybridse_proto + APPEND + PROPERTY ADDITIONAL_CLEAN_FILES ${PROTO_FILES} +) diff --git a/hybridse/src/sdk/CMakeLists.txt b/hybridse/src/sdk/CMakeLists.txt index 404e7a6f6f3..8af1970df7d 100644 --- a/hybridse/src/sdk/CMakeLists.txt +++ b/hybridse/src/sdk/CMakeLists.txt @@ -72,5 +72,5 @@ if(SQL_JAVASDK_ENABLE) COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_SOURCE_DIR}/java/hybridse-native/src/main/resources/ COMMAND ${CMAKE_COMMAND} -E copy $ ${CMAKE_SOURCE_DIR}/java/hybridse-native/src/main/resources/ COMMENT "copy hybridse native so") - add_dependencies(cp_hybridse_native_so hybridse_proto_java) + add_dependencies(cp_hybridse_native_so hybridse_proto) endif() diff --git a/java/openmldb-batch/pom.xml b/java/openmldb-batch/pom.xml index 505935cd833..8a00ca8db16 100644 --- a/java/openmldb-batch/pom.xml +++ b/java/openmldb-batch/pom.xml @@ -210,12 +210,6 @@ - - com.4paradigm.openmldb - openmldb-common - ${project.parent.version} - - com.4paradigm.openmldb openmldb-spark-connector diff --git a/java/openmldb-jdbc/pom.xml b/java/openmldb-jdbc/pom.xml index f42bac7f354..32125a53596 100644 --- a/java/openmldb-jdbc/pom.xml +++ b/java/openmldb-jdbc/pom.xml @@ -38,11 +38,6 @@ openmldb-native ${variant.native.version} - - com.4paradigm.openmldb - openmldb-common - ${project.parent.version} - diff --git a/java/openmldb-native/pom.xml b/java/openmldb-native/pom.xml index 52ffaadaa31..e399c470ce5 100644 --- a/java/openmldb-native/pom.xml +++ b/java/openmldb-native/pom.xml @@ -21,7 +21,12 @@ - + + com.4paradigm.openmldb + openmldb-common + ${project.parent.version} + + org.slf4j slf4j-api 1.7.21 diff --git a/java/openmldb-synctool/pom.xml b/java/openmldb-synctool/pom.xml index ee815531406..6547b3e5475 100644 --- a/java/openmldb-synctool/pom.xml +++ b/java/openmldb-synctool/pom.xml @@ -28,11 +28,6 @@ test - - com.4paradigm.openmldb - openmldb-common - ${project.parent.version} - com.4paradigm.openmldb openmldb-jdbc diff --git a/java/openmldb-taskmanager/pom.xml b/java/openmldb-taskmanager/pom.xml index e2a14daa33c..ff207fc7741 100644 --- a/java/openmldb-taskmanager/pom.xml +++ b/java/openmldb-taskmanager/pom.xml @@ -159,12 +159,6 @@ ${project.parent.version} - - com.4paradigm.openmldb - openmldb-common - ${project.parent.version} - - com.4paradigm.openmldb openmldb-batchjob diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b2edc2d417f..aa341bea32d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,19 +23,26 @@ add_subdirectory(statistics) file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/src/proto) set(PROTO_FILES "") -function(compile_proto proto_name project_dir) - add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc - COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto - --cpp_out=${CMAKE_BINARY_DIR}/src/proto - --java_out=${project_dir}/java/openmldb-native/src/main/java - --java_out=${project_dir}/java/openmldb-import/src/main/java - --java_out=${project_dir}/java/openmldb-taskmanager/src/main/java - --java_out=${project_dir}/java/openmldb-common/src/main/java - --java_out=${project_dir}/java/openmldb-synctool/src/main/java - ${project_dir}/src/proto/${proto_name}.proto - DEPENDS ${project_dir}/src/proto/${proto_name}.proto) - list(APPEND PROTO_FILES ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc) - set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE) +set(PROTO_CPP_FILES "") +function(compile_proto proto_name java_out_file_name) + # will rerun if DEPENDS changed or OUTPUT not exists + add_custom_command( + OUTPUT ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc + ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.h + ${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java/com/_4paradigm/openmldb/proto/${java_out_file_name}.java + COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto + --cpp_out=${CMAKE_BINARY_DIR}/src/proto + --java_out=${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java + ${CMAKE_SOURCE_DIR}/src/proto/${proto_name}.proto + DEPENDS ${CMAKE_SOURCE_DIR}/src/proto/${proto_name}.proto) + list(APPEND PROTO_CPP_FILES ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc) + list(APPEND PROTO_FILES + ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc + ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.h + ${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java/com/_4paradigm/openmldb/proto/${java_out_file_name}.java + ) + set(PROTO_CPP_FILES ${PROTO_CPP_FILES} PARENT_SCOPE) + set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE) endfunction(compile_proto) function(compile_lib LIB_NAME DIR DEPEND_FILE_LIST) @@ -98,16 +105,23 @@ function(compile_test DIR) set(test_list ${test_list} PARENT_SCOPE) endfunction(compile_test) -compile_proto(type ${PROJECT_SOURCE_DIR}) -compile_proto(common ${PROJECT_SOURCE_DIR}) -compile_proto(tablet ${PROJECT_SOURCE_DIR}) -compile_proto(name_server ${PROJECT_SOURCE_DIR}) -compile_proto(sql_procedure ${PROJECT_SOURCE_DIR}) -compile_proto(api_server ${PROJECT_SOURCE_DIR}) -compile_proto(taskmanager ${PROJECT_SOURCE_DIR}) -compile_proto(data_sync ${PROJECT_SOURCE_DIR}) +compile_proto(type Type) +compile_proto(common Common) +compile_proto(tablet Tablet) +compile_proto(name_server NS) +compile_proto(sql_procedure SQLProcedure) +compile_proto(api_server ApiServer) +compile_proto(taskmanager TaskManager) +compile_proto(data_sync DataSync) + +add_library(openmldb_proto STATIC ${PROTO_CPP_FILES}) +# clean generated proto file, including '{cmake_source}/java/openmldb-common' +set_property( + TARGET openmldb_proto + APPEND + PROPERTY ADDITIONAL_CLEAN_FILES ${PROTO_FILES} +) -add_library(openmldb_proto STATIC ${PROTO_FILES}) add_library(openmldb_flags STATIC flags.cc) compile_lib(openmldb_codec codec "")