build(proto): improve protobuf generation (#3636)

1. tracking all generated files (.cpp, .h, .java), instead of .cpp file only 2. auto clean all generated files on clean target, fix #891 for proto 3. simplify dependency in java modules, fix #1090 - proto generated files to openmldb-common only, instead of multiple projects - openmldb-native now depends on openmldb-common, no direct dependency to openmldb-common from other modules
4paradigm · Dec 8, 2023 · e7538bd · e7538bd
1 parent d4f6346
commit e7538bd
Show file tree

Hide file tree

Showing 8 changed files with 83 additions and 123 deletions.
diff --git a/hybridse/src/proto/CMakeLists.txt b/hybridse/src/proto/CMakeLists.txt
@@ -12,80 +12,43 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-include_directories(${INCLUDE_DIRECTORIES}
-        ${PROJECT_SOURCE_DIR}/src
-        ${PROJECT_BINARY_DIR}/src)
-
-set(TYPE_PB_CC
-        ${CMAKE_CURRENT_BINARY_DIR}/fe_type.pb.cc)
-
-set(COMMON_PB_CC
-        ${CMAKE_CURRENT_BINARY_DIR}/fe_common.pb.cc
-        )
-
-set(DBMS_PB_CC
-        ${CMAKE_CURRENT_BINARY_DIR}/dbms.pb.cc)
-
-set(TABLET_PB_CC
-    ${CMAKE_CURRENT_BINARY_DIR}/fe_tablet.pb.cc)
-
-set(BATCH_PB_CC
-    ${CMAKE_CURRENT_BINARY_DIR}/batch.pb.cc)
-
-set(PLAN_PB_CC
-    ${CMAKE_CURRENT_BINARY_DIR}/plan.pb.cc)
-
-add_custom_command(OUTPUT ${COMMON_PB_CC}
-        COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
-        --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_common.proto
-        DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_common.proto
-        )
-
-add_custom_command(OUTPUT ${TYPE_PB_CC}
-        COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_type.proto
-        DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_type.proto
-        )
-
-add_custom_command(OUTPUT ${DBMS_PB_CC}
-        COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/dbms.proto
-        DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/dbms.proto
-        )
-
-add_custom_command(OUTPUT ${TABLET_PB_CC}
-        COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_tablet.proto
-        DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_tablet.proto
-        )
-
-add_custom_command(OUTPUT ${BATCH_PB_CC}
-        COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
-        --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/batch.proto
-        DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/batch.proto
-        )
-
-add_custom_command(OUTPUT ${PLAN_PB_CC}
-        COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/plan.proto
-        DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/plan.proto
-        )
-
-set(HYBRIDSE_PROTO_SRC_FILES
-    ${TABLET_PB_CC}
-    ${DBMS_PB_CC}
-    ${COMMON_PB_CC}
-    ${TYPE_PB_CC}
-    ${BATCH_PB_CC}
-    ${PLAN_PB_CC})
-
-ADD_CUSTOM_TARGET(run_gen_proto DEPENDS ${HYBRIDSE_PROTO_SRC_FILES})
-
-add_library(hybridse_proto OBJECT ${HYBRIDSE_PROTO_SRC_FILES})
-add_dependencies(hybridse_proto run_gen_proto)
-
-if (SQL_JAVASDK_ENABLE)
-    add_custom_target(hybridse_proto_java ALL
-        COMMAND ${Protobuf_PROTOC_EXECUTABLE}
-        -I ${PROJECT_SOURCE_DIR}/src/proto
-        --java_out=${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java
-        ${PROJECT_SOURCE_DIR}/src/proto/*.proto
-        COMMENT "Generating protobuf java class into ${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java"
-        )
-endif()
+include_directories(${INCLUDE_DIRECTORIES} ${PROJECT_SOURCE_DIR}/src
+                    ${PROJECT_BINARY_DIR}/src)
+
+set(PROTO_CPP_FILES "")
+set(PROTO_FILES "")
+function(compile_proto proto_name java_file_suffix_path)
+  add_custom_command(
+    OUTPUT
+      ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc
+      ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.h
+      ${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java/com/_4paradigm/hybridse/${java_file_suffix_path}
+    COMMAND ${Protobuf_PROTOC_EXECUTABLE}
+      -I ${PROJECT_SOURCE_DIR}/src/proto
+      --cpp_out=${CMAKE_CURRENT_BINARY_DIR}
+      --java_out=${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java
+      ${PROJECT_SOURCE_DIR}/src/proto/${proto_name}.proto
+    DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/${proto_name}.proto)
+  list(APPEND PROTO_CPP_FILES ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc)
+  list(APPEND PROTO_FILES
+    ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc
+    ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.h
+    ${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java/com/_4paradigm/hybridse/${java_file_suffix_path}
+  )
+  set(PROTO_CPP_FILES ${PROTO_CPP_FILES} PARENT_SCOPE)
+  set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE)
+endfunction(compile_proto)
+
+compile_proto(fe_type type/TypeOuterClass.java)
+compile_proto(fe_common common/Common.java)
+compile_proto(dbms dbms/DBMS.java)
+compile_proto(fe_tablet tablet/Tablet.java)
+compile_proto(batch batch/Batch.java)
+compile_proto(plan batch/Plan.java)
+
+add_library(hybridse_proto OBJECT ${PROTO_CPP_FILES})
+set_property(
+  TARGET hybridse_proto
+  APPEND
+  PROPERTY ADDITIONAL_CLEAN_FILES ${PROTO_FILES}
+)
diff --git a/hybridse/src/sdk/CMakeLists.txt b/hybridse/src/sdk/CMakeLists.txt
@@ -72,5 +72,5 @@ if(SQL_JAVASDK_ENABLE)
     COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_SOURCE_DIR}/java/hybridse-native/src/main/resources/
     COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:hybridse_jsdk_core> ${CMAKE_SOURCE_DIR}/java/hybridse-native/src/main/resources/
     COMMENT "copy hybridse native so")
-  add_dependencies(cp_hybridse_native_so hybridse_proto_java)
+  add_dependencies(cp_hybridse_native_so hybridse_proto)
 endif()
diff --git a/java/openmldb-batch/pom.xml b/java/openmldb-batch/pom.xml
@@ -210,12 +210,6 @@
             </exclusions>
         </dependency>
 
-        <dependency>
-            <groupId>com.4paradigm.openmldb</groupId>
-            <artifactId>openmldb-common</artifactId>
-            <version>${project.parent.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>com.4paradigm.openmldb</groupId>
             <artifactId>openmldb-spark-connector</artifactId>

diff --git a/java/openmldb-jdbc/pom.xml b/java/openmldb-jdbc/pom.xml
@@ -38,11 +38,6 @@
             <artifactId>openmldb-native</artifactId>
             <version>${variant.native.version}</version>
         </dependency>
-        <dependency>
-            <groupId>com.4paradigm.openmldb</groupId>
-            <artifactId>openmldb-common</artifactId>
-            <version>${project.parent.version}</version>
-        </dependency>
 
         <!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
         <dependency>

diff --git a/java/openmldb-native/pom.xml b/java/openmldb-native/pom.xml
@@ -21,7 +21,12 @@
     </properties>
 
     <dependencies>
-       <dependency>
+        <dependency>
+            <groupId>com.4paradigm.openmldb</groupId>
+            <artifactId>openmldb-common</artifactId>
+            <version>${project.parent.version}</version>
+        </dependency>
+        <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
             <version>1.7.21</version>

diff --git a/java/openmldb-synctool/pom.xml b/java/openmldb-synctool/pom.xml
@@ -28,11 +28,6 @@
             <scope>test</scope>
         </dependency>
 
-        <dependency>
-            <groupId>com.4paradigm.openmldb</groupId>
-            <artifactId>openmldb-common</artifactId>
-            <version>${project.parent.version}</version>
-        </dependency>
         <dependency>
           <groupId>com.4paradigm.openmldb</groupId>
           <artifactId>openmldb-jdbc</artifactId>

diff --git a/java/openmldb-taskmanager/pom.xml b/java/openmldb-taskmanager/pom.xml
@@ -159,12 +159,6 @@
             <version>${project.parent.version}</version>
         </dependency>
 
-        <dependency>
-            <groupId>com.4paradigm.openmldb</groupId>
-            <artifactId>openmldb-common</artifactId>
-            <version>${project.parent.version}</version>
-        </dependency>
-
         <dependency>
             <groupId>com.4paradigm.openmldb</groupId>
             <artifactId>openmldb-batchjob</artifactId>

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -23,19 +23,26 @@ add_subdirectory(statistics)
 file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/src/proto)
 
 set(PROTO_FILES "")
-function(compile_proto proto_name project_dir)
-    add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc
-        COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
-        --cpp_out=${CMAKE_BINARY_DIR}/src/proto
-        --java_out=${project_dir}/java/openmldb-native/src/main/java
-        --java_out=${project_dir}/java/openmldb-import/src/main/java
-        --java_out=${project_dir}/java/openmldb-taskmanager/src/main/java
-        --java_out=${project_dir}/java/openmldb-common/src/main/java
-        --java_out=${project_dir}/java/openmldb-synctool/src/main/java
-        ${project_dir}/src/proto/${proto_name}.proto
-        DEPENDS ${project_dir}/src/proto/${proto_name}.proto)
-    list(APPEND PROTO_FILES ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc)
-    set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE)
+set(PROTO_CPP_FILES "")
+function(compile_proto proto_name java_out_file_name)
+  # will rerun if DEPENDS changed or OUTPUT not exists
+  add_custom_command(
+    OUTPUT ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc
+      ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.h
+      ${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java/com/_4paradigm/openmldb/proto/${java_out_file_name}.java
+    COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
+      --cpp_out=${CMAKE_BINARY_DIR}/src/proto
+      --java_out=${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java
+      ${CMAKE_SOURCE_DIR}/src/proto/${proto_name}.proto
+    DEPENDS ${CMAKE_SOURCE_DIR}/src/proto/${proto_name}.proto)
+  list(APPEND PROTO_CPP_FILES ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc)
+  list(APPEND PROTO_FILES
+    ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc
+    ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.h
+    ${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java/com/_4paradigm/openmldb/proto/${java_out_file_name}.java
+  )
+  set(PROTO_CPP_FILES ${PROTO_CPP_FILES} PARENT_SCOPE)
+  set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE)
 endfunction(compile_proto)
 
 function(compile_lib LIB_NAME DIR DEPEND_FILE_LIST)
@@ -98,16 +105,23 @@ function(compile_test DIR)
     set(test_list ${test_list} PARENT_SCOPE)
 endfunction(compile_test)
 
-compile_proto(type ${PROJECT_SOURCE_DIR})
-compile_proto(common ${PROJECT_SOURCE_DIR})
-compile_proto(tablet ${PROJECT_SOURCE_DIR})
-compile_proto(name_server ${PROJECT_SOURCE_DIR})
-compile_proto(sql_procedure ${PROJECT_SOURCE_DIR})
-compile_proto(api_server ${PROJECT_SOURCE_DIR})
-compile_proto(taskmanager ${PROJECT_SOURCE_DIR})
-compile_proto(data_sync ${PROJECT_SOURCE_DIR})
+compile_proto(type Type)
+compile_proto(common Common)
+compile_proto(tablet Tablet)
+compile_proto(name_server NS)
+compile_proto(sql_procedure SQLProcedure)
+compile_proto(api_server ApiServer)
+compile_proto(taskmanager TaskManager)
+compile_proto(data_sync DataSync)
+
+add_library(openmldb_proto STATIC ${PROTO_CPP_FILES})
+# clean generated proto file, including '{cmake_source}/java/openmldb-common'
+set_property(
+  TARGET openmldb_proto
+  APPEND
+  PROPERTY ADDITIONAL_CLEAN_FILES ${PROTO_FILES}
+)
 
-add_library(openmldb_proto STATIC ${PROTO_FILES})
 add_library(openmldb_flags STATIC flags.cc)
 
 compile_lib(openmldb_codec codec "")