[IO][Input] Support ORCInputFormat and add Cube application (#261)

husky-team · Mar 22, 2017 · c641cd6 · c641cd6
1 parent 069e8c1
commit c641cd6
Show file tree

Hide file tree

Showing 17 changed files with 1,425 additions and 0 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -72,6 +72,13 @@ if(THRIFT_FOUND)
     list(APPEND HUSKY_EXTERNAL_DEFINITION ${THRIFT_DEFINITION})
 endif(THRIFT_FOUND)
 
+# ORC
+if(ORC_FOUND)
+    list(APPEND HUSKY_EXTERNAL_INCLUDE ${ORC_INCLUDE_DIR})
+    list(APPEND HUSKY_EXTERNAL_LIB ${ORC_LIBRARY})
+    list(APPEND HUSKY_EXTERNAL_DEFINITION ${ORC_DEFINITION})
+endif(ORC_FOUND)
+
 if(WIN32)
     list(APPEND HUSKY_EXTERNAL_LIB wsock32 ws2_32)
 endif()

diff --git a/cmake/dep.cmake b/cmake/dep.cmake
@@ -135,3 +135,36 @@ if(WITHOUT_THRIFT)
     unset(THRIFT_FOUND)
     message(STATUS "Not using Thrift due to WITHOUT_THRIFT option")
 endif(WITHOUT_THRIFT)
+
+### ORC ###
+
+#NAMES liblz4.a liborc.a libprotobuf.a libsnappy.a libz.a 
+#NAMES ColumnPrinter.hh Int128.hh MemoryPool.hh orc-config.hh OrcFile.hh Reader.hh Type.hh  Vector.hh
+find_path(ORC_INCLUDE_DIR NAMES orc/OrcFile.hh)
+find_library(ORC_L0 NAMES protobuf NO_CMAKE_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH NO_SYSTEM_ENVIRONMENT_PATH)
+find_library(ORC_L1 NAMES z NO_CMAKE_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH NO_SYSTEM_ENVIRONMENT_PATH)
+find_library(ORC_L2 NAMES lz4 NO_CMAKE_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH NO_SYSTEM_ENVIRONMENT_PATH)
+find_library(ORC_L3 NAMES snappy NO_CMAKE_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH NO_SYSTEM_ENVIRONMENT_PATH)
+find_library(ORC_L4 NAMES orc)
+
+if (ORC_INCLUDE_DIR AND ORC_L1 AND ORC_L0 AND ORC_L2 AND ORC_L3 AND ORC_L4)
+    set(ORC_FOUND true)
+endif (ORC_INCLUDE_DIR AND ORC_L1 AND ORC_L0 AND ORC_L2 AND ORC_L3 AND ORC_L4)
+if (ORC_FOUND)
+    set(ORC_DEFINITION "-DWITH_ORC")
+    # The order is important for dependencies.
+    set(ORC_LIBRARY ${ORC_L4} ${ORC_L3} ${ORC_L2} ${ORC_L1} ${ORC_L0})
+    message (STATUS "Found ORC:")
+    message (STATUS "  (Headers)       ${ORC_INCLUDE_DIR}")
+    message (STATUS "  (Library)       ${ORC_L0}")
+    message (STATUS "  (Library)       ${ORC_L1}")
+    message (STATUS "  (Library)       ${ORC_L2}")
+    message (STATUS "  (Library)       ${ORC_L3}")
+    message (STATUS "  (Library)       ${ORC_L4}")
+else(ORC_FOUND)
+    message (STATUS "Could NOT find ORC")
+endif(ORC_FOUND)
+if(WITHOUT_ORC)
+    unset(ORC_FOUND)
+    message(STATUS "Not using ORC due to WITHOUT_ORC option")
+endif(WITHOUT_ORC)
diff --git a/core/constants.hpp b/core/constants.hpp
@@ -73,6 +73,7 @@ const uint32_t TYPE_KAFKA_END_REQ = 0xfa091344;
 const uint32_t TYPE_MONGODB_REQ = 0xfa091388;
 const uint32_t TYPE_MONGODB_END_REQ = 0xfa091389;
 const uint32_t TYPE_LOCAL_BLK_REQ = 0xfa0e12a2;
+const uint32_t TYPE_ORC_BLK_REQ = 0xfa2e32a1;
 const uint32_t TYPE_STOP_ASYNC_REQ = 0xf89d74b4;
 const uint32_t TYPE_STOP_ASYNC_YES = 0x09b8ab2b;
 const uint32_t TYPE_STOP_ASYNC_NO = 0x192a241a;

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
@@ -61,6 +61,12 @@ target_link_libraries(WordCountFlume ${husky})
 target_link_libraries(WordCountFlume ${HUSKY_EXTERNAL_LIB})
 husky_default_properties(WordCountFlume)
 
+# WordCountORC
+add_executable(WordCountORC wc_mr_orc.cpp)
+target_link_libraries(WordCountORC ${husky})
+target_link_libraries(WordCountORC ${HUSKY_EXTERNAL_LIB})
+husky_default_properties(WordCountORC)
+
 # PI
 add_executable(PI pi.cpp)
 target_link_libraries(PI ${husky})
@@ -132,3 +138,9 @@ add_executable(NN_MNIST nn_mnist.cpp)
 target_link_libraries(NN_MNIST ${husky})
 target_link_libraries(NN_MNIST ${HUSKY_EXTERNAL_LIB})
 husky_default_properties(NN_MNIST)
+
+# Cube
+add_executable(Cube cube.cpp)
+target_link_libraries(Cube ${husky})
+target_link_libraries(Cube ${HUSKY_EXTERNAL_LIB})
+husky_default_properties(Cube)