HAWQ-1799. include libhdfs3 in building path

.github/scripts/toolchain.sh

@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export HAWQ_TOOLCHAIN_PATH=$(cd "$( dirname "${BASH_SOURCE[0]-$0}" )" && pwd)/download
+
+# GitHub Release for third-party package
+# https://github.com/apache/hawq/releases/tag/thirdparty
+REPO=https://github.com/apache/hawq/releases/download/thirdparty/
+
+# ISA and OS checking
+arch=$(uname -m)
+case $arch in
+  x86_64) ;;
+  *) echo "Not supported on ISA $arch"; return 1; ;;
+esac
+os=$(uname -s)
+case $os in
+  Darwin) ;;
+  Linux)  ;;
+  *) echo "Not supported on OS $os"; return 1; ;;
+esac
+
+
+###
+### macOS
+###
+if [[ $(uname -s) == Darwin ]]; then
+  if [ ! -d $HAWQ_TOOLCHAIN_PATH/dependency-Darwin ]; then
+    (cd $HAWQ_TOOLCHAIN_PATH && curl -OL $REPO/dependency-Darwin.tar.xz)
+    (cd $HAWQ_TOOLCHAIN_PATH && tar xJf dependency-Darwin.tar.xz -C $HAWQ_TOOLCHAIN_PATH)
+    ln -snf dependency-Darwin $HAWQ_TOOLCHAIN_PATH/dependency
+
+    for file in $(find $HAWQ_TOOLCHAIN_PATH/dependency/package/bin -name '*' -type f) $(find $HAWQ_TOOLCHAIN_PATH/dependency/package/lib -name '*.dylib' -type f); do
+      if [[ $(file $file | grep Mach-O) ]]; then
+        install_name_tool -add_rpath $HAWQ_TOOLCHAIN_PATH/dependency/package/lib $file 2>/dev/null || true
+      fi
+    done
+    install_name_tool -add_rpath $HAWQ_TOOLCHAIN_PATH/dependency/package/lib/perl5/5.28.0/darwin-thread-multi-2level/CORE/ $HAWQ_TOOLCHAIN_PATH/dependency/package/bin/perl
+  fi
+
+  export MAKEFLAGS=-j$(sysctl -n hw.ncpu)
+  export CFLAGS="$CFLAGS -w"
+  export LDFLAGS="$LDFLAGS -Wl,-rpath,$HAWQ_TOOLCHAIN_PATH/dependency/package/lib"
+fi
+
+
+
+###
+### Linux
+###
+if [[ $(uname -s) == Linux ]]; then
+  if [ ! -d $HAWQ_TOOLCHAIN_PATH/gcc ]; then
+    (cd $HAWQ_TOOLCHAIN_PATH && curl -OL $REPO/gcc-7.4.0-x86_64-linux-sles11.4.tar.xz)
+    (cd $HAWQ_TOOLCHAIN_PATH && tar xJf gcc-7.4.0-x86_64-linux-sles11.4.tar.xz -C $HAWQ_TOOLCHAIN_PATH)
+    ln -snf gcc-7.4.0-x86_64-linux-sles11.4 $HAWQ_TOOLCHAIN_PATH/gcc
+  fi
+  if [ ! -d $HAWQ_TOOLCHAIN_PATH/cmake ]; then
+    (cd $HAWQ_TOOLCHAIN_PATH && curl -OL $REPO/cmake-3.12.4-Linux-x86_64.tar.gz)
+    (cd $HAWQ_TOOLCHAIN_PATH && tar xzf cmake-3.12.4-Linux-x86_64.tar.gz -C $HAWQ_TOOLCHAIN_PATH)
+    ln -snf cmake-3.12.4-Linux-x86_64 $HAWQ_TOOLCHAIN_PATH/cmake
+  fi
+  if [ ! -d $HAWQ_TOOLCHAIN_PATH/dependency-gcc-x86_64-Linux/ ]; then
+    (cd $HAWQ_TOOLCHAIN_PATH && curl -OL $REPO/dependency-gcc-x86_64-Linux.tar.gz)
+    (cd $HAWQ_TOOLCHAIN_PATH && tar xzf dependency-gcc-x86_64-Linux.tar.gz -C $HAWQ_TOOLCHAIN_PATH)
+    ln -snf dependency-gcc-x86_64-Linux $HAWQ_TOOLCHAIN_PATH/dependency
+  fi
+
+  export PATH=$HAWQ_TOOLCHAIN_PATH/gcc/bin:$HAWQ_TOOLCHAIN_PATH/cmake/bin:$PATH
+  export LD_LIBRARY_PATH=$HAWQ_TOOLCHAIN_PATH/gcc/lib64/:$LD_LIBRARY_PATH
+
+  export CPATH=$HAWQ_TOOLCHAIN_PATH/gcc/include/c++/7.4.0/:$HAWQ_TOOLCHAIN_PATH/gcc/include/c++/7.4.0/x86_64-pc-linux-gnu/
+  export CPATH=$CPATH:/usr/include/x86_64-linux-gnu/
+  export LIBRARY_PATH=$HAWQ_TOOLCHAIN_PATH/gcc/lib64/:/usr/lib/x86_64-linux-gnu/
+
+  unset CPPFLAGS
+  export CFLAGS='-std=gnu11 -fno-use-linker-plugin'
+  export CXXFLAGS='-fpermissive -fno-use-linker-plugin'
+  unset LDFLAGS
+
+  export CC=gcc
+  export CXX=g++
+  export LD=ld
+
+  export MAKEFLAGS=-j$(nproc)
+fi
+
+
+
+###
+find . $HAWQ_TOOLCHAIN_PATH/../../../../ -name CMakeCache.txt -delete
+find . $HAWQ_TOOLCHAIN_PATH/../../../../ -name '*build_timestamp' -delete
+rm -rf $HAWQ_TOOLCHAIN_PATH/dependency/package/include/hdfs
+rm -rf $HAWQ_TOOLCHAIN_PATH/dependency/package/lib/libhdfs3*
+source $HAWQ_TOOLCHAIN_PATH/dependency/package/env.sh

GNUmakefile.in

@@ -10,6 +10,7 @@ include $(top_builddir)/src/Makefile.global
 
 all:
 #	$(MAKE) -C doc $@
+	$(MAKE) -C depends/libhdfs3 $@
 	$(MAKE) -C src $@
 	$(MAKE) -C config $@
 	$(MAKE) -C contrib $@
@@ -19,6 +20,7 @@ all:
 
 install:
 #	$(MAKE) -C doc $@
+	$(MAKE) -C depends/libhdfs3 $@
 	$(MAKE) -C src $@
 	$(MAKE) -C config $@
 	$(MAKE) -C contrib $@
@@ -56,6 +58,7 @@ feature-test-clean:
 # it's not built by default
 clean:
 #	$(MAKE) -C doc $@
+	$(MAKE) -C depends/libhdfs3 $@
 	$(MAKE) -C contrib $@
 	$(MAKE) -C src $@
 	$(MAKE) -C config $@
@@ -72,6 +75,7 @@ clean:
 # will be gone too soon.
 distclean maintainer-clean:
 #	-$(MAKE) -C doc $@
+	$(MAKE) -C depends/libhdfs3 $@
 	-$(MAKE) -C contrib $@
 	-$(MAKE) -C config $@
 	-$(MAKE) -C tools $@

config/c-compiler.m4

@@ -73,8 +73,8 @@ int does_int64_work()
     return 0;
   return 1;
 }
-main() {
-  exit(! does_int64_work());
+int main() {
+  return(! does_int64_work());
 }],
 [Ac_cachevar=yes],
 [Ac_cachevar=no],

config/c-library.m4

@@ -250,8 +250,8 @@ int does_int64_snprintf_work()
     return 0;			/* either multiply or snprintf is busted */
   return 1;
 }
-main() {
-  exit(! does_int64_snprintf_work());
+int main() {
+  return(! does_int64_snprintf_work());
 }],
 [pgac_cv_snprintf_long_long_int_format=$pgac_format; break],
 [],

configure

@@ -6711,7 +6711,7 @@ $as_echo "$as_me: WARNING: *** Include directory $dir does not exist." >&2;}
   fi
 done
 IFS=$ac_save_IFS
-
+INCLUDES="$INCLUDES -I/usr/local/hawq/include"
 
 
 #
@@ -6731,12 +6731,14 @@ $as_echo "$as_me: WARNING: *** Library directory $dir does not exist." >&2;}
   fi
 done
 IFS=$ac_save_IFS
-
+LIBDIRS="$LIBDIRS -L/usr/local/hawq/lib"
 
 #
 # Assignments
 #
 
+CFLAGS="$CFLAGS -w"
+CXXFLAGS="$CXXFLAGS -w"
 CPPFLAGS="$CPPFLAGS $INCLUDES"
 LDFLAGS="$LDFLAGS $LIBDIRS"
 
@@ -15246,8 +15248,8 @@ int does_int64_work()
     return 0;
   return 1;
 }
-main() {
-  exit(! does_int64_work());
+int main() {
+  return(! does_int64_work());
 }
 _ACEOF
 if ac_fn_c_try_run "$LINENO"; then :
@@ -15326,8 +15328,8 @@ int does_int64_work()
     return 0;
   return 1;
 }
-main() {
-  exit(! does_int64_work());
+int main() {
+  return(! does_int64_work());
 }
 _ACEOF
 if ac_fn_c_try_run "$LINENO"; then :
@@ -15419,8 +15421,8 @@ int does_int64_snprintf_work()
     return 0;			/* either multiply or snprintf is busted */
   return 1;
 }
-main() {
-  exit(! does_int64_snprintf_work());
+int main() {
+  return(! does_int64_snprintf_work());
 }
 _ACEOF
 if ac_fn_c_try_run "$LINENO"; then :
@@ -17094,6 +17096,8 @@ fi
 
 ac_config_files="$ac_config_files GNUmakefile src/VERSIONS.mk src/pl/pljava/src/java/Makefile.global"
 
+ac_config_files="$ac_config_files depends/libhdfs3/Makefile.global"
+
 ac_config_files="$ac_config_files src/Makefile.global"
 
 ac_config_files="$ac_config_files ranger-plugin/Makefile.global"
@@ -17823,6 +17827,7 @@ do
   case $ac_config_target in
     "src/backend/port/tas.s") CONFIG_LINKS="$CONFIG_LINKS src/backend/port/tas.s:src/backend/port/tas/${tas_file}" ;;
     "GNUmakefile") CONFIG_FILES="$CONFIG_FILES GNUmakefile" ;;
+    "depends/libhdfs3/Makefile.global") CONFIG_FILES="$CONFIG_FILES depends/libhdfs3/Makefile.global" ;;
     "src/VERSIONS.mk") CONFIG_FILES="$CONFIG_FILES src/VERSIONS.mk" ;;
     "src/pl/pljava/src/java/Makefile.global") CONFIG_FILES="$CONFIG_FILES src/pl/pljava/src/java/Makefile.global" ;;
     "src/Makefile.global") CONFIG_FILES="$CONFIG_FILES src/Makefile.global" ;;

configure.in

@@ -924,6 +924,7 @@ for dir in $with_includes $SRCH_INC; do
   fi
 done
 IFS=$ac_save_IFS
+INCLUDES="$INCLUDES -I/usr/local/hawq/include"
 AC_SUBST(INCLUDES)
 
 
@@ -943,12 +944,14 @@ for dir in $LIBRARY_DIRS $SRCH_LIB; do
   fi
 done
 IFS=$ac_save_IFS
-
+LIBDIRS="$LIBDIRS -L/usr/local/hawq/lib"
 
 #
 # Assignments
 #
 
+CFLAGS="$CFLAGS -w"
+CXXFLAGS="$CXXFLAGS -w"
 CPPFLAGS="$CPPFLAGS $INCLUDES"
 LDFLAGS="$LDFLAGS $LIBDIRS"
 
@@ -2205,6 +2208,7 @@ fi
 AC_SUBST(vpath_build)
 
 AC_CONFIG_FILES([GNUmakefile src/VERSIONS.mk src/pl/pljava/src/java/Makefile.global])
+AC_CONFIG_FILES([depends/libhdfs3/Makefile.global])
 AC_CONFIG_FILES([src/Makefile.global])
 AC_CONFIG_FILES([ranger-plugin/Makefile.global])
 

depends/libhdfs3/src/client/hdfs.h

@@ -726,6 +726,8 @@ Namenode * hdfsGetHANamenodesWithConfig(const char * conf, const char * nameserv
 void hdfsFreeNamenodeInformation(Namenode * namenodes, int size);
 
 typedef struct BlockLocation {
+    uint32_t rangeId;
+    uint32_t replicaGroupId;
     int corrupt;            // If the block is corrupt
     int numOfNodes;         // Number of Datanodes which keep the block
     char ** hosts;          // Datanode hostnames

src/Makefile

@@ -13,7 +13,7 @@ top_builddir = ..
 
 include Makefile.global
 
-all install installdirs uninstall distprep: install-cwrapper
+all install installdirs uninstall distprep: check-and-install-cwrapper
 	$(MAKE) -C port $@
 	$(MAKE) -C timezone $@
 	$(MAKE) -C backend $@
@@ -26,7 +26,8 @@ all install installdirs uninstall distprep: install-cwrapper
 	$(MAKE) -C test/regress $@
 	$(MAKE) -C tools/fsync $@
 
-install-cwrapper:
+check-and-install-cwrapper:
+ifeq ("$(wildcard $(DEPENDENCY_PATH)/include/cwrapper-installed)", "")
 	rm -rf $(DEPENDENCY_PATH)/include/executor
 	rm -rf $(DEPENDENCY_PATH)/include/scheduler
 	rm -rf $(DEPENDENCY_PATH)/include/magma
@@ -54,6 +55,8 @@ install-cwrapper:
 	$(INSTALL_DATA) include/cwrapper/orc-format-c.h '$(DEPENDENCY_PATH)/include/storage/cwrapper/'
 	$(INSTALL_DATA) include/cwrapper/text-format-c.h '$(DEPENDENCY_PATH)/include/storage/cwrapper/'
 	$(INSTALL_DATA) include/cwrapper/magma-format-c.h '$(DEPENDENCY_PATH)/include/storage/cwrapper/'
+	touch $(DEPENDENCY_PATH)/include/cwrapper-installed
+endif
 
 feature-test:
 	$(MAKE) -C test feature-test
@@ -86,7 +89,16 @@ uninstall: uninstall-local
 uninstall-local:
 	rm -f $(addprefix '$(DESTDIR)$(pgxsdir)/$(subdir)'/, Makefile.global Makefile.port Makefile.shlib nls-global.mk)
 
-clean:
+clear-cwrapper:
+	rm -rf $(DEPENDENCY_PATH)/include/executor
+	rm -rf $(DEPENDENCY_PATH)/include/scheduler
+	rm -rf $(DEPENDENCY_PATH)/include/magma
+	rm -rf $(DEPENDENCY_PATH)/include/dbcommon
+	rm -rf $(DEPENDENCY_PATH)/include/univplan
+	rm -rf $(DEPENDENCY_PATH)/include/storage
+	rm -f $(DEPENDENCY_PATH)/include/cwrapper-installed
+
+clean: clear-cwrapper
 	$(MAKE) -C port $@
 	$(MAKE) -C timezone $@
 	$(MAKE) -C backend $@
@@ -102,7 +114,7 @@ clean:
 	$(MAKE) -C test/feature $@
 	$(MAKE) -C tools/fsync $@
 
-distclean maintainer-clean:
+distclean maintainer-clean: clear-cwrapper
 	$(MAKE) -C port $@
 	$(MAKE) -C timezone $@
 	$(MAKE) -C backend $@

src/include/cwrapper/instrument.h

@@ -10,6 +10,10 @@
 
 #include <stdint.h>
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 typedef struct MyInstrumentation {
   struct MyInstrumentation *leftTree;
   struct MyInstrumentation *rightTree;
@@ -51,4 +55,8 @@ __attribute__((weak)) void MyInstrStopNode(MyInstrumentation *instr, uint64_t nT
 
 __attribute__((weak)) void MyInstrEndLoop(MyInstrumentation *instr) {}
 
+#ifdef __cplusplus
+}
+#endif
+
 #endif  // DBCOMMON_SRC_DBCOMMON_UTILS_INSTRUMENT_H_