ODP-2739: Use ambari-python-wrap instead of python

be/src/codegen/gen_ir_descriptions.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # This uses system python to avoid a dependency on impala-python,
 # because this runs during the build.
 #

bin/bootstrap_toolchain.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

bin/check-rat-report.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/collect_minidumps.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

bin/compare_branches.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

bin/create-test-configuration.sh

@@ -205,7 +205,7 @@ if [ $CREATE_RANGER_POLICY_DB -eq 1 ]; then
   createdb -U hiveuser "${RANGER_POLICY_DB}"
   pushd "${RANGER_HOME}"
   generate_config "${RANGER_TEST_CONF_DIR}/install.properties.template" install.properties
-  python ./db_setup.py
+  ambari-python-wrap ./db_setup.py
   popd
 fi
 

bin/diagnostics/collect_diagnostics.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/diagnostics/experimental/plan-graph.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/diagnostics/experimental/tpcds_run_comparator.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/gen-backend-test-script.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

bin/gen_build_version.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 # This uses system python to avoid a dependency on impala-python,
 # because this runs during the build.
 #

bin/generate_xml_config.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/impala-config.sh

@@ -70,7 +70,7 @@ fi
 # WARNING: If changing this value, also run these commands:
 # cd ${IMPALA_HOME}/java
 # mvn versions:set -DnewVersion=YOUR_NEW_VERSION
-export IMPALA_VERSION=4.4.0.3.2.3.3.2
+export IMPALA_VERSION=4.4.0.3.2.3.3-3
 
 # Whether to build on Apache Hive (or CDP Hive). Versions of some toolchain dependencies
 # (e.g. thrift) will also depend on this.
@@ -237,9 +237,39 @@ export IMPALA_THRIFT_PY_VERSION=0.16.0-p7
 unset IMPALA_THRIFT_PY_URL
 
 # Find system python versions for testing
-export IMPALA_SYSTEM_PYTHON2="${IMPALA_SYSTEM_PYTHON2_OVERRIDE-$(command -v python2)}"
-export IMPALA_SYSTEM_PYTHON3="${IMPALA_SYSTEM_PYTHON3_OVERRIDE-$(command -v python3)}"
+IMPALA_SYSTEM_PYTHON2_OVERRIDE="${IMPALA_SYSTEM_PYTHON2_OVERRIDE:-}"
+IMPALA_SYSTEM_PYTHON3_OVERRIDE="${IMPALA_SYSTEM_PYTHON3_OVERRIDE:-}"
 
+if command -v python2 >/dev/null || [ -n "$IMPALA_SYSTEM_PYTHON2_OVERRIDE" ]; then
+  export IMPALA_SYSTEM_PYTHON2="${IMPALA_SYSTEM_PYTHON2_OVERRIDE-$(command -v python2)}"
+fi
+
+if command -v ambari-python-wrap >/dev/null || [ -n "$IMPALA_SYSTEM_PYTHON3_OVERRIDE" ]; then
+    # Extract major python version from ambari-python-wrap
+    PYTHON_VER=$($(command -v ambari-python-wrap) --version 2>&1 | tr -s '[:blank:]' . | cut -d '.' -f2)
+
+    if printf "%s" "$PYTHON_VER" | grep "^[23]$" >/dev/null && [ "$PYTHON_VER" -eq 3 ] 2>/dev/null && [ -z "$IMPALA_SYSTEM_PYTHON3_OVERRIDE" ]; then
+        mkdir -p /tmp/acceldata-python/
+        \rm -f /tmp/acceldata-python/python3
+        # CMakeLists.txt expects the binary to match exactly python3, so we create a symlink
+        ln -sv "$(command -v ambari-python-wrap)" /tmp/acceldata-python/python3
+        export IMPALA_SYSTEM_PYTHON3="/tmp/acceldata-python/python3"
+    else 
+        export IMPALA_SYSTEM_PYTHON3="$IMPALA_SYSTEM_PYTHON3_OVERRIDE"
+    fi
+fi
+
+if [ -z "$IMPALA_SYSTEM_PYTHON2" ] && [ -z "$IMPALA_SYSTEM_PYTHON3" ]; then
+    printf "Python 2 was not found in PATH\n" >&2
+    printf "ambari-python-wrap not found in PATH.\n" >&2
+    printf "Neither IMPALA_SYSTEM_PYTHON2_OVERRIDE or IMPALA_SYSTEM_PYTHON3_OVERRIDE have been set\n" >&2
+    printf "At least one of these must be true to continue bulding impala\n" >&2
+    exit 1
+elif [ -z "$IMPALA_SYSTEM_PYTHON3" ]; then
+    printf "Skipping configuration for python 3\n"
+elif [ -z "$IMPALA_SYSTEM_PYTHON2" ]; then
+    printf "Skipping configuration for python 2\n"
+fi
 # Additional Python versions to use when building the impala-shell prebuilt tarball
 # via make_shell_tarball.sh. That tarball includes precompiled packages, so it can be
 # used without additional system dependencies needed for pip install.
@@ -256,19 +286,19 @@ fi
 export IMPALA_TOOLCHAIN_HOST
 
 export CDP_BUILD_NUMBER=49623641
-export ODP_BUILD_NUMBER=3.2.3.3-2
+export ODP_BUILD_NUMBER=3.2.3.3-3
 export CDP_MAVEN_REPOSITORY=\
 "https://${IMPALA_TOOLCHAIN_HOST}/build/cdp_components/${CDP_BUILD_NUMBER}/maven"
 export CDP_AVRO_JAVA_VERSION=1.11.3
-export CDP_HADOOP_VERSION=3.2.3.3.2.3.3-2
-export CDP_HBASE_VERSION=2.4.11.3.2.3.3-2
-export CDP_HIVE_VERSION=3.1.4.3.2.3.3-2
-export CDP_ICEBERG_VERSION=1.3.1.7.3.0.0-128
-export CDP_KNOX_VERSION=1.6.1.3.2.3.3-2
-export CDP_OZONE_VERSION=1.4.0.3.2.3.3-2
+export CDP_HADOOP_VERSION=3.2.3.3.2.3.3-3
+export CDP_HBASE_VERSION=2.4.11.3.2.3.3-3
+export CDP_HIVE_VERSION=3.1.4.3.2.3.3-3
+export CDP_ICEBERG_VERSION=1.4.3
+export CDP_KNOX_VERSION=1.6.1.3.2.3.3-3
+export CDP_OZONE_VERSION=1.4.0.3.2.3.3-3
 export CDP_PARQUET_VERSION=1.13.1
-export CDP_RANGER_VERSION=2.3.0.3.2.3.3-2
-export CDP_TEZ_VERSION=0.10.1.3.2.3.3-2
+export CDP_RANGER_VERSION=2.3.0.3.2.3.3-3
+export CDP_TEZ_VERSION=0.10.1.3.2.3.3-3
 
 # Ref: https://infra.apache.org/release-download-pages.html#closer
 : ${APACHE_MIRROR:="https://www.apache.org/dyn/closer.cgi"}

bin/impala-python-common.sh

@@ -23,10 +23,9 @@ set -euo pipefail
 setup_report_build_error
 
 . $IMPALA_HOME/bin/set-pythonpath.sh
-
-export LD_LIBRARY_PATH="$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
+export LD_LIBRARY_PATH="$(ambari-python-wrap "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
   --print-ld-library-path)"
 
 PY_DIR="$(dirname "$0")/../infra/python"
 PY_ENV_DIR="${PY_DIR}/env-gcc${IMPALA_GCC_VERSION}"
-python "$PY_DIR/bootstrap_virtualenv.py"
+ambari-python-wrap "$PY_DIR/bootstrap_virtualenv.py"

bin/impala-python3-common.sh

@@ -24,9 +24,9 @@ setup_report_build_error
 
 . $IMPALA_HOME/bin/set-pythonpath.sh
 
-export LD_LIBRARY_PATH="$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
+export LD_LIBRARY_PATH="$(ambari-python-wrap "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
   --print-ld-library-path)"
 
 PY_DIR="$(dirname "$0")/../infra/python"
 PY_ENV_DIR="${PY_DIR}/env-gcc${IMPALA_GCC_VERSION}-py3"
-python "$PY_DIR/bootstrap_virtualenv.py" --python3
+ambari-python-wrap "$PY_DIR/bootstrap_virtualenv.py" --python3

bin/impala-shell.sh

@@ -30,9 +30,8 @@ PYTHONPATH=${PYTHONPATH}:${IMPALA_HOME}/bin
 PYTHONPATH=${PYTHONPATH}:${SHELL_HOME}/gen-py
 
 THRIFT_PY_ROOT="${IMPALA_TOOLCHAIN_PACKAGES_HOME}/thrift-${IMPALA_THRIFT_PY_VERSION}"
-
 export LD_LIBRARY_PATH=":$(PYTHONPATH=${PYTHONPATH} \
-  python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
+  ambari-python-wrap "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
   --print-ld-library-path)"
 
 IMPALA_PY_DIR="$(dirname "$0")/../infra/python"
@@ -48,7 +47,7 @@ for PYTHON_LIB_DIR in ${THRIFT_PY_ROOT}/python/lib{64,}; do
 done
 
 # Note that this uses the external system python executable
-PYTHONPATH=${PYTHONPATH} python "${IMPALA_PY_DIR}/bootstrap_virtualenv.py"
+PYTHONPATH=${PYTHONPATH} ambari-python-wrap "${IMPALA_PY_DIR}/bootstrap_virtualenv.py"
 
 # Enable remote debugging if port was specified via environment variable
 if [[ ${IMPALA_SHELL_DEBUG_PORT:-0} -ne 0 ]]; then

bin/jenkins/critique-gerrit-review.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

bin/jenkins/dockerized-impala-preserve-vars.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/jenkins/populate_m2_directory.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information

bin/push_to_asf.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env ambari-python-wrap
 
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/resolve_minidumps.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

bin/validate-unified-backend-test-filters.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

common/function-registry/gen_builtins_catalog.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

common/function-registry/gen_geospatial_udf_wrappers.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

common/thrift/generate_error_codes.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

common/thrift/generate_metrics.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

docker/annotate.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python -u
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

docker/monitor.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

docker/test-with-docker.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

fe/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
-    <version>4.4.0.3.2.3.3-2</version>
+    <version>4.4.0.3.2.3.3-3</version>
     <relativePath>../java/pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>

fe/src/test/resources/hive-site.xml.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

infra/python/deps/pip_download.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/env ambari-python-wrap
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file

java/TableFlattener/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
-    <version>4.4.0.3.2.3.3-2</version>
+    <version>4.4.0.3.2.3.3-3</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>nested-table-flattener</artifactId>

java/calcite-planner/pom.xml

@@ -22,12 +22,12 @@ under the License.
   <parent>
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
-    <version>4.4.0.3.2.3.3-2</version>
+    <version>4.4.0.3.2.3.3-3</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
   <artifactId>calcite-planner</artifactId>
-  <version>4.4.0.3.2.3.3-2</version>
+  <version>4.4.0.3.2.3.3-3</version>
   <packaging>jar</packaging>
 
   <name>calcite-planner</name>
@@ -36,7 +36,7 @@ under the License.
     <dependency>
       <groupId>org.apache.impala</groupId>
       <artifactId>impala-frontend</artifactId>
-      <version>4.4.0.3.2.3.3-2</version>
+      <version>4.4.0.3.2.3.3-3</version>
     </dependency>
     <dependency>
       <groupId>org.apache.calcite</groupId>

java/datagenerator/pom.xml

@@ -23,7 +23,7 @@ under the License.
   <parent>
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
-    <version>4.4.0.3.2.3.3-2</version>
+    <version>4.4.0.3.2.3.3-3</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

java/executor-deps/pom.xml

@@ -34,7 +34,7 @@ under the License.
   <parent>
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
-    <version>4.4.0.3.2.3.3-2</version>
+    <version>4.4.0.3.2.3.3-3</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.apache.impala</groupId>

java/ext-data-source/api/pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-data-source</artifactId>
-    <version>4.4.0.3.2.3.3-2</version>
+    <version>4.4.0.3.2.3.3-3</version>
   </parent>
   <artifactId>impala-data-source-api</artifactId>
   <name>Apache Impala External Data Source API</name>

java/ext-data-source/jdbc/pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-data-source</artifactId>
-    <version>4.4.0.3.2.3.3-2</version>
+    <version>4.4.0.3.2.3.3-3</version>
   </parent>
   <artifactId>impala-data-source-jdbc</artifactId>
   <name>Apache Impala External Data Source JDBC Library</name>

java/ext-data-source/pom.xml

@@ -22,7 +22,7 @@
   <parent>
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-parent</artifactId>
-    <version>4.4.0.3.2.3.3-2</version>
+    <version>4.4.0.3.2.3.3-3</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>impala-data-source</artifactId>

java/ext-data-source/sample/pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-data-source</artifactId>
-    <version>4.4.0.3.2.3.3-2</version>
+    <version>4.4.0.3.2.3.3-3</version>
   </parent>
   <artifactId>impala-data-source-sample</artifactId>
   <name>Apache Impala External Data Source Sample</name>

java/ext-data-source/test/pom.xml

@@ -23,7 +23,7 @@
   <parent>
     <groupId>org.apache.impala</groupId>
     <artifactId>impala-data-source</artifactId>
-    <version>4.4.0.3.2.3.3-2</version>
+    <version>4.4.0.3.2.3.3-3</version>
   </parent>
   <artifactId>impala-data-source-test</artifactId>
   <name>Apache Impala External Data Source Test Library</name>