feat(sdk): move version info to dedicated file (datahub-project#12456)

docker/datahub-ingestion-base/smoke.Dockerfile

@@ -20,9 +20,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 COPY . /datahub-src
 ARG RELEASE_VERSION
 RUN cd /datahub-src && \
-    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion/src/datahub/__init__.py && \
-    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/__init__.py && \
-    cat metadata-ingestion/src/datahub/__init__.py && \
+    sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion/src/datahub/_version.py && \
+    sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_version.py && \
+    cat metadata-ingestion/src/datahub/_version.py && \
     ./gradlew :metadata-ingestion:codegen && \
     pip install file:metadata-ingestion-modules/airflow-plugin#egg=acryl-datahub-airflow-plugin file:metadata-ingestion#egg=acryl-datahub
 

docker/datahub-ingestion/Dockerfile

@@ -26,10 +26,10 @@ COPY --chown=datahub ./metadata-ingestion-modules/airflow-plugin /metadata-inges
 
 ARG RELEASE_VERSION
 WORKDIR /metadata-ingestion
-RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py && \
-    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" airflow-plugin/src/datahub_airflow_plugin/__init__.py && \
-    cat src/datahub/__init__.py | grep __version__ && \
-    cat airflow-plugin/src/datahub_airflow_plugin/__init__.py | grep __version__
+RUN sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/_version.py && \
+    sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" airflow-plugin/src/datahub_airflow_plugin/_version.py && \
+    cat src/datahub/_version.py | grep __version__ && \
+    cat airflow-plugin/src/datahub_airflow_plugin/_version.py | grep __version__
 
 FROM base AS slim-install
 

docker/datahub-ingestion/Dockerfile-slim-only

@@ -15,8 +15,8 @@ COPY --chown=datahub ./metadata-ingestion /metadata-ingestion
 
 ARG RELEASE_VERSION
 WORKDIR /metadata-ingestion
-RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/__init__.py && \
-    cat src/datahub/__init__.py
+RUN sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/datahub/_version.py && \
+    cat src/datahub/_version.py
 
 FROM base as slim-install
 

metadata-ingestion-modules/airflow-plugin/.gitignore

@@ -1,5 +1,4 @@
 .envrc
-src/datahub_airflow_plugin/__init__.py.bak
 .vscode/
 output
 pvenv36/

metadata-ingestion-modules/airflow-plugin/scripts/release.sh

@@ -1,26 +1,31 @@
 #!/bin/bash
+# Auto-generated by python-build/generate_release_scripts.py. Do not edit manually.
+
 set -euxo pipefail
 
+ROOT=../..
+MODULE=datahub_airflow_plugin
+
 if [[ ! ${RELEASE_SKIP_TEST:-} ]] && [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
-	../../gradlew build  # also runs tests
+    ${ROOT}/gradlew build  # also runs tests
 elif [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
-	../../gradlew install
+    ${ROOT}/gradlew install
 fi
 
-MODULE=datahub_airflow_plugin
-
 # Check packaging constraint.
 python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"'
-if [[ ${RELEASE_VERSION:-} ]]; then
-    # Replace version with RELEASE_VERSION env variable
-    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/__init__.py
-else
-    vim src/${MODULE}/__init__.py
+
+# Update the release version.
+if [[ ! ${RELEASE_VERSION:-} ]]; then
+    echo "RELEASE_VERSION is not set"
+    exit 1
 fi
+sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/_version.py
 
+# Build and upload the release.
 rm -rf build dist || true
 python -m build
 if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then
     python -m twine upload 'dist/*'
 fi
-mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py
+mv src/${MODULE}/_version.py.bak src/${MODULE}/_version.py

metadata-ingestion-modules/airflow-plugin/setup.py

@@ -5,7 +5,7 @@
 import setuptools
 
 package_metadata: dict = {}
-with open("./src/datahub_airflow_plugin/__init__.py") as fp:
+with open("./src/datahub_airflow_plugin/_version.py") as fp:
     exec(fp.read(), package_metadata)
 
 

metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/__init__.py

@@ -1,23 +1,12 @@
-# Published at https://pypi.org/project/acryl-datahub/.
-__package_name__ = "acryl-datahub-airflow-plugin"
-__version__ = "1!0.0.0.dev0"
+from datahub_airflow_plugin._version import __package_name__, __version__
 
 
-def is_dev_mode() -> bool:
-    return __version__.endswith("dev0")
-
-
-def nice_version_name() -> str:
-    if is_dev_mode():
-        return "unavailable (installed in develop mode)"
-    return __version__
-
-
-def get_provider_info():
+def get_provider_info() -> dict:
+    # Register our hooks with Airflow.
     return {
         "package-name": f"{__package_name__}",
         "name": f"{__package_name__}",
-        "description": "Datahub metadata collector plugin",
+        "description": "DataHub metadata collector plugin",
         "connection-types": [
             {
                 "hook-class-name": "datahub_airflow_plugin.hooks.datahub.DatahubRestHook",

metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/_version.py

@@ -0,0 +1,3 @@
+# Published at https://pypi.org/project/acryl-datahub-airflow-plugin/.
+__package_name__ = "acryl-datahub-airflow-plugin"
+__version__ = "1!0.0.0.dev0"

metadata-ingestion-modules/airflow-plugin/src/datahub_airflow_plugin/datahub_plugin.py

@@ -4,12 +4,12 @@
 
 from airflow.plugins_manager import AirflowPlugin
 
-from datahub_airflow_plugin import __package_name__
 from datahub_airflow_plugin._airflow_compat import AIRFLOW_PATCHED
 from datahub_airflow_plugin._airflow_shims import (
     HAS_AIRFLOW_LISTENER_API,
     NEEDS_AIRFLOW_LISTENER_MODULE,
 )
+from datahub_airflow_plugin._version import __package_name__
 
 assert AIRFLOW_PATCHED
 logger = logging.getLogger(__name__)

metadata-ingestion-modules/dagster-plugin/.gitignore

@@ -1,5 +1,4 @@
 .envrc
-src/datahub_dagster_plugin/__init__.py.bak
 .vscode/
 output
 pvenv36/

metadata-ingestion-modules/dagster-plugin/scripts/release.sh

@@ -1,26 +1,31 @@
 #!/bin/bash
+# Auto-generated by python-build/generate_release_scripts.py. Do not edit manually.
+
 set -euxo pipefail
 
+ROOT=../..
+MODULE=datahub_dagster_plugin
+
 if [[ ! ${RELEASE_SKIP_TEST:-} ]] && [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
-	../../gradlew build  # also runs tests
+    ${ROOT}/gradlew build  # also runs tests
 elif [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
-	../../gradlew install
+    ${ROOT}/gradlew install
 fi
 
-MODULE=datahub_dagster_plugin
-
 # Check packaging constraint.
 python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"'
-if [[ ${RELEASE_VERSION:-} ]]; then
-    # Replace version with RELEASE_VERSION env variable
-    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/__init__.py
-else
-    vim src/${MODULE}/__init__.py
+
+# Update the release version.
+if [[ ! ${RELEASE_VERSION:-} ]]; then
+    echo "RELEASE_VERSION is not set"
+    exit 1
 fi
+sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/_version.py
 
+# Build and upload the release.
 rm -rf build dist || true
 python -m build
 if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then
     python -m twine upload 'dist/*'
 fi
-mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py
+mv src/${MODULE}/_version.py.bak src/${MODULE}/_version.py

metadata-ingestion-modules/dagster-plugin/setup.py

@@ -4,7 +4,7 @@
 import setuptools
 
 package_metadata: dict = {}
-with open("./src/datahub_dagster_plugin/__init__.py") as fp:
+with open("./src/datahub_dagster_plugin/_version.py") as fp:
     exec(fp.read(), package_metadata)
 
 

metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/__init__.py

@@ -1,21 +1 @@
-# Published at https://pypi.org/project/acryl-datahub/.
-__package_name__ = "acryl-datahub-dagster-plugin"
-__version__ = "1!0.0.0.dev0"
-
-
-def is_dev_mode() -> bool:
-    return __version__.endswith("dev0")
-
-
-def nice_version_name() -> str:
-    if is_dev_mode():
-        return "unavailable (installed in develop mode)"
-    return __version__
-
-
-def get_provider_info():
-    return {
-        "package-name": f"{__package_name__}",
-        "name": f"{__package_name__}",
-        "description": "Datahub metadata collector plugin",
-    }
+from datahub_dagster_plugin._version import __package_name__, __version__

metadata-ingestion-modules/dagster-plugin/src/datahub_dagster_plugin/_version.py

@@ -0,0 +1,3 @@
+# Published at https://pypi.org/project/acryl-datahub-dagster-plugin/.
+__package_name__ = "acryl-datahub-dagster-plugin"
+__version__ = "1!0.0.0.dev0"

metadata-ingestion-modules/gx-plugin/.gitignore

@@ -1,5 +1,4 @@
 .envrc
-src/datahub_gx_plugin/__init__.py.bak
 .vscode/
 output
 pvenv36/

metadata-ingestion-modules/gx-plugin/scripts/release.sh

@@ -1,26 +1,31 @@
 #!/bin/bash
+# Auto-generated by python-build/generate_release_scripts.py. Do not edit manually.
+
 set -euxo pipefail
 
+ROOT=../..
+MODULE=datahub_gx_plugin
+
 if [[ ! ${RELEASE_SKIP_TEST:-} ]] && [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
-	../../gradlew build  # also runs tests
+    ${ROOT}/gradlew build  # also runs tests
 elif [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
-	../../gradlew install
+    ${ROOT}/gradlew install
 fi
 
-MODULE=datahub_gx_plugin
-
 # Check packaging constraint.
 python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"'
-if [[ ${RELEASE_VERSION:-} ]]; then
-    # Replace version with RELEASE_VERSION env variable
-    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/__init__.py
-else
-    vim src/${MODULE}/__init__.py
+
+# Update the release version.
+if [[ ! ${RELEASE_VERSION:-} ]]; then
+    echo "RELEASE_VERSION is not set"
+    exit 1
 fi
+sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/_version.py
 
+# Build and upload the release.
 rm -rf build dist || true
 python -m build
 if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then
     python -m twine upload 'dist/*'
 fi
-mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py
+mv src/${MODULE}/_version.py.bak src/${MODULE}/_version.py

metadata-ingestion-modules/gx-plugin/setup.py

@@ -4,7 +4,7 @@
 import setuptools
 
 package_metadata: dict = {}
-with open("./src/datahub_gx_plugin/__init__.py") as fp:
+with open("./src/datahub_gx_plugin/_version.py") as fp:
     exec(fp.read(), package_metadata)
 
 

metadata-ingestion-modules/gx-plugin/src/datahub_gx_plugin/__init__.py

@@ -1,21 +1 @@
-# Published at https://pypi.org/project/acryl-datahub/.
-__package_name__ = "acryl-datahub-gx-plugin"
-__version__ = "1!0.0.0.dev0"
-
-
-def is_dev_mode() -> bool:
-    return __version__.endswith("dev0")
-
-
-def nice_version_name() -> str:
-    if is_dev_mode():
-        return "unavailable (installed in develop mode)"
-    return __version__
-
-
-def get_provider_info():
-    return {
-        "package-name": f"{__package_name__}",
-        "name": f"{__package_name__}",
-        "description": "Datahub metadata collector plugin",
-    }
+from datahub_gx_plugin._version import __package_name__, __version__

metadata-ingestion-modules/gx-plugin/src/datahub_gx_plugin/_version.py

@@ -0,0 +1,3 @@
+# Published at https://pypi.org/project/acryl-datahub-gx-plugin/.
+__package_name__ = "acryl-datahub-gx-plugin"
+__version__ = "1!0.0.0.dev0"

metadata-ingestion-modules/prefect-plugin/.gitignore

@@ -1,5 +1,4 @@
 .envrc
-src/prefect_datahub/__init__.py.bak
 .vscode/
 output
 pvenv36/

metadata-ingestion-modules/prefect-plugin/scripts/release.sh

@@ -1,26 +1,31 @@
 #!/bin/bash
+# Auto-generated by python-build/generate_release_scripts.py. Do not edit manually.
+
 set -euxo pipefail
 
+ROOT=../..
+MODULE=prefect_datahub
+
 if [[ ! ${RELEASE_SKIP_TEST:-} ]] && [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
-	../../gradlew build  # also runs tests
+    ${ROOT}/gradlew build  # also runs tests
 elif [[ ! ${RELEASE_SKIP_INSTALL:-} ]]; then
-	../../gradlew install
+    ${ROOT}/gradlew install
 fi
 
-MODULE=prefect_datahub
-
 # Check packaging constraint.
 python -c 'import setuptools; where="./src"; assert setuptools.find_packages(where) == setuptools.find_namespace_packages(where), "you seem to be missing or have extra __init__.py files"'
-if [[ ${RELEASE_VERSION:-} ]]; then
-    # Replace version with RELEASE_VERSION env variable
-    sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/__init__.py
-else
-    vim src/${MODULE}/__init__.py
+
+# Update the release version.
+if [[ ! ${RELEASE_VERSION:-} ]]; then
+    echo "RELEASE_VERSION is not set"
+    exit 1
 fi
+sed -i.bak "s/__version__ = .*$/__version__ = \"$(echo $RELEASE_VERSION|sed s/-/+/)\"/" src/${MODULE}/_version.py
 
+# Build and upload the release.
 rm -rf build dist || true
 python -m build
 if [[ ! ${RELEASE_SKIP_UPLOAD:-} ]]; then
     python -m twine upload 'dist/*'
 fi
-mv src/${MODULE}/__init__.py.bak src/${MODULE}/__init__.py
+mv src/${MODULE}/_version.py.bak src/${MODULE}/_version.py

metadata-ingestion-modules/prefect-plugin/setup.py

@@ -4,7 +4,7 @@
 import setuptools
 
 package_metadata: dict = {}
-with open("./src/prefect_datahub/__init__.py") as fp:
+with open("./src/prefect_datahub/_version.py") as fp:
     exec(fp.read(), package_metadata)
 
 
@@ -30,9 +30,7 @@ def get_long_description():
     # Temporary pinning to 2.0.0 until we can upgrade to 3.0.0
     "prefect >= 2.0.0,<3.0.0",
     *rest_common,
-    # Ignoring the dependency below because it causes issues with the vercel built wheel install
-    # f"acryl-datahub[datahub-rest]{_self_pin}",
-    "acryl-datahub[datahub-rest]",
+    f"acryl-datahub[datahub-rest]{_self_pin}",
 }