Merge pull request #159 from netbox-community/multistage-build

Refactor to multistage builds

.dockerignore

@@ -2,3 +2,4 @@
 .travis.yml
 build*
 *.env
+.git

.gitignore

@@ -1 +1,2 @@
 *.sql.gz
+.netbox

DOCKER_HUB.md

@@ -12,28 +12,25 @@ Repository Links: Enable for Base Image
 Build Rules:
 - Source Type: Branch
   Source: master
-  Docker Tag: branches-main
+  Docker Tag: branches
   Dockerfile location: Dockerfile
+  Build Context: /
+  Autobuild: on
+  Build Caching: on
 - Source Type: Branch
   Source: master
-  Docker Tag: branches-ldap
-  Dockerfile location: Dockerfile.ldap
-- Source Type: Branch
-  Source: master
-  Docker Tag: prerelease-main
+  Docker Tag: prerelease
   Dockerfile location: Dockerfile
+  Build Context: /
+  Autobuild: on
+  Build Caching: on
 - Source Type: Branch
   Source: master
-  Docker Tag: prerelease-ldap
-  Dockerfile location: Dockerfile.ldap
-- Source Type: Branch
-  Source: master
-  Docker Tag: release-main
+  Docker Tag: release
   Dockerfile location: Dockerfile
-- Source Type: Branch
-  Source: master
-  Docker Tag: release-ldap
-  Dockerfile location: Dockerfile.ldap
+  Build Context: /
+  Autobuild: on
+  Build Caching: on
 Build Environment Variables:
 # Create an app on Github and use it's OATH credentials here
 - Key: GITHUB_OAUTH_CLIENT_ID
@@ -42,6 +39,7 @@ Build Environment Variables:
   Value: <secret>
 Build Triggers:
 - Name: Cron Trigger
+  Trigger URL: <generated>
 # Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds
 ```
 
@@ -51,16 +49,15 @@ The build system of cloud.docker.com is not made for this kind of project.
 But we found a way to make it work, and this is how:
 
 1. The docker hub build system [allows to overwrite the scripts that get executed
-   for `build`, `test` and `push`](overwrite). See `hooks/*`.
-2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `hooks/common`.
-3. The `build` script runs `run_build()` from `hooks/common`.
-   This triggers either `build-branches.sh`, `build-latest.sh` or directly `build.sh`.
+   for `build`, `test` and `push`](overwrite). See `/hooks/*`.
+2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `/hooks/common`.
+3. The `build` script runs `run_build()` from `/hooks/common`.
+   This triggers either `/build-branches.sh`, `/build-latest.sh` or directly `/build.sh`.
 4. The `test` script just invokes `docker-compose` commands.
 5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag.
    This causes the `build.sh` script to not re-build the Docker image, but just the just built image.
 
-The _Docker Tag_ configuration setting is misused to select the type (_release_, _prerelease_, _branches_) of the build as well as the variant (_main_, _ldap_).
-
-The _Dockerfile location_ configuration setting is completely ignored by the build scripts.
+The _Docker Tag_ configuration setting (`$DOCKER_TAG`) is only used to select the type (_release_, _prerelease_, _branches_) of the build in `hooks/common`.
+Because it has a different meaning in all the other build scripts, it is `unset` after it has served it's purpose.
 
 [overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands

Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.7-alpine3.10
+ARG FROM=python:3.7-alpine
+FROM ${FROM} as builder
 
 RUN apk add --no-cache \
       bash \
@@ -7,51 +8,55 @@ RUN apk add --no-cache \
       cyrus-sasl-dev \
       graphviz \
       jpeg-dev \
+      libevent-dev \
       libffi-dev \
-      libxml2-dev \
       libxslt-dev \
       openldap-dev \
-      postgresql-dev \
-      ttf-ubuntu-font-family \
-      wget
+      postgresql-dev
 
-RUN pip install \
+WORKDIR /install
+
+RUN pip install --install-option="--prefix=/install" \
 # gunicorn is used for launching netbox
       gunicorn \
+      greenlet \
+      eventlet \
 # napalm is used for gathering information from network devices
       napalm \
 # ruamel is used in startup_scripts
       'ruamel.yaml>=0.15,<0.16' \
-# pinning django to the version required by netbox
-# adding it here, to install the correct version of
-# django-rq
-      'Django>=2.2,<2.3' \
-# django-rq is used for webhooks
-      django-rq
-
-ARG BRANCH=master
-
-WORKDIR /tmp
-
-# As the requirements don't change very often,
-# and as they take some time to compile,
-# we try to cache them very agressively.
-ARG REQUIREMENTS_URL=https://raw.githubusercontent.com/netbox-community/netbox/$BRANCH/requirements.txt
-ADD ${REQUIREMENTS_URL} requirements.txt
-RUN pip install -r requirements.txt
-
-# Cache bust when the upstream branch changes:
-# ADD will fetch the file and check if it has changed
-# If not, Docker will use the existing build cache.
-# If yes, Docker will bust the cache and run every build step from here on.
-ARG REF_URL=https://api.github.com/repos/netbox-community/netbox/contents?ref=$BRANCH
-ADD ${REF_URL} version.json
+# django_auth_ldap is required for ldap
+      django_auth_ldap
+
+ARG NETBOX_PATH
+COPY ${NETBOX_PATH}/requirements.txt /
+RUN pip install --install-option="--prefix=/install" -r /requirements.txt 
+
+###
+# Main stage
+###
+
+ARG FROM
+FROM ${FROM} as main
+
+RUN apk add --no-cache \
+      bash \
+      ca-certificates \
+      graphviz \
+      libevent \
+      libffi \
+      libjpeg-turbo \
+      libressl \
+      libxslt \
+      postgresql-libs \
+      ttf-ubuntu-font-family
 
 WORKDIR /opt
 
-ARG URL=https://github.com/netbox-community/netbox/archive/$BRANCH.tar.gz
-RUN wget -q -O - "${URL}" | tar xz \
-  && mv netbox* netbox
+COPY --from=builder /install /usr/local
+
+ARG NETBOX_PATH
+COPY ${NETBOX_PATH} /opt/netbox
 
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
 COPY configuration/gunicorn_config.py /etc/netbox/config/
@@ -67,7 +72,22 @@ ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
 
 CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"]
 
-LABEL SRC_URL="$URL"
+LABEL NETBOX_DOCKER_PROJECT_VERSION="custom build" \
+      NETBOX_BRANCH="custom build" \
+      ORIGINAL_DOCKER_TAG="custom build" \
+      NETBOX_GIT_COMMIT="not built from git" \
+      NETBOX_GIT_URL="not built from git"
+
+#####
+## LDAP specific configuration
+#####
+
+FROM main as ldap
+
+RUN apk add --no-cache \
+      libsasl \
+      libldap \
+      util-linux
 
-ARG NETBOX_DOCKER_PROJECT_VERSION=snapshot
-LABEL NETBOX_DOCKER_PROJECT_VERSION="$NETBOX_DOCKER_PROJECT_VERSION"
+COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py
+COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py

README.md

@@ -49,8 +49,8 @@ Default credentials:
 
 This project relies only on *Docker* and *docker-compose* meeting this requirements:
 
-* The *Docker version* must be at least `1.13.0`.
-* The *docker-compose version* must be at least `1.10.0`.
+* The *Docker version* must be at least `17.05`.
+* The *docker-compose version* must be at least `1.17.0`.
 
 To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above.
 
@@ -69,24 +69,20 @@ To use this feature, set the environment-variable `VERSION` before launching `do
 [any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub].
 
 ```bash
-export VERSION=v2.2.6
+export VERSION=v2.6.6
 docker-compose pull netbox
 docker-compose up -d
 ```
 
-You can also build a specific version of the Netbox image. This time, `VERSION` indicates any valid
-[Git Reference][git-ref] declared on [the 'netbox-community/netbox' Github repository][netbox-github].
-Most commonly you will specify a tag or branch name.
+You can also build a specific version of the Netbox Docker image yourself.
+`VERSION` can be any valid [git ref][git-ref] in that case.
 
 ```bash
-export VERSION=develop
-docker-compose build --no-cache netbox
+export VERSION=v2.6.6
+./build.sh $VERSION
 docker-compose up -d
 ```
 
-Hint: If you're building a specific version by tag name, the `--no-cache` argument is not strictly necessary.
-This can increase the build speed if you're just adjusting the config, for example.
-
 [git-ref]: https://git-scm.com/book/en/v2/Git-Internals-Git-References
 [netbox-github]: https://github.com/netbox-community/netbox/releases
 

build-all.sh

@@ -12,73 +12,40 @@ BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}")
 
 echo "⚙️ Configured builds: ${BUILDS[*]}"
 
-VARIANTS=("main" "ldap")
-
 if [ -n "${DEBUG}" ]; then
   export DEBUG
 fi
 
 ERROR=0
 
-# Don't build if not on `master` and don't build if on a pull request,
-# but build when DEBUG is not empty
-for VARIANT in "${VARIANTS[@]}"; do
-  export VARIANT
-
-  # Checking which VARIANT to build
-  if [ "${VARIANT}" == "main" ]; then
-    DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}"
-  else
-    DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}.${VARIANT}"
-
-    # Fail fast
-    if [ ! -f "${DOCKERFILE}" ]; then
-      echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist."
-      ERROR=1
+for BUILD in "${BUILDS[@]}"; do
+  echo "🛠 Building '$BUILD' from '$DOCKERFILE'"
+  case $BUILD in
+    release)
+      # build the latest release
+      # shellcheck disable=SC2068
+      ./build-latest.sh $@ || ERROR=1
+      ;;
+    prerelease)
+      # build the latest pre-release
+      # shellcheck disable=SC2068
+      PRERELEASE=true ./build-latest.sh $@ || ERROR=1
+      ;;
+    branches)
+      # build all branches
+      # shellcheck disable=SC2068
+      ./build-branches.sh $@ || ERROR=1
+      ;;
+    *)
+      echo "🚨 Unrecognized build '$BUILD'."
 
       if [ -z "$DEBUG" ]; then
-        continue
+        exit 1
       else
-        echo "⚠️ Would skip this, but DEBUG is enabled."
+        echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
       fi
-    fi
-  fi
-
-  for BUILD in "${BUILDS[@]}"; do
-    echo "🛠 Building '$BUILD' from '$DOCKERFILE'"
-    case $BUILD in
-      release)
-        # build the latest release
-        # shellcheck disable=SC2068
-        ./build-latest.sh $@ || ERROR=1
-        ;;
-      prerelease)
-        # build the latest pre-release
-        # shellcheck disable=SC2068
-        PRERELEASE=true ./build-latest.sh $@ || ERROR=1
-        ;;
-      branches)
-        # build all branches
-        # shellcheck disable=SC2068
-        ./build-branches.sh $@ || ERROR=1
-        ;;
-      special)
-        # special build
-        # shellcheck disable=SC2068
-        #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ || ERROR=1
-        echo "✅ No special builds today."
-        ;;
-      *)
-        echo "🚨 Unrecognized build '$BUILD'."
-
-        if [ -z "$DEBUG" ]; then
-          exit 1
-        else
-          echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
-        fi
-        ;;
-    esac
-  done
+      ;;
+  esac
 done
 
 exit $ERROR

build-branches.sh

@@ -3,6 +3,10 @@
 
 echo "▶️ $0 $*"
 
+###
+# Checking for the presence of GITHUB_OAUTH_CLIENT_ID
+# and GITHUB_OAUTH_CLIENT_SECRET
+###
 if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
   echo "🗝 Performing authenticated Github API calls."
   GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
@@ -11,18 +15,33 @@ else
   GITHUB_OAUTH_PARAMS=""
 fi
 
+###
+# Calling Github to get the all branches
+###
 ORIGINAL_GITHUB_REPO="${SRC_ORG-netbox-community}/${SRC_REPO-netbox}"
 GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
 URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}"
 
+# Composing the JQ commans to extract the most recent version number
+JQ_BRANCHES='map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string'
+
 CURL="curl -sS"
 
-BRANCHES=$($CURL "${URL_RELEASES}" | jq -r 'map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string')
+# Querying the Github API to fetch all branches
+BRANCHES=$($CURL "${URL_RELEASES}" | jq -r "$JQ_BRANCHES")
+
+###
+# Building each branch
+###
 
+# keeping track whether an error occured
 ERROR=0
 
+# calling build.sh for each branch
 for BRANCH in $BRANCHES; do
   # shellcheck disable=SC2068
   ./build.sh "${BRANCH}" $@ || ERROR=1
 done
+
+# returning whether an error occured
 exit $ERROR