Merge pull request #113 from opensafely-core/versioning

versioning

.github/ISSUE_TEMPLATE/new-package.md

@@ -0,0 +1,29 @@
+---
+name: New Package
+about: Request a new python package and/or system library be installed
+title: "Request a New Package"
+labels: 'new-package'
+
+---
+
+Please fill in the fields below to request new packages for the OpenSAFELY python image.
+
+
+### Python package(s) you wish to add to the image
+
+<!-- please add packages, including links to pypi.org page if possible -->
+
+
+### System Libraries
+
+<! --Any system libraries that this package may require. Leave blank if unsure -->
+
+
+### Requesting Project
+
+<!-- Link to the OpenSAFELY Project that will use these packages -->
+
+
+### Rationale
+
+<!-- Rationale for use in OpenSAFELY -->

.github/workflows/build_and_publish.yaml

@@ -1,33 +1,34 @@
 name: Build and publish
 on:
+  workflow_dispatch:
   push:
     branches: [main]
-  workflow_dispatch:
-permissions:
-  packages: write
-env:
-  IMAGE_NAME: python
+
 jobs:
-  build-and-publish:
-    runs-on: ubuntu-20.04
+  publish:
+    # note: this builds/tests all versions in serial for two reasons. Firstly we
+    # want all versions to release or none of them. Secondly, we will be able
+    # publish the exact images that were built and tested.
+    runs-on: ubuntu-22.04
     steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Build image
-      run: make build
+    - uses: actions/checkout@v3
+    - uses: "opensafely-core/setup-action@v1"
+      with:
+          install-just: true
+    - name: Build images
+      run: |
+        just build v1
+        just build v2
     - name: Run tests
-      run: make test functional-test
-    - name: Run lint
-      run: make lint 
+      run: |
+        just test v1
+        just test v2
+    - name: Run linters
+      run: just check 
+
     - name: Log into GitHub Container Registry
       run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
     - name: Push image to GitHub Container Registry
       run: |
-        IMAGE_ID="ghcr.io/${{ github.repository_owner }}/$IMAGE_NAME"
-        docker tag "$IMAGE_NAME" "$IMAGE_ID:latest"
-        docker push "$IMAGE_ID:latest"
-
-        JUPYTER_ID="ghcr.io/${{ github.repository_owner }}/jupyter"
-        # also publish as jupyter image for backward compatibility
-        docker tag "$IMAGE_NAME" "$JUPYTER_ID:latest"
-        docker push "$JUPYTER_ID:latest"
+        just publish v1 true
+        just publish v2 true

.github/workflows/tests.yaml

@@ -1,17 +1,28 @@
 name: Run tests
 on:
   pull_request:
-env:
-  IMAGE_NAME: python
 jobs:
-  tests:
-    runs-on: ubuntu-20.04
+  version-tests:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        version: [v1, v2]
     steps:
-    - name: Checkout
-      uses: actions/checkout@master
+    - uses: actions/checkout@v3
+    - uses: "opensafely-core/setup-action@v1"
+      with:
+          install-just: true
     - name: Build image
-      run: make build
+      run: just build ${{ matrix.version }}
     - name: Run tests
-      run: make test functional-test
-    - name: Run lint
-      run: make lint 
+      run: just test ${{ matrix.version }}
+  lint:
+    runs-on: ubuntu-22.04
+    steps:
+    - uses: actions/checkout@v3
+    - uses: "opensafely-core/setup-action@v1"
+      with:
+          install-just: true
+    - name: Run linters
+      run: just check 
+

DEVELOPERS.md

@@ -0,0 +1,71 @@
+# Basics
+
+Each major version has its configuration in a subdirectory named after the
+version, e.g. ./v1/ has all the configuration for the `v1` image.
+
+Inside each version's directory there are 4 main files:
+
+- `env`: environment variables used to parameterise the Docker/docker-compose
+  files:
+    - `BASE`: the base Ubuntu version to build from, e.g. `22.04`
+    - `MAJOR_VERSION`: this shoud match the directory name.
+- `dependencies.txt`: the Ubuntu packages that need to be installed
+- `build-dependencies.txt`: the Ubuntu package needed to *build* any
+  dependencies (these will *not* be included in the final image).
+- `requirements.in`: the list of packages to install (*without* version
+  specfiers, unless needed for some reason).
+
+There will also be two autogenerated files:
+
+- `requirements.txt`: the fully pinned set of python dependences generated with
+  `pip-compile`.
+- `packages.md`: generated user facing documentation of package versions
+
+
+Use just to build and test image versions:
+
+```
+just build v2
+just test v2
+```
+
+
+## Add a new package to existing version
+
+* Add the new package without version specifier to all relevant version's
+  `requirement.in` files
+* For each version, do the following:
+    * Run `just update $VERSION`. This will update pacakges, then build and
+      test the new image.
+    * If the build fails, depending on the error message:
+        - you may need to add a new system package to `dependencies.txt`
+        - you may need to add a new build dependency package to
+          `build-dependencies.txt`
+        - you may need to finesse the tests for poorly packaged libraries: see
+          [`BAD_PACKAGES`](./tests/test_import.py)
+    * Inspect the changes to requirements.txt
+        - ensure no pre-existing package has been updated by this change.
+
+
+## Create a new version
+
+TODO, but basically, `cp -a v$N v${N+1}` and edit.
+
+
+## Publishing
+
+> ![WARNING]
+> By default, these images are published via CI, so only do this if you know
+> you need to, e.g. testing publishing a new version
+
+To publish a version locally, you will need to be logged in to ghcr.io with the
+right permissions (`docker login ghcr.io`)
+
+By default, this command is a dry run, and will show you the commands it *will* run:
+
+`just publish $version`
+
+To run for real, pass `true`:
+
+`just publish $version true`
+

Dockerfile

@@ -8,61 +8,77 @@
 # and b) we specifically always want to build on the latest base image, by
 # design.
 #
+ARG BASE
 # hadolint ignore=DL3007
-FROM ghcr.io/opensafely-core/base-action:latest as base-python
-COPY dependencies.txt /root/dependencies.txt
+FROM ghcr.io/opensafely-core/base-action:$BASE as base-python
+
+RUN mkdir /workspace
+WORKDIR /workspace
+
+ARG MAJOR_VERSION
+ARG BASE
+# ACTION_EXEC sets the default executable for the entrypoint in the base-docker image
+ENV ACTION_EXEC=python MAJOR_VERSION=${MAJOR_VERSION} BASE=${BASE}
+
+COPY ${MAJOR_VERSION}/dependencies.txt /opt/dependencies.txt
 # use space efficient utility from base image
-RUN /root/docker-apt-install.sh /root/dependencies.txt
+RUN /root/docker-apt-install.sh /opt/dependencies.txt
+
+# now we have python, set up a venv to install packages to, for isolation from
+# system python libraries 
+# hadolint ignore=DL3059
+RUN python3 -m venv /opt/venv
+# "activate" the venv
+ENV VIRTUAL_ENV=/opt/venv/ PATH="/opt/venv/bin:$PATH"
+# We ensure up-to-date build tools (which why we ignore DL3013)
+# hadolint ignore=DL3013,DL3042
+RUN --mount=type=cache,target=/root/.cache python -m pip install -U pip setuptools wheel pip-tools
+
 
 #################################################
 #
 # Next, use the base-docker-plus-python image to create a build image
 FROM base-python as builder
+ARG MAJOR_VERSION
 
 # install build time dependencies 
-COPY build-dependencies.txt /root/build-dependencies.txt
-RUN /root/docker-apt-install.sh /root/build-dependencies.txt
-
-# install everything in venv for isolation from system python libraries
-# hadolint ignore=DL3059
-RUN python3 -m venv /opt/venv
-ENV VIRTUAL_ENV=/opt/venv/ PATH="/opt/venv/bin:$PATH" LLVM_CONFIG=/usr/bin/llvm-config-10
+COPY ${MAJOR_VERSION}/build-dependencies.txt /opt/build-dependencies.txt
+RUN /root/docker-apt-install.sh /opt/build-dependencies.txt
 
-COPY requirements.txt /root/requirements.txt
-# We ensure up-to-date build tools (which why we ignore DL3013)
+COPY ${MAJOR_VERSION}/requirements.txt /opt/requirements.txt
+COPY ${MAJOR_VERSION}/packages.md /opt/packages.md
 # Note: the mount command does two things: 1) caches across builds to speed up
 # local development and 2) ensures the pip cache does not get committed to the
 # layer (which is why we ignore DL3042).
-# hadolint ignore=DL3013,DL3042
+# hadolint ignore=DL3042
 RUN --mount=type=cache,target=/root/.cache \
-    python -m pip install -U pip setuptools wheel && \
-    python -m pip install --requirement /root/requirements.txt
+    python -m pip install --requirement /opt/requirements.txt
 
 ################################################
 #
 # Finally, build the actual image from the base-python image
 FROM base-python as python
 
+
+ARG MAJOR_VERSION
 # Some static metadata for this specific image, as defined by:
 # https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys
 # The org.opensafely.action label is used by the jobrunner to indicate this is
 # an approved action image to run.
-LABEL org.opencontainers.image.title="python" \
+LABEL org.opencontainers.image.title="python:${MAJOR_VERSION}" \
       org.opencontainers.image.description="Python action for opensafely.org" \
       org.opencontainers.image.source="https://github.com/opensafely-core/python-docker" \
-      org.opensafely.action="python"
+      org.opensafely.action="python:${MAJOR_VERSION}"
 
 # copy venv over from builder image
-COPY --from=builder /opt/venv /opt/venv
-# ACTION_EXEC sets the default executable for the entrypoint in the base-docker image
-ENV VIRTUAL_ENV=/opt/venv/ PATH="/opt/venv/bin:$PATH" ACTION_EXEC=python
-
-RUN mkdir /workspace
-WORKDIR /workspace
+COPY --from=builder /opt/ /opt/
 
-# tag with build info as the very last step, as it will never be cached
+# tag with build info as the very last step, as it will never be cacheable
 ARG BUILD_DATE
 ARG REVISION
+ARG BUILD_NUMBER
 # RFC 3339.
 LABEL org.opencontainers.image.created=$BUILD_DATE \
-      org.opencontainers.image.revision=$REVISION
+      org.opencontainers.image.revision=$REVISION \
+      org.opencontainers.image.build=$BUILD_NUMBER \
+      org.opencontainers.image.version=$MAJOR_VERSION.$BUILD_NUMBER