Merge remote-tracking branch 'remotes/kanaka/master' into mal-impl-607

.github/workflows/main.yml

@@ -1,5 +1,9 @@
 name: Build and Test
 
+permissions:
+  contents: read
+  packages: write
+
 on:
   push: {}
   pull_request: {}
@@ -12,32 +16,46 @@ on:
 
 jobs:
   get-matrix:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     outputs:
-      do-linux: ${{ steps.get-matrix-step.outputs.do-linux }}
+      do-linux: ${{ steps.get-matrix-step.outputs.do_linux }}
       matrix-linux: ${{ steps.get-matrix-step.outputs.linux }}
-      do-macos: ${{ steps.get-matrix-step.outputs.do-macos }}
+      do-macos: ${{ steps.get-matrix-step.outputs.do_macos }}
       matrix-macos: ${{ steps.get-matrix-step.outputs.macos }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - id: files
         if: ${{ github.event_name != 'workflow_dispatch' }}
-        uses: kanaka/get-changed-files@v1
+        uses: kanaka/get-changed-files@v2
+        with:
+          default_base: master
       - id: get-matrix-step
         run: |
           export OVERRIDE_IMPLS="${{ github.event.inputs.impls }}" # "
           echo "OVERRIDE_IMPLS: ${OVERRIDE_IMPLS}"
-          ./get-ci-matrix.py ${{ steps.files.outputs.all }}
+          ./get-ci-matrix.py ${{ steps.files.outputs.all }} > "${GITHUB_OUTPUT}"
 
   linux:
     needs: get-matrix
     if: ${{ needs.get-matrix.outputs.do-linux == 'true' }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.get-matrix.outputs.matrix-linux) }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Need full history for voom like versions
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Docker Build/Push
+        run: |
+          export ${{ matrix.IMPL }}
+          ./ci.sh docker-build-push ${IMPL}
       - name: Build
         run: |
           export ${{ matrix.IMPL }}
@@ -55,22 +73,22 @@ jobs:
           export ${{ matrix.IMPL }}
           ./ci.sh perf ${IMPL}
       - name: Archive logs and debug output
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
-          name: logs
+          name: logs.${{ matrix.IMPL }}
           path: |
             *.log
             *.debug
 
   macos:
     needs: get-matrix
     if: ${{ needs.get-matrix.outputs.do-macos == 'true' }}
-    runs-on: macos-10.15
+    runs-on: macos-12
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.get-matrix.outputs.matrix-macos) }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Build
         run: |
           export ${{ matrix.IMPL }}
@@ -88,9 +106,9 @@ jobs:
           export ${{ matrix.IMPL }}
           ./ci.sh perf ${IMPL}
       - name: Archive logs and debug output
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
-          name: logs
+          name: logs.${{ matrix.IMPL }}
           path: |
             *.log
             *.debug

Makefile

@@ -126,7 +126,7 @@ STEP_TEST_FILES = $(strip $(wildcard \
 
 # DOCKERIZE utility functions
 lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
-impl_to_image = kanaka/mal-test-$(call lc,$(1))
+impl_to_image = ghcr.io/kanaka/mal-test-$(call lc,$(1)):$(shell ./voom-like-version.sh impls/$(1)/Dockerfile)
 
 actual_impl = $(if $(filter mal,$(1)),$(patsubst %-mal,%,$(MAL_IMPL)),$(1))
 
@@ -143,7 +143,7 @@ get_build_command = $(strip $(foreach mode,$(1)_MODE, \
       $(if $(strip $($(mode))),-e $(mode)=$($(mode)),) \
       $(if $(filter factor,$(1)),-e FACTOR_ROOTS=$(FACTOR_ROOTS),) \
       $(call impl_to_image,$(1)) \
-      $(MAKE) $(if $(strip $($(mode))),$(mode)=$($(mode)),) \
+      make $(if $(strip $($(mode))),$(mode)=$($(mode)),) \
       ,\
       $(MAKE) $(if $(strip $($(mode))),$(mode)=$($(mode)),) -C impls/$(impl))))
 

README.md

@@ -35,9 +35,8 @@ Here is the final diagram for [step A](process/guide.md#stepA):
 ![stepA_mal architecture](process/stepA_mal.png)
 
 If you are interested in creating a mal implementation (or just
-interested in using mal for something), you are welcome to to join our
-[Discord](https://discord.gg/CKgnNbJBpF) or join #mal on
-[libera.chat](https://libera.chat/). In addition to the [make-a-lisp
+interested in using mal for something) you are welcome to to join our
+[Discord](https://discord.gg/CKgnNbJBpF). In addition to the [make-a-lisp
 process guide](process/guide.md) there is also a [mal/make-a-lisp
 FAQ](docs/FAQ.md) where I attempt to answer some common questions.
 
@@ -1452,7 +1451,7 @@ make "docker-build^IMPL"
 
 
 **Notes**:
-* Docker images are named *"kanaka/mal-test-IMPL"*
+* Docker images are named *"ghcr.io/kanaka/mal-test-IMPL"*
 * JVM-based language implementations (Groovy, Java, Clojure, Scala):
   you will probably need to run this command once manually
   first `make DOCKERIZE=1 "repl^IMPL"` before you can run tests because

ci.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -ex
 
@@ -24,11 +24,30 @@ mode_var=${raw_mode_var/-/__}
 mode_var=${mode_var/./__}
 mode_val=${!mode_var}
 
-MAKE="make ${mode_val:+${mode_var}=${mode_val}}"
-
 log_prefix="${ACTION}${REGRESS:+-regress}-${IMPL}${mode_val:+-${mode_val}}${MAL_IMPL:+-${MAL_IMPL}}"
 TEST_OPTS="${TEST_OPTS} --debug-file ../../${log_prefix}.debug"
 
+step_summary() {
+    echo "${*}"
+    if [ "${GITHUB_STEP_SUMMARY}" ]; then
+        echo "${*}" >> "${GITHUB_STEP_SUMMARY}"
+    fi
+}
+
+img_base="${MAL_IMPL:-${IMPL}}"
+img_impl="${img_base%%-mal}"
+img_name="mal-test-$(echo "${img_impl}" | tr '[:upper:]' '[:lower:]')"
+img_ver=$(./voom-like-version.sh impls/${img_impl}/Dockerfile)
+IMAGE="ghcr.io/kanaka/${img_name}:${img_ver}"
+
+# If NO_DOCKER is blank then run make in a docker image
+MAKE="make ${mode_val:+${mode_var}=${mode_val}}"
+if [ -z "${NO_DOCKER}" ]; then
+    # We could just use make DOCKERIZE=1 instead but that does add
+    # non-trivial startup overhead for each step.
+    MAKE="docker run -i -u $(id -u) -v `pwd`:/mal ${IMAGE} ${MAKE}"
+fi
+
 # Log everything below this point:
 exec &> >(tee ./${log_prefix}.log)
 
@@ -47,17 +66,21 @@ echo "IMPL: ${IMPL}"
 echo "BUILD_IMPL: ${BUILD_IMPL}"
 echo "MAL_IMPL: ${MAL_IMPL}"
 echo "TEST_OPTS: ${TEST_OPTS}"
-
-# If NO_DOCKER is blank then launch use a docker image, otherwise use
-# the Travis/Github Actions image/tools directly.
-if [ -z "${NO_DOCKER}" ]; then
-    img_impl=$(echo "${MAL_IMPL:-${IMPL}}" | tr '[:upper:]' '[:lower:]')
-    # We could just use make DOCKERIZE=1 instead but that does add
-    # non-trivial startup overhead for each step.
-    MAKE="docker run -i -u $(id -u) -v `pwd`:/mal kanaka/mal-test-${img_impl%%-mal} ${MAKE}"
-fi
+echo "IMAGE: ${IMAGE}"
+echo "MAKE: ${MAKE}"
 
 case "${ACTION}" in
+docker-build-push)
+    if ! docker pull ${IMAGE}; then
+        step_summary "${MAL_IMPL:-${IMPL}} - building ${IMAGE}"
+        make "docker-build^${MAL_IMPL:-${IMPL}}"
+        step_summary "${MAL_IMPL:-${IMPL}} - built ${IMAGE}"
+        if [ "${GITHUB_REF}" = "refs/heads/main" ]; then
+            docker push ${IMAGE}
+            step_summary "${MAL_IMPL:-${IMPL}} - pushed ${IMAGE}"
+        fi
+    fi
+    ;;
 build)
     # rpython often fails on step9 in compute_vars_longevity
     # so build step9, then continue with the full build

docs/FAQ.md

@@ -154,11 +154,57 @@ into the main repository:
   `time-ms` function which is needed to run the micro-benchmark tests).
 
 * Create a `Dockerfile` in your directory that installs all the
-  packages necessary to build and run your implementation. Refer to other
-  implementations for examples of what the Dockerfile should contain.
-  Build your docker image and tag it `kanaka/mal-test-[IMPL_NAME]`.
-  The top-level Makefile has support for building/testing within
-  docker with the `DOCKERIZE` flag:
+  packages necessary to build and run your implementation. In order to
+  integrate fully with the Github Actions CI workflow, the
+  `Dockerfile` needs to include the following boilerplate (with your
+  name, email, and implementation filled in):
+  ```
+  MAINTAINER Your Name <[email protected]>
+  LABEL org.opencontainers.image.source=https://github.com/kanaka/mal
+  LABEL org.opencontainers.image.description="mal test container: Your_Implementation"
+  ```
+
+  In addition, the docker image should provide python3 (with a python
+  symlink to it) to enable running tests using the image. Here is the
+  typical `Dockerfile` template you should use if your
+  implementation does not require a special base distro:
+
+  ```
+  FROM ubuntu:24.04
+  MAINTAINER Your Name <[email protected]>
+  LABEL org.opencontainers.image.source=https://github.com/kanaka/mal
+  LABEL org.opencontainers.image.description="mal test container: Your_Implementation"
+  ##########################################################
+  # General requirements for testing or common across many
+  # implementations
+  ##########################################################
+
+  RUN apt-get -y update
+
+  # Required for running tests
+  RUN apt-get -y install make python3
+  RUN ln -sf /usr/bin/python3 /usr/bin/python
+
+  # Some typical implementation and test requirements
+  RUN apt-get -y install curl libreadline-dev libedit-dev
+
+  RUN mkdir -p /mal
+  WORKDIR /mal
+
+  ##########################################################
+  # Specific implementation requirements
+  ##########################################################
+
+  ... Your packages ...
+  ```
+
+* Build and tag your docker image. The image tag will have the
+  form `ghcr.io/kanaka/mal-test-[IMPL_NAME]:[VOOM_VERSION]`.
+  ```
+  make "docker-build^[IMPL_NAME]"
+
+* The top-level Makefile has support for building/testing using
+  the docker image with the `DOCKERIZE` flag:
   ```bash
   make DOCKERIZE=1 "test^[IMPL_NAME]"
   make DOCKERIZE=1 MAL_IMPL=[IMPL_NAME] "test^mal"
@@ -170,6 +216,9 @@ into the main repository:
   ./ci.sh test [IMPL_NAME]
   ```
 
+* Push your code to a branch and make sure that the automated Github
+  Actions CI passes for your implementation.
+
 * If you are creating a new implementation for an existing
   implementation (or somebody beats you to the punch while you are
   working on it), there is still a chance I will merge your

get-ci-matrix.py

@@ -12,6 +12,9 @@
 
 OVERRIDE_IMPLS = os.environ.get('OVERRIDE_IMPLS', '').split()
 
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
 def impl_text(impl):
     s = "IMPL=%s" % impl['IMPL']
     for k, v in impl.items():
@@ -37,11 +40,11 @@ def impl_text(impl):
         do_full = True
 
 
-print("OVERRIDE_IMPLS: %s" % OVERRIDE_IMPLS)
-print("code_changes: %s (%d)" % (code_changes, len(code_changes)))
-print("impl_changes: %s (%d)" % (impl_changes, len(impl_changes)))
-print("run_impls: %s (%d)" % (run_impls, len(run_impls)))
-print("do_full: %s" % do_full)
+eprint("OVERRIDE_IMPLS: %s" % OVERRIDE_IMPLS)
+eprint("code_changes: %s (%d)" % (code_changes, len(code_changes)))
+eprint("impl_changes: %s (%d)" % (impl_changes, len(impl_changes)))
+eprint("run_impls: %s (%d)" % (run_impls, len(run_impls)))
+eprint("do_full: %s" % do_full)
 
 # Load the full implementation description file
 all_impls = yaml.safe_load(open(IMPLS_FILE))
@@ -60,7 +63,7 @@ def impl_text(impl):
     elif do_full:
         targ.append(impl_text(impl))
 
-print("::set-output name=do-linux::%s" % json.dumps(len(linux_impls)>0))
-print("::set-output name=do-macos::%s" % json.dumps(len(macos_impls)>0))
-print("::set-output name=linux::{\"IMPL\":%s}" % json.dumps(linux_impls))
-print("::set-output name=macos::{\"IMPL\":%s}" % json.dumps(macos_impls))
+print("do_linux=%s" % json.dumps(len(linux_impls)>0))
+print("do_macos=%s" % json.dumps(len(macos_impls)>0))
+print("linux={\"IMPL\":%s}" % json.dumps(linux_impls))
+print("macos={\"IMPL\":%s}" % json.dumps(macos_impls))

impls/ada/Dockerfile

@@ -1,5 +1,7 @@
-FROM ubuntu:vivid
+FROM ubuntu:xenial
 MAINTAINER Joel Martin <[email protected]>
+LABEL org.opencontainers.image.source=https://github.com/kanaka/mal
+LABEL org.opencontainers.image.description="mal test container: ada"
 
 ##########################################################
 # General requirements for testing or common across many

impls/awk/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:vivid
+FROM ubuntu:xenial
 MAINTAINER Joel Martin <[email protected]>
 
 ##########################################################

impls/bash/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:vivid
+FROM ubuntu:xenial
 MAINTAINER Joel Martin <[email protected]>
 
 ##########################################################

impls/c.2/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:bionic
+FROM ubuntu:24.04
 MAINTAINER Duncan Watts <[email protected]>
 
 ##########################################################
@@ -9,10 +9,8 @@ MAINTAINER Duncan Watts <[email protected]>
 RUN apt-get -y update
 
 # Required for running tests
-RUN apt-get -y install make python
-
-# Some typical implementation and test requirements
-#RUN apt-get -y install curl
+RUN apt-get -y install make python3
+RUN ln -fs /usr/bin/python3 /usr/local/bin/python
 
 RUN mkdir -p /mal
 WORKDIR /mal

impls/common-lisp/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:vivid
+FROM ubuntu:xenial
 MAINTAINER Joel Martin <[email protected]>
 
 ##########################################################

impls/cs/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:vivid
+FROM ubuntu:xenial
 MAINTAINER Joel Martin <[email protected]>
 
 ##########################################################
@@ -22,4 +22,4 @@ WORKDIR /mal
 ##########################################################
 
 # Deps for Mono-based languages (C#, VB.Net)
-RUN apt-get -y install mono-runtime mono-mcs mono-vbnc mono-devel
+RUN apt-get -y install tzdata mono-runtime mono-mcs mono-vbnc mono-devel

impls/forth/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:vivid
+FROM ubuntu:xenial
 MAINTAINER Joel Martin <[email protected]>
 
 ##########################################################