Merge remote-tracking branch 'remotes/kanaka/master' into latex3

.github/workflows/main.yml

@@ -1,5 +1,9 @@
 name: Build and Test
 
+permissions:
+  contents: read
+  packages: write
+
 on:
   push: {}
   pull_request: {}
@@ -12,32 +16,46 @@ on:
 
 jobs:
   get-matrix:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     outputs:
-      do-linux: ${{ steps.get-matrix-step.outputs.do-linux }}
+      do-linux: ${{ steps.get-matrix-step.outputs.do_linux }}
       matrix-linux: ${{ steps.get-matrix-step.outputs.linux }}
-      do-macos: ${{ steps.get-matrix-step.outputs.do-macos }}
+      do-macos: ${{ steps.get-matrix-step.outputs.do_macos }}
       matrix-macos: ${{ steps.get-matrix-step.outputs.macos }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - id: files
         if: ${{ github.event_name != 'workflow_dispatch' }}
-        uses: kanaka/get-changed-files@v1
+        uses: kanaka/get-changed-files@v2
+        with:
+          default_base: master
       - id: get-matrix-step
         run: |
           export OVERRIDE_IMPLS="${{ github.event.inputs.impls }}" # "
           echo "OVERRIDE_IMPLS: ${OVERRIDE_IMPLS}"
-          ./get-ci-matrix.py ${{ steps.files.outputs.all }}
+          ./get-ci-matrix.py ${{ steps.files.outputs.all }} > "${GITHUB_OUTPUT}"
 
   linux:
     needs: get-matrix
     if: ${{ needs.get-matrix.outputs.do-linux == 'true' }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.get-matrix.outputs.matrix-linux) }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Need full history for voom like versions
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Docker Build/Push
+        run: |
+          export ${{ matrix.IMPL }}
+          ./ci.sh docker-build-push ${IMPL}
       - name: Build
         run: |
           export ${{ matrix.IMPL }}
@@ -55,22 +73,22 @@ jobs:
           export ${{ matrix.IMPL }}
           ./ci.sh perf ${IMPL}
       - name: Archive logs and debug output
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
-          name: logs
+          name: logs.${{ matrix.IMPL }}
           path: |
             *.log
             *.debug
 
   macos:
     needs: get-matrix
     if: ${{ needs.get-matrix.outputs.do-macos == 'true' }}
-    runs-on: macos-10.15
+    runs-on: macos-12
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.get-matrix.outputs.matrix-macos) }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Build
         run: |
           export ${{ matrix.IMPL }}
@@ -88,9 +106,9 @@ jobs:
           export ${{ matrix.IMPL }}
           ./ci.sh perf ${IMPL}
       - name: Archive logs and debug output
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
-          name: logs
+          name: logs.${{ matrix.IMPL }}
           path: |
             *.log
             *.debug

IMPLS.yml

@@ -49,7 +49,7 @@ IMPL:
   - {IMPL: julia}
   - {IMPL: kotlin}
   - {IMPL: livescript}
-  - {IMPL: logo, NO_SELF_HOST: 1}  # step4 timeout
+  - {IMPL: logo}
   - {IMPL: lua}
   - {IMPL: make, NO_SELF_HOST: 1}  # step4 timeout
   - {IMPL: mal, MAL_IMPL: js,      BUILD_IMPL: js,  NO_SELF_HOST: 1}

Makefile

@@ -126,7 +126,7 @@ STEP_TEST_FILES = $(strip $(wildcard \
 
 # DOCKERIZE utility functions
 lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
-impl_to_image = kanaka/mal-test-$(call lc,$(1))
+impl_to_image = ghcr.io/kanaka/mal-test-$(call lc,$(1)):$(shell ./voom-like-version.sh impls/$(1)/Dockerfile)
 
 actual_impl = $(if $(filter mal,$(1)),$(patsubst %-mal,%,$(MAL_IMPL)),$(1))
 
@@ -143,7 +143,7 @@ get_build_command = $(strip $(foreach mode,$(1)_MODE, \
       $(if $(strip $($(mode))),-e $(mode)=$($(mode)),) \
       $(if $(filter factor,$(1)),-e FACTOR_ROOTS=$(FACTOR_ROOTS),) \
       $(call impl_to_image,$(1)) \
-      $(MAKE) $(if $(strip $($(mode))),$(mode)=$($(mode)),) \
+      make $(if $(strip $($(mode))),$(mode)=$($(mode)),) \
       ,\
       $(MAKE) $(if $(strip $($(mode))),$(mode)=$($(mode)),) -C impls/$(impl))))
 

Makefile.impls

@@ -44,7 +44,6 @@ IMPLS = ada ada.2 awk bash basic bbc-basic c c.2 chuck clojure coffee common-lis
 step5_EXCLUDES += bash        # never completes at 10,000
 step5_EXCLUDES += basic       # too slow, and limited to ints of 2^16
 step5_EXCLUDES += latex3      # no iteration, limited native stack
-step5_EXCLUDES += logo        # too slow for 10,000
 step5_EXCLUDES += make        # no TCO capability (iteration or recursion)
 step5_EXCLUDES += mal         # host impl dependent
 step5_EXCLUDES += matlab      # never completes at 10,000

README.md

@@ -35,9 +35,8 @@ Here is the final diagram for [step A](process/guide.md#stepA):
 ![stepA_mal architecture](process/stepA_mal.png)
 
 If you are interested in creating a mal implementation (or just
-interested in using mal for something), you are welcome to to join our
-[Discord](https://discord.gg/CKgnNbJBpF) or join #mal on
-[libera.chat](https://libera.chat/). In addition to the [make-a-lisp
+interested in using mal for something) you are welcome to to join our
+[Discord](https://discord.gg/CKgnNbJBpF). In addition to the [make-a-lisp
 process guide](process/guide.md) there is also a [mal/make-a-lisp
 FAQ](docs/FAQ.md) where I attempt to answer some common questions.
 
@@ -1465,7 +1464,7 @@ make "docker-build^IMPL"
 
 
 **Notes**:
-* Docker images are named *"kanaka/mal-test-IMPL"*
+* Docker images are named *"ghcr.io/kanaka/mal-test-IMPL"*
 * JVM-based language implementations (Groovy, Java, Clojure, Scala):
   you will probably need to run this command once manually
   first `make DOCKERIZE=1 "repl^IMPL"` before you can run tests because

ci.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -ex
 
@@ -24,11 +24,30 @@ mode_var=${raw_mode_var/-/__}
 mode_var=${mode_var/./__}
 mode_val=${!mode_var}
 
-MAKE="make ${mode_val:+${mode_var}=${mode_val}}"
-
 log_prefix="${ACTION}${REGRESS:+-regress}-${IMPL}${mode_val:+-${mode_val}}${MAL_IMPL:+-${MAL_IMPL}}"
 TEST_OPTS="${TEST_OPTS} --debug-file ../../${log_prefix}.debug"
 
+step_summary() {
+    echo "${*}"
+    if [ "${GITHUB_STEP_SUMMARY}" ]; then
+        echo "${*}" >> "${GITHUB_STEP_SUMMARY}"
+    fi
+}
+
+img_base="${MAL_IMPL:-${IMPL}}"
+img_impl="${img_base%%-mal}"
+img_name="mal-test-$(echo "${img_impl}" | tr '[:upper:]' '[:lower:]')"
+img_ver=$(./voom-like-version.sh impls/${img_impl}/Dockerfile)
+IMAGE="ghcr.io/kanaka/${img_name}:${img_ver}"
+
+# If NO_DOCKER is blank then run make in a docker image
+MAKE="make ${mode_val:+${mode_var}=${mode_val}}"
+if [ -z "${NO_DOCKER}" ]; then
+    # We could just use make DOCKERIZE=1 instead but that does add
+    # non-trivial startup overhead for each step.
+    MAKE="docker run -i -u $(id -u) -v `pwd`:/mal ${IMAGE} ${MAKE}"
+fi
+
 # Log everything below this point:
 exec &> >(tee ./${log_prefix}.log)
 
@@ -47,17 +66,21 @@ echo "IMPL: ${IMPL}"
 echo "BUILD_IMPL: ${BUILD_IMPL}"
 echo "MAL_IMPL: ${MAL_IMPL}"
 echo "TEST_OPTS: ${TEST_OPTS}"
-
-# If NO_DOCKER is blank then launch use a docker image, otherwise use
-# the Travis/Github Actions image/tools directly.
-if [ -z "${NO_DOCKER}" ]; then
-    img_impl=$(echo "${MAL_IMPL:-${IMPL}}" | tr '[:upper:]' '[:lower:]')
-    # We could just use make DOCKERIZE=1 instead but that does add
-    # non-trivial startup overhead for each step.
-    MAKE="docker run -i -u $(id -u) -v `pwd`:/mal kanaka/mal-test-${img_impl%%-mal} ${MAKE}"
-fi
+echo "IMAGE: ${IMAGE}"
+echo "MAKE: ${MAKE}"
 
 case "${ACTION}" in
+docker-build-push)
+    if ! docker pull ${IMAGE}; then
+        step_summary "${MAL_IMPL:-${IMPL}} - building ${IMAGE}"
+        make "docker-build^${MAL_IMPL:-${IMPL}}"
+        step_summary "${MAL_IMPL:-${IMPL}} - built ${IMAGE}"
+        if [ "${GITHUB_REF}" = "refs/heads/main" ]; then
+            docker push ${IMAGE}
+            step_summary "${MAL_IMPL:-${IMPL}} - pushed ${IMAGE}"
+        fi
+    fi
+    ;;
 build)
     # rpython often fails on step9 in compute_vars_longevity
     # so build step9, then continue with the full build

docs/FAQ.md

@@ -154,11 +154,57 @@ into the main repository:
   `time-ms` function which is needed to run the micro-benchmark tests).
 
 * Create a `Dockerfile` in your directory that installs all the
-  packages necessary to build and run your implementation. Refer to other
-  implementations for examples of what the Dockerfile should contain.
-  Build your docker image and tag it `kanaka/mal-test-[IMPL_NAME]`.
-  The top-level Makefile has support for building/testing within
-  docker with the `DOCKERIZE` flag:
+  packages necessary to build and run your implementation. In order to
+  integrate fully with the Github Actions CI workflow, the
+  `Dockerfile` needs to include the following boilerplate (with your
+  name, email, and implementation filled in):
+  ```
+  MAINTAINER Your Name <[email protected]>
+  LABEL org.opencontainers.image.source=https://github.com/kanaka/mal
+  LABEL org.opencontainers.image.description="mal test container: Your_Implementation"
+  ```
+
+  In addition, the docker image should provide python3 (with a python
+  symlink to it) to enable running tests using the image. Here is the
+  typical `Dockerfile` template you should use if your
+  implementation does not require a special base distro:
+
+  ```
+  FROM ubuntu:24.04
+  MAINTAINER Your Name <[email protected]>
+  LABEL org.opencontainers.image.source=https://github.com/kanaka/mal
+  LABEL org.opencontainers.image.description="mal test container: Your_Implementation"
+  ##########################################################
+  # General requirements for testing or common across many
+  # implementations
+  ##########################################################
+
+  RUN apt-get -y update
+
+  # Required for running tests
+  RUN apt-get -y install make python3
+  RUN ln -sf /usr/bin/python3 /usr/bin/python
+
+  # Some typical implementation and test requirements
+  RUN apt-get -y install curl libreadline-dev libedit-dev
+
+  RUN mkdir -p /mal
+  WORKDIR /mal
+
+  ##########################################################
+  # Specific implementation requirements
+  ##########################################################
+
+  ... Your packages ...
+  ```
+
+* Build and tag your docker image. The image tag will have the
+  form `ghcr.io/kanaka/mal-test-[IMPL_NAME]:[VOOM_VERSION]`.
+  ```
+  make "docker-build^[IMPL_NAME]"
+
+* The top-level Makefile has support for building/testing using
+  the docker image with the `DOCKERIZE` flag:
   ```bash
   make DOCKERIZE=1 "test^[IMPL_NAME]"
   make DOCKERIZE=1 MAL_IMPL=[IMPL_NAME] "test^mal"
@@ -170,6 +216,9 @@ into the main repository:
   ./ci.sh test [IMPL_NAME]
   ```
 
+* Push your code to a branch and make sure that the automated Github
+  Actions CI passes for your implementation.
+
 * If you are creating a new implementation for an existing
   implementation (or somebody beats you to the punch while you are
   working on it), there is still a chance I will merge your

get-ci-matrix.py

@@ -12,6 +12,9 @@
 
 OVERRIDE_IMPLS = os.environ.get('OVERRIDE_IMPLS', '').split()
 
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
 def impl_text(impl):
     s = "IMPL=%s" % impl['IMPL']
     for k, v in impl.items():
@@ -37,11 +40,11 @@ def impl_text(impl):
         do_full = True
 
 
-print("OVERRIDE_IMPLS: %s" % OVERRIDE_IMPLS)
-print("code_changes: %s (%d)" % (code_changes, len(code_changes)))
-print("impl_changes: %s (%d)" % (impl_changes, len(impl_changes)))
-print("run_impls: %s (%d)" % (run_impls, len(run_impls)))
-print("do_full: %s" % do_full)
+eprint("OVERRIDE_IMPLS: %s" % OVERRIDE_IMPLS)
+eprint("code_changes: %s (%d)" % (code_changes, len(code_changes)))
+eprint("impl_changes: %s (%d)" % (impl_changes, len(impl_changes)))
+eprint("run_impls: %s (%d)" % (run_impls, len(run_impls)))
+eprint("do_full: %s" % do_full)
 
 # Load the full implementation description file
 all_impls = yaml.safe_load(open(IMPLS_FILE))
@@ -60,7 +63,7 @@ def impl_text(impl):
     elif do_full:
         targ.append(impl_text(impl))
 
-print("::set-output name=do-linux::%s" % json.dumps(len(linux_impls)>0))
-print("::set-output name=do-macos::%s" % json.dumps(len(macos_impls)>0))
-print("::set-output name=linux::{\"IMPL\":%s}" % json.dumps(linux_impls))
-print("::set-output name=macos::{\"IMPL\":%s}" % json.dumps(macos_impls))
+print("do_linux=%s" % json.dumps(len(linux_impls)>0))
+print("do_macos=%s" % json.dumps(len(macos_impls)>0))
+print("linux={\"IMPL\":%s}" % json.dumps(linux_impls))
+print("macos={\"IMPL\":%s}" % json.dumps(macos_impls))

impls/ada.2/envs.adb

@@ -53,6 +53,25 @@ package body Envs is
       return HM.Element (Position);
    end Get;
 
+   function Get_Or_Nil (Env : Instance;
+                        Key : Types.String_Ptr) return Types.T is
+      Position : HM.Cursor := Env.Data.Find (Key);
+      Ref      : Link;
+   begin
+      if not HM.Has_Element (Position) then
+         Ref := Env.Outer;
+         loop
+            if Ref = null then
+               return Types.Nil;
+            end if;
+            Position := Ref.all.Data.Find (Key);
+            exit when HM.Has_Element (Position);
+            Ref := Ref.all.Outer;
+         end loop;
+      end if;
+      return HM.Element (Position);
+   end Get_Or_Nil;
+
    procedure Keep_References (Object : in out Instance) is
    begin
       for Position in Object.Data.Iterate loop

impls/ada.2/envs.ads

@@ -27,6 +27,9 @@ package Envs is
    function Get (Env : in Instance;
                  Key : in Types.String_Ptr) return Types.T;
 
+   function Get_Or_Nil (Env : Instance;
+                        Key : Types.String_Ptr) return Types.T;
+
    procedure Set (Env      : in out Instance;
                   Key      : in     Types.T;
                   New_Item : in     Types.T) with Inline;