refactor all code that builds csources (#17815)

* refactor all code that builds csources * fixup * nim_csourcesDir_v0 + nim_csourcesDir * remove deprecated, unused scripts from ci/ * reuse nimCsourcesHash in ci * simplify CI pipelines by reusing nimBuildCsourcesIfNeeded * simplify ci_docs.yml by reusing nimBuildCsourcesIfNeeded * cleanup * use csources_v1 as destination dir * fixup * remove pushCsources * address comment: remove build.sh support for now * fixup
nim-lang · Apr 23, 2021 · dce0b3b · dce0b3b
1 parent dbb0534
commit dce0b3b
Show file tree

Hide file tree

Showing 22 changed files with 137 additions and 340 deletions.
diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml
@@ -14,10 +14,10 @@ environment:
   CC: /usr/bin/clang
 tasks:
 - setup: |
+    set -e
     cd Nim
-    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
-    gmake -C csources -j $(sysctl -n hw.ncpu)
-    bin/nim c --skipUserCfg --skipParentCfg koch
+    . ci/funs.sh && nimBuildCsourcesIfNeeded
+    $nim_csources c --skipUserCfg --skipParentCfg koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     cd Nim

diff --git a/.builds/openbsd_0.yml b/.builds/openbsd_0.yml
@@ -17,15 +17,15 @@ environment:
   CC: /usr/bin/clang
 tasks:
 - setup: |
+    set -e
     cd Nim
-    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
-    gmake -C csources -j $(sysctl -n hw.ncpuonline)
-    bin/nim c koch
+    . ci/funs.sh && nimBuildCsourcesIfNeeded
+    $nim_csources c koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     cd Nim
     if ! ./koch runCI; then
-      nim c -r tools/ci_testresults.nim
+      nim r tools/ci_testresults.nim
       exit 1
     fi
 triggers:

diff --git a/.builds/openbsd_1.yml b/.builds/openbsd_1.yml
@@ -17,15 +17,15 @@ environment:
   CC: /usr/bin/clang
 tasks:
 - setup: |
+    set -e
     cd Nim
-    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
-    gmake -C csources -j $(sysctl -n hw.ncpuonline)
-    bin/nim c koch
+    . ci/funs.sh && nimBuildCsourcesIfNeeded
+    $nim_csources c koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
 - test: |
     cd Nim
     if ! ./koch runCI; then
-      nim c -r tools/ci_testresults.nim
+      nim r tools/ci_testresults.nim
       exit 1
     fi
 triggers:

diff --git a/.github/workflows/ci.yml.disabled b/.github/workflows/ci.yml.disabled
@@ -1,3 +1,5 @@
+# out of date and unused but revivable
+
 name: Continous Integration
 on: [push, pull_request]
 
@@ -30,11 +32,6 @@ jobs:
     steps:
       - name: 'Checkout'
         uses: actions/checkout@v2
-      - name: 'Checkout csources'
-        uses: actions/checkout@v2
-        with:
-          repository: nim-lang/csources
-          path: csources
 
       - name: 'Install node.js 8.x'
         uses: actions/setup-node@v1
@@ -91,24 +88,10 @@ jobs:
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'Build csources'
+      - name: 'Build csourcesAny'
         shell: bash
-        run: |
-          ncpu=
-          case '${{ runner.os }}' in
-          'Linux')
-            ncpu=$(nproc)
-            ;;
-          'macOS')
-            ncpu=$(sysctl -n hw.ncpu)
-            ;;
-          'Windows')
-            ncpu=$NUMBER_OF_PROCESSORS
-            ;;
-          esac
-          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
 
-          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
       - name: 'Build koch'
         shell: bash
         run: nim c koch

diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml
@@ -70,53 +70,12 @@ jobs:
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'Get current csources version'
-        id: csources-version
+      - name: 'Build csourcesAny'
         shell: bash
-        run: |
-          sha=$(git ls-remote https://github.com/nim-lang/csources_v1 master | cut -f 1)
-          echo "::set-output name=sha::$sha"
-
-      - name: 'Get prebuilt csources from cache'
-        id: csources-cache
-        uses: actions/cache@v1
-        with:
-          path: bin
-          key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}'
-
-      - name: 'Checkout csources'
-        if: steps.csources-cache.outputs.cache-hit != 'true'
-        uses: actions/checkout@v2
-        with:
-          repository: nim-lang/csources_v1
-          path: csources
-
-      - name: 'Build 1-stage compiler from csources'
-        shell: bash
-        run: |
-          ext=
-          [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe
-          if [[ ! -x bin/nim-csources$ext ]]; then
-            ncpu=
-            case '${{ runner.os }}' in
-            'Linux')
-              ncpu=$(nproc)
-              ;;
-            'macOS')
-              ncpu=$(sysctl -n hw.ncpu)
-              ;;
-            'Windows')
-              ncpu=$NUMBER_OF_PROCESSORS
-              ;;
-            esac
-            [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
-
-            make -C csources -j $ncpu CC=gcc
-            cp bin/nim{,-csources}$ext
-          else
-            echo 'Cache hit, using prebuilt csources'
-            cp bin/nim{-csources,}$ext
-          fi
+        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc
+          # was previously using caching via `actions/cache@v1` but this wasn't
+          # used in other CI pipelines and it's unclear the added complexity
+          # was worth the saving; can be revisited if needed.
 
       - name: 'Build koch'
         shell: bash

diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml
@@ -20,16 +20,11 @@ jobs:
         with:
           fetch-depth: 2
 
-      - name: 'Checkout csources'
-        uses: actions/checkout@v2
-        with:
-          repository: nim-lang/csources
-          path: csources
-
       - name: 'Install node.js 12.x'
         uses: actions/setup-node@v1
         with:
           node-version: '12.x'
+
       - name: 'Install dependencies (Linux amd64)'
         if: runner.os == 'Linux' && matrix.cpu == 'amd64'
         run: |
@@ -56,24 +51,10 @@ jobs:
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'Build csources'
+      - name: 'Build csourcesAny'
         shell: bash
-        run: |
-          ncpu=
-          case '${{ runner.os }}' in
-          'Linux')
-            ncpu=$(nproc)
-            ;;
-          'macOS')
-            ncpu=$(sysctl -n hw.ncpu)
-            ;;
-          'Windows')
-            ncpu=$NUMBER_OF_PROCESSORS
-            ;;
-          esac
-          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
 
-          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
       - name: 'Build koch'
         shell: bash
         run: nim c koch

diff --git a/.gitignore b/.gitignore
@@ -64,7 +64,10 @@ lib/**/*.html
 testament.db
 /tests/**/*.json
 /tests/**/*.js
+
 /csources
+/csources_v1
+
 /dist/
 # /lib/fusion # fusion is now unbundled; `git status` should reveal if it's there so users can act on it
 

diff --git a/.travis.yml b/.travis.yml
@@ -1,3 +1,4 @@
+# deprecated, out of date and unused
 sudo: false
 language: c
 
@@ -22,9 +23,8 @@ addons:
     - valgrind
 
 before_script:
-  - git clone --depth 1 https://github.com/nim-lang/csources.git
+  -  . ci/funs.sh && nimBuildCsourcesIfNeeded LD=$CC ucpu=$CPU
   - export PATH="$PWD/bin${PATH:+:$PATH}"
-  - make -C csources -j 2 LD=$CC ucpu=$CPU
 
 script:
   - echo "travis_fold:start:nim_c_koch"

diff --git a/appveyor.yml.disabled b/appveyor.yml.disabled
@@ -1,3 +1,4 @@
+# deprecated, out of date and unused
 version: '{build}'
 
 environment:
@@ -23,10 +24,9 @@ install:
   - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%"
   - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul
   - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH%
-  - git clone --depth 1 https://github.com/nim-lang/csources
-  - cd csources
-  - build64.bat
-  - cd ..
+  # may need adjustments, untested, see similar examples here:
+  # https://chromium.googlesource.com/external/github.com/swig/swig/+/1e36f51346d95f8b9848e682c2eb986e9cb9b4f4/appveyor.yml
+  - bash -c ". ci/funs.sh && nimBuildCsourcesIfNeeded"
 
 build_script:
   - openssl version

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -69,10 +69,6 @@ jobs:
         fi
       displayName: 'Check whether to skip CI'
 
-    - bash: git clone --depth 1 https://github.com/nim-lang/csources_v1 csources
-      displayName: 'Checkout Nim csources'
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
-
     - task: NodeTool@0
       inputs:
         versionSpec: '12.x'
@@ -163,45 +159,17 @@ jobs:
       condition: and(succeeded(), eq(variables['skipci'], 'false'))
       displayName: 'System information'
 
-    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'Get csources version'
-
-    - task: Cache@2
-      inputs:
-        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
-        path: csources/bin
-      condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'Restore built csources'
-
-    - bash: |
-        set -e
-        . ci/funs.sh
-        ncpu=
-        ext=
-        case '$(Agent.OS)' in
-        'Linux')
-          ncpu=$(nproc)
-          ;;
-        'Darwin')
-          ncpu=$(sysctl -n hw.ncpu)
-          ;;
-        'Windows_NT')
-          ncpu=$NUMBER_OF_PROCESSORS
-          ext=.exe
-          ;;
-        esac
-        [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
-
-        if [[ -x csources/bin/nim$ext ]]; then
-          echo_run echo "Found cached compiler, skipping build"
-        else
-          echo_run make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
-        fi
-
-        echo_run cp csources/bin/nim$ext bin
+    - bash: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu=$(CPU)
       condition: and(succeeded(), eq(variables['skipci'], 'false'))
-      displayName: 'Build 1-stage compiler from csources'
+      displayName: 'Build csourcesAny'
+
+    # this could be revived if performance justifies it (needs a few updates)
+    # - task: Cache@2
+    #   inputs:
+    #     key: 'csourcesAny | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+    #     path: $(nim_csources)
+    #   condition: and(succeeded(), eq(variables['skipci'], 'false'))
+    #   displayName: 'Restore built csourcesAny'
 
     - bash: nim c koch
       condition: and(succeeded(), eq(variables['skipci'], 'false'))

diff --git a/build_all.bat b/build_all.bat
@@ -1,10 +1,13 @@
 @echo off
 rem build development version of the compiler; can be rerun safely
-if not exist csources (
-  git clone --depth 1 https://github.com/nim-lang/csources.git
+rem TODO: call nimBuildCsourcesIfNeeded or auto-generate this file (from a nim script)
+rem to avoid duplication.
+if not exist csources_v1 (
+  git clone --depth 1 https://github.com/nim-lang/csources_v1.git
 )
 if not exist bin\nim.exe (
-  cd csources
+  cd csources_v1
+  git checkout a8a5241f9475099c823cfe1a5e0ca4022ac201ff
   if PROCESSOR_ARCHITECTURE == AMD64 (
     SET ARCH=64
   )

diff --git a/build_all.sh b/build_all.sh
@@ -1,52 +1,16 @@
 #! /bin/sh
 
 # build development version of the compiler; can be rerun safely.
-# arguments can be passed, e.g. `--os freebsd`
+# arguments can be passed, e.g.:
+# CC=gcc ucpu=amd64 uos=darwin
 
 set -u # error on undefined variables
 set -e # exit on first error
 
-echo_run(){
-  echo "$*"
-  "$@"
-}
+. ci/funs.sh
+nimBuildCsourcesIfNeeded "$@"
 
-[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources.git
-
-nim_csources=bin/nim_csources
-
-build_nim_csources_via_script(){
-  echo_run cd csources
-  echo_run sh build.sh "$@"
-}
-
-build_nim_csources(){
-  # avoid changing dir in case of failure
-  (
-    if [ $# -ne 0 ]; then
-      # some args were passed (e.g.: `--cpu i386`), need to call build.sh
-      build_nim_csources_via_script "$@"
-    else
-      # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s)
-      makeX=make
-      unamestr=$(uname)
-      if [ "$unamestr" = 'FreeBSD' ]; then
-        makeX=gmake
-      fi
-      nCPU=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
-      which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script
-    fi
-  )
-  # keep $nim_csources in case needed to investigate bootstrap issues
-  # without having to rebuild from csources
-  echo_run cp bin/nim $nim_csources
-}
-
-[ -f $nim_csources ] || echo_run build_nim_csources "$@"
-
-# Note: if fails, may need to `cd csources && git pull`
+# Note: if fails, may need to update csourcesAny manually
 echo_run bin/nim c --skipUserCfg --skipParentCfg koch
-
 echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg
 echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools.
-