use csources_v1 as destination dir

nim-lang · Apr 22, 2021 · 62f04a0 · 62f04a0
1 parent d2cac32
commit 62f04a0
Show file tree

Hide file tree

Showing 12 changed files with 70 additions and 139 deletions.
diff --git a/.github/workflows/ci.yml.disabled b/.github/workflows/ci.yml.disabled
@@ -1,3 +1,5 @@
+# out of date and unused but revivable
+
 name: Continous Integration
 on: [push, pull_request]
 
@@ -30,11 +32,6 @@ jobs:
     steps:
       - name: 'Checkout'
         uses: actions/checkout@v2
-      - name: 'Checkout csources'
-        uses: actions/checkout@v2
-        with:
-          repository: nim-lang/csources
-          path: csources
 
       - name: 'Install node.js 8.x'
         uses: actions/setup-node@v1
@@ -91,24 +88,10 @@ jobs:
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'Build csources'
+      - name: 'Build csourcesAny'
         shell: bash
-        run: |
-          ncpu=
-          case '${{ runner.os }}' in
-          'Linux')
-            ncpu=$(nproc)
-            ;;
-          'macOS')
-            ncpu=$(sysctl -n hw.ncpu)
-            ;;
-          'Windows')
-            ncpu=$NUMBER_OF_PROCESSORS
-            ;;
-          esac
-          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+        run: . ci/funs.sh && nimBuildCsourcesIfNeeded
 
-          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
       - name: 'Build koch'
         shell: bash
         run: nim c koch

diff --git a/.github/workflows/ci_docs.yml b/.github/workflows/ci_docs.yml
@@ -81,7 +81,7 @@ jobs:
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'Build csources'
+      - name: 'Build csourcesAny'
         shell: bash
         run: . ci/funs.sh && nimBuildCsourcesIfNeeded
           # was previously using caching via `actions/cache@v1` but this wasn't

diff --git a/.github/workflows/ci_packages.yml b/.github/workflows/ci_packages.yml
@@ -62,10 +62,9 @@ jobs:
         shell: bash
         run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
-      - name: 'Build csources'
+      - name: 'Build csourcesAny'
         shell: bash
-        run: . ci/funs.sh && nimBuildCsourcesIfNeeded
-          # was: make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+        run: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu='${{ matrix.cpu }}'
 
       - name: 'Build koch'
         shell: bash

diff --git a/.gitignore b/.gitignore
@@ -65,9 +65,8 @@ testament.db
 /tests/**/*.json
 /tests/**/*.js
 
-# factor with D20210421T185609
 /csources
-/csources_v0
+/csources_v1
 
 /dist/
 # /lib/fusion # fusion is now unbundled; `git status` should reveal if it's there so users can act on it

diff --git a/.travis.yml b/.travis.yml
@@ -23,8 +23,7 @@ addons:
     - valgrind
 
 before_script:
-  -  . ci/funs.sh && nimBuildCsourcesIfNeeded
-  # was using: make -C csources -j 2 LD=$CC ucpu=$CPU
+  -  . ci/funs.sh && nimBuildCsourcesIfNeeded LD=$CC ucpu=$CPU
   - export PATH="$PWD/bin${PATH:+:$PATH}"
 
 script:

diff --git a/appveyor.yml.disabled b/appveyor.yml.disabled
@@ -24,10 +24,9 @@ install:
   - IF not exist "%MINGW_ARCHIVE%" appveyor DownloadFile "%MINGW_URL%" -FileName "%MINGW_ARCHIVE%"
   - 7z x -y "%MINGW_ARCHIVE%" -o"%CD%\DIST"> nul
   - SET PATH=%CD%\DIST\%MINGW_DIR%\BIN;%CD%\BIN;%PATH%
-  - git clone --depth 1 https://github.com/nim-lang/csources # xxx use nimBuildCsourcesIfNeeded
-  - cd csources
-  - build64.bat
-  - cd ..
+  # may need adjustments, untested, see similar examples here:
+  # https://chromium.googlesource.com/external/github.com/swig/swig/+/1e36f51346d95f8b9848e682c2eb986e9cb9b4f4/appveyor.yml
+  - bash -c ". ci/funs.sh && nimBuildCsourcesIfNeeded"
 
 build_script:
   - openssl version

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -67,9 +67,6 @@ jobs:
         echo $commitMsg | grep -v '\[skip ci\]' # fails if [skip ci] not in commit msg
       displayName: 'Check whether to skip CI'
 
-    - bash: . ci/funs.sh && nimBuildCsourcesIfNeeded
-      displayName: 'Checkout Nim csources'
-
     - task: NodeTool@0
       inputs:
         versionSpec: '12.x'
@@ -142,6 +139,16 @@ jobs:
       displayName: 'Install dependencies (Windows)'
       condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
 
+    - bash: . ci/funs.sh && nimBuildCsourcesIfNeeded CC=gcc ucpu=$(CPU)
+      displayName: 'Build csourcesAny'
+
+    # this could be revived if performance justifies it (needs a few updates)
+    # - task: Cache@2
+    #   inputs:
+    #     key: 'csourcesAny | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+    #     path: $(nim_csources)
+    #   displayName: 'Restore built csourcesAny'
+
     - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin'
       displayName: 'Add build binaries to PATH'
 
@@ -157,43 +164,6 @@ jobs:
         echo_run make -v
       displayName: 'System information'
 
-    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
-      displayName: 'Get csources version'
-
-    - task: Cache@2
-      inputs:
-        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
-        path: csources/bin
-      displayName: 'Restore built csources'
-
-    - bash: |
-        set -e
-        . ci/funs.sh
-        ncpu=
-        ext=
-        case '$(Agent.OS)' in
-        'Linux')
-          ncpu=$(nproc)
-          ;;
-        'Darwin')
-          ncpu=$(sysctl -n hw.ncpu)
-          ;;
-        'Windows_NT')
-          ncpu=$NUMBER_OF_PROCESSORS
-          ext=.exe
-          ;;
-        esac
-        [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
-
-        if [[ -x csources/bin/nim$ext ]]; then
-          echo_run echo "Found cached compiler, skipping build"
-        else
-          echo_run make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
-        fi
-
-        echo_run cp csources/bin/nim$ext bin
-      displayName: 'Build 1-stage compiler from csources'
-
     - bash: nim c koch
       displayName: 'Build koch'
 

diff --git a/build_all.bat b/build_all.bat
@@ -1,13 +1,13 @@
 @echo off
 rem build development version of the compiler; can be rerun safely
-rem TODO: this is not up to date, call nimBuildCsourcesIfNeeded if possible
-rem or auto-generate this file (from a nim script) to avoid duplication wrt
-rem csources hashes, etc.
-if not exist csources (
-  git clone --depth 1 https://github.com/nim-lang/csources.git
+rem TODO: call nimBuildCsourcesIfNeeded or auto-generate this file (from a nim script)
+rem to avoid duplication.
+if not exist csources_v1 (
+  git clone --depth 1 https://github.com/nim-lang/csources_v1.git
 )
 if not exist bin\nim.exe (
-  cd csources
+  cd csources_v1
+  git checkout a8a5241f9475099c823cfe1a5e0ca4022ac201ff
   if PROCESSOR_ARCHITECTURE == AMD64 (
     SET ARCH=64
   )

diff --git a/build_all.sh b/build_all.sh
@@ -1,16 +1,21 @@
 #! /bin/sh
 
 # build development version of the compiler; can be rerun safely.
-# arguments can be passed, e.g. `--os freebsd`
+# arguments can be passed, e.g.:
+# CC=gcc ucpu=amd64 uos=darwin
+#
+# when `NIM_CSOURCES_USE_BUILD=1` environment is set, it will build csourcesAny with `build.sh`
+# instead, and the arguments are:
+# --cpu amd64 --os darwin (also honors some environment variables, e.g. `CC`)
 
 set -u # error on undefined variables
 set -e # exit on first error
 
 . ci/funs.sh
 
-nimBuildCsourcesIfNeeded
+nimBuildCsourcesIfNeeded "$@"
 
-# Note: if fails, may need to update csources manually
+# Note: if fails, may need to update csourcesAny manually
 echo_run bin/nim c --skipUserCfg --skipParentCfg koch
 
 echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg

diff --git a/ci/funs.sh b/ci/funs.sh
@@ -10,22 +10,12 @@ echo_run () {
 }
 
 nimDefineVars(){
-  # D20210421T185609:here
-  nim_csourcesDir=csources # where we clone the latest csources (currently csources_v1.git)
-  nim_csourcesDir_v0=csources_v0 # where we move or clone the old csources.git
+  nim_csourcesDir=csources_v1 # where we clone
   nim_csourcesUrl=https://github.com/nim-lang/csources_v1.git
   nim_csourcesHash=a8a5241f9475099c823cfe1a5e0ca4022ac201ff
   nim_csources=bin/nim_csources_$nim_csourcesHash
 }
 
-_build_nim_csources_via_script(){
-  # avoid changing dir in case of failure
-  (
-    echo_run cd $nim_csourcesDir
-    echo_run sh build.sh "$@"
-  )
-}
-
 _nimNumCpu(){
   # linux: $(nproc)
   # FreeBSD | macOS: $(sysctl -n hw.ncpu)
@@ -35,12 +25,16 @@ _nimNumCpu(){
 }
 
 _nimBuildCsourcesIfNeeded(){
-  if [ $# -ne 0 ]; then
-    # some args were passed (e.g.: `--cpu i386`), need to call build.sh
-    _build_nim_csources_via_script "$@"
-  else
-    # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s)
-
+  if [ "$NIM_CSOURCES_USE_BUILD" == "1" ]; then
+    # call build.sh; slower and should be less useful
+    # allows passing args (e.g.: `--cpu i386`),
+    # but note that `make` also allows args (e.g. `CC`, `ucpu`).
+    (
+      # `()` avoid changing dir in case of failure
+      echo_run cd $nim_csourcesDir
+      echo_run sh build.sh "$@"
+    )
+  else # use `make`, allowing parallel jobs (5X faster on 16 cores: 10s instead of 50s)
     unamestr=$(uname)
     # uname values: https://en.wikipedia.org/wiki/Uname
     if [ "$unamestr" = 'FreeBSD' ]; then
@@ -51,47 +45,33 @@ _nimBuildCsourcesIfNeeded(){
       makeX=make
     fi
     nCPU=$(_nimNumCpu)
-    which $makeX && echo_run $makeX -C $nim_csourcesDir -j $((nCPU + 2)) -l $nCPU || _build_nim_csources_via_script
+    echo_run which $makeX
+    echo_run $makeX -C $nim_csourcesDir -j $((nCPU + 2)) -l $nCPU "$@"
   fi
   # keep $nim_csources in case needed to investigate bootstrap issues
-  # without having to rebuild from csources
+  # without having to rebuild
   echo_run cp bin/nim $nim_csources
 }
 
-_cloneCsources(){
-  # depth 1: adjust as needed in case useful for `git bisect`
-  echo_run git clone -q --depth 1 $nim_csourcesUrl "$nim_csourcesDir"
-  echo_run git -C "$nim_csourcesDir" checkout $nim_csourcesHash
-}
-
 nimCsourcesHash(){
   nimDefineVars
   echo $nim_csourcesHash
 }
 
 nimBuildCsourcesIfNeeded(){
   # goal: allow cachine each tagged version independently
-  # to avoid rebuilding csources, so that tools
-  # like `git bisect` can grab a cached past version
-  # of `$nim_csources` without rebuilding.
+  # to avoid rebuilding, so that tools like `git bisect`
+  # can grab a cached past version without rebuilding.
   nimDefineVars
   if test -f "$nim_csources"; then
     echo "$nim_csources exists."
   else
     if test -d "$nim_csourcesDir"; then
-      if [ $(git config --get remote.origin.url) = "$nim_csourcesUrl" ]; then
-        echo "$nim_csourcesDir exists with correct url."
-      else
-        if test -d "$nim_csourcesDir_v0"; then
-          echo "$nim_csourcesDir_v0 already exists, bailing out"
-          exit 1
-        else
-          echo_run mv "$nim_csourcesDir" "$nim_csourcesDir_v0"
-          _cloneCsources
-        fi
-      fi
+      echo "$nim_csourcesDir exists."
     else
-      _cloneCsources
+      # depth 1: adjust as needed in case useful for `git bisect`
+      echo_run git clone -q --depth 1 $nim_csourcesUrl "$nim_csourcesDir"
+      echo_run git -C "$nim_csourcesDir" checkout $nim_csourcesHash
     fi
     _nimBuildCsourcesIfNeeded "$@"
   fi

diff --git a/koch.nim b/koch.nim
@@ -283,7 +283,7 @@ template doUseCpp(): bool = getEnv("NIM_COMPILE_TO_CPP", "false") == "true"
 
 proc boot(args: string) =
   ## bootstrapping is a process that involves 3 steps:
-  ## 1. use csources to produce nim1.exe. This nim1.exe is buggy but
+  ## 1. use csourcesAny to produce nim1.exe. This nim1.exe is buggy but
   ## rock solid for building a Nim compiler. It shouldn't be used for anything else.
   ## 2. use nim1.exe to produce nim2.exe. nim2.exe is the one you really need.
   ## 3. We use nim2.exe to build nim3.exe. nim3.exe is equal to nim2.exe except for timestamps.
@@ -315,13 +315,8 @@ proc boot(args: string) =
       let ret = execCmdEx(nimStart & " --version")
       doAssert ret.exitCode == 0
       let version = ret.output.splitLines[0]
-      # remove these when csources get updated
-      template addLib() =
+      if version.startsWith "Nim Compiler Version 0.20.0":
         extraOption.add " --lib:lib" # see https://github.com/nim-lang/Nim/pull/14291
-      if version.startsWith "Nim Compiler Version 0.19.0":
-        extraOption.add " -d:nimBoostrapCsources0_19_0"
-        addLib()
-      elif version.startsWith "Nim Compiler Version 0.20.0": addLib()
 
     # in order to use less memory, we split the build into two steps:
     # --compileOnly produces a $project.json file and does not run GCC/Clang.
@@ -587,23 +582,25 @@ proc runCI(cmd: string) =
     when defined(posix):
       execFold("Run nimsuggest tests", "nim c -r nimsuggest/tester")
 
+const csourcesDir = "csources_v1" # xxx should this be `../csources_v1` ?
+
 proc pushCsources() =
-  if not dirExists("../csources/.git"):
-    quit "[Error] no csources git repository found"
+  if not dirExists(csourcesDir/".git"):
+    quit "[Error] no csourcesAny git repository found"
   csource("-d:danger")
   let cwd = getCurrentDir()
   try:
-    copyDir("build/c_code", "../csources/c_code")
-    copyFile("build/build.sh", "../csources/build.sh")
-    copyFile("build/build.bat", "../csources/build.bat")
-    copyFile("build/build64.bat", "../csources/build64.bat")
-    copyFile("build/makefile", "../csources/makefile")
+    copyDir("build/c_code", csourcesDir/"c_code")
+    copyFile("build/build.sh", csourcesDir/"build.sh")
+    copyFile("build/build.bat", csourcesDir/"build.bat")
+    copyFile("build/build64.bat", csourcesDir/"build64.bat")
+    copyFile("build/makefile", csourcesDir/"makefile")
 
-    setCurrentDir("../csources")
+    setCurrentDir(csourcesDir) # xxx instead use `-C csourcesDir`
     for kind, path in walkDir("c_code"):
       if kind == pcDir:
         exec("git add " & path / "*.c")
-    exec("git commit -am \"updated csources to version " & NimVersion & "\"")
+    exec("git commit -am \"updated csourcesAny to version " & NimVersion & "\"")
     exec("git push origin master")
     exec("git tag -am \"Version $1\" v$1" % NimVersion)
     exec("git push origin v$1" % NimVersion)

diff --git a/lib/std/private/globs.nim b/lib/std/private/globs.nim
@@ -52,5 +52,5 @@ proc nativeToUnixPath*(path: string): string =
 
 when isMainModule:
   import sugar
-  for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", ".csources", "bin"]):
+  for a in walkDirRecFilter(".", follow = a=>a.path.lastPathPart notin ["nimcache", ".git", "csources_v1", "csources", "bin"]):
     echo a