diff --git a/compiler/index.nim b/compiler/index.nim
index 2c2a34fb57bbd..696d1e6cafe59 100644
--- a/compiler/index.nim
+++ b/compiler/index.nim
@@ -1,5 +1,5 @@
##[
-This module only exists to generate docs for the compiler.
+This module only exists to generate internal docs for the compiler.
## links
* [main docs](../lib.html)
diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg
index ed1b346a22f1c..fb7dfe0e5312b 100644
--- a/config/nimdoc.cfg
+++ b/config/nimdoc.cfg
@@ -139,6 +139,9 @@ doc.body_toc_group = """
Compiler docs
+
+ Tools docs
+
Fusion docs
diff --git a/tests/tools/tnimdigger.nim b/tests/tools/tnimdigger.nim
new file mode 100644
index 0000000000000..d2d307277b2d9
--- /dev/null
+++ b/tests/tools/tnimdigger.nim
@@ -0,0 +1,11 @@
+import tools/nimdigger {.all.}
+
+block: # parseNimGitTag
+ doAssert parseNimGitTag("v1.4.2") == (1, 4, 2)
+ doAssertRaises(ValueError): discard parseNimGitTag("v1.4")
+ doAssertRaises(ValueError): discard parseNimGitTag("v1.4.2a")
+ doAssertRaises(ValueError): discard parseNimGitTag("av1.4.2")
+
+block: # isGitNimTag
+ doAssert isGitNimTag("v1.4.2")
+ doAssert not isGitNimTag("v1.4.2a")
diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim
index 52b84f0d89553..add2041f7c7c2 100644
--- a/tools/ci_generate.nim
+++ b/tools/ci_generate.nim
@@ -5,7 +5,7 @@ duplication that could be removed.
## usage
edit this file as needed and then re-generate via:
-```
+```bash
nim r tools/ci_generate.nim
```
]##
diff --git a/tools/index.nim b/tools/index.nim
new file mode 100644
index 0000000000000..a7ac065c8934c
--- /dev/null
+++ b/tools/index.nim
@@ -0,0 +1,15 @@
+##[
+This module only exists to generate internal docs for `tools/`.
+
+## links
+* [main docs](../lib.html)
+* [compiler user guide](../nimc.html)
+* [Internals of the Nim Compiler](../intern.html)
+]##
+
+#[
+* see also `compiler/index.nim`
+* move src/fusion/docutils.nim to std/private/docutils so it can be reused here too
+]#
+
+import nimdigger, ci_generate, nimgrep
diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim
index f25564fad4d73..bd5e0780deb5b 100644
--- a/tools/kochdocs.nim
+++ b/tools/kochdocs.nim
@@ -243,6 +243,8 @@ proc buildDocPackages(nimArgs, destPath: string) =
# xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather,
# start using nims instead of nimdoc.cfg
docProject(destPath/"compiler", extra, "compiler/index.nim")
+ docProject(destPath/"tools", extra & " --threads", "tools/index.nim")
+ # --threads needed for nimgrep
proc buildDoc(nimArgs, destPath: string) =
# call nim for the documentation:
diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim
new file mode 100644
index 0000000000000..a7b1713ae2420
--- /dev/null
+++ b/tools/nimdigger.nim
@@ -0,0 +1,324 @@
+##[
+`nimdigger` is a tool to build nim at any revision (including custom branches), taking
+care of details such as figuring out automatically the correct csources/csources_v1 revision to use.
+
+## design goals
+* ease of use: 1 liner for running `git bisect` workflows, or to build nim at past revisions
+* performance: via caching both csources built binaries, and intermediate nim binaries
+* lazyness: build artifacts on demand
+* go as far back as possible, currently oldest buildable nim version is v0.12.0~157
+
+## examples
+build at any revision >= v0.12.0~157
+```bash
+$ nim r tools/nimdigger.nim --compileNim --rev:v0.15.2~10
+$ $NIMDIGGER_CACHE/Nim/bin/nim -v
+Nim Compiler Version 0.15.2 (2021-05-28) [MacOSX: amd64] [...]
+```
+
+find a which commit introduced a regression
+```bash
+$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 \
+ --bisectCmd:'bin/nim -v | grep 0.19.0'
+66c0f7c3fb214485ca6cfd799af6e50798fcdf6d is the first REGRESSION commit
+```
+
+find a which commit introduced a bugfix
+```bash
+$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectBugfix \
+ --bisectCmd:'bin/nim -v | grep 0.20.0'
+be9c38d2659496f918fb39e129b9b5b055eafd88 is the first BUGFIX commit
+```
+Note that this is fast (e.g. 3s) if intermediate nim binaries have already been built/cached in prior runs.
+
+find an actual regression, e.g. for https://github.com/nim-lang/Nim/issues/16376,
+copy this snippet to /tmp/t16376.nim
+```nim
+type Matrix[T] = object
+ data: T
+proc randMatrix*[T](m, n: int, max: T): Matrix[T] = discard
+proc randMatrix*[T](m, n: int, x: Slice[T]): Matrix[T] = discard
+template randMatrix*[T](m, n: int): Matrix[T] = randMatrix[T](m, n, T(1.0))
+let B = randMatrix[float32](20, 10)
+```
+```bash
+$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 -- \
+ bin/nim c --hints:off --skipparentcfg --skipusercfg /tmp/t16376.nim
+fd16875561634e3ef24072631cf85eeead6213f2 is the first REGRESSION commit
+```
+
+## notes
+* this uses `git` (in particular `bisect`), `csources`, `csources_v`, `bash`, `make`/`gmake`
+* Unstable API, subject to change
+]##
+
+#[
+## TODO
+allow a way to verify that oldnew revisions honor what's implied by bisectBugfix:true|false
+
+## note
+we should give exit code = 125 to commits where nim won't build, to skip over, see also:
+https://stackoverflow.com/a/22592593/1426932 (Magic exit statuses)
+> anything above 127 makes the bisection fail with something like:
+> 125 is magic and makes the run be skipped with git bisect skip.
+]#
+
+import std/[os, osproc, strformat, macros, strutils, tables, algorithm]
+
+proc `$`(a: ref): string =
+ if a == nil: "nil" else: $a[]
+
+template dbg(args: varargs[untyped]): untyped =
+ # so users can swap in their own better logging until stdlib has one
+ echo args
+
+type
+ DiggerOpt = object ## nimdigger input
+ rev: string
+ nimDir: string
+ compileNim: bool
+ fetch: bool
+ csourcesBuildArgs: string
+ buildAllCsources: bool
+ verbose: bool
+
+ # bisect cmds
+ # TODO: allow user to not compile nim, for cases where it's not needed
+ oldnew: string # eg: v0.20.0~10..v0.20.0
+ bisectCmd: string # eg: bin/nim c --hints:off --skipparentcfg --skipusercfg $timn_D/tests/nim/all/t12329.nim 'arg1 bar' 'arg2'
+ bisectBugfix: bool
+ CsourcesState = ref object ## represents csources or csources_v1 repos
+ url: string
+ dir: string # e.g. /pathto/Nim/csources
+ rev: string
+ binDir: string
+ csourcesBuildArgs: string ## extra args to build csources
+ revs: seq[string]
+ fetch: bool
+ name: string
+ nimCsourcesExe: string
+ DiggerState = ref object ## nimdigger internal state
+ nimDir: string # e.g.: /pathto/Nim
+ binDir: string # e.g.: $nimDir/bin
+ rev: string # e.g.: hash obtained from `git rev-parse HEAD`
+ csourceV0, csourceV1: CsourcesState
+
+const
+ csourcesRevs = "v0.9.4 v0.13.0 v0.15.2 v0.16.0 v0.17.0 v0.17.2 v0.18.0 v0.19.0 v0.20.0".split &
+ "64e34778fa7e114b4afc753c7845dee250584167"
+ csourcesV1Revs = "a8a5241f9475099c823cfe1a5e0ca4022ac201ff".split
+ NimDiggerEnv = "NIMDIGGER_CACHE"
+ ExeExt2 = when ExeExt.len > 0: "." & ExeExt else: ""
+
+var verbose = false
+
+proc isSimulate(): bool =
+ defined(nimDiggerSimulate)
+
+proc runCmd(cmd: string) =
+ # TODO: allow `dir` param (or use `runCmdOutput`)
+ if isSimulate():
+ dbg cmd
+ else:
+ if verbose: dbg cmd
+ doAssert execShellCmd(cmd) == 0, cmd
+
+proc runCmdOutput(cmd: string, dir = ""): string =
+ if verbose: dbg cmd, dir
+ let (outp, status) = execCmdEx(cmd, workingDir = dir)
+ doAssert status == 0, indent(&"status: {status}\ncmd: {cmd}\ndir: {dir}\noutput: {outp}", 2)
+ result = outp
+ stripLineEnd(result)
+
+macro construct(obj: untyped, a: varargs[untyped]): untyped =
+ ## Generates an object constructor call from a list of fields.
+ # xxx expose in std/sugar, factor with https://github.com/nim-lang/fusion/pull/32
+ runnableExamples:
+ type Foo = object
+ a, b: int
+ doAssert Foo.construct(a,b) == Foo(a: a, b: b)
+ result = nnkObjConstr.newTree(obj)
+ for ai in a: result.add nnkExprColonExpr.newTree(ai, ai)
+
+proc parseKeyVal(a: string): OrderedTable[string, string] =
+ ## parse bash-like entries of the form key=val
+ for ai in a.splitLines:
+ if ai.len == 0 or ai.startsWith "#": continue
+ let kv = split(ai, "=", maxsplit = 1)
+ doAssert kv.len == 2, $(ai, kv)
+ result[kv[0]] = kv[1]
+
+# xxx move some of these to std/private/gitutils.nim
+proc gitClone(url: string, dir: string) = runCmd fmt"git clone -q {url.quoteShell} {dir.quoteShell}"
+proc gitResetHard(dir: string, rev: string) = runCmd fmt"git -C {dir.quoteShell} reset --hard {rev}"
+proc gitCleanDanger(dir: string, requireConfirmation = true) =
+ #[
+ This is needed to avoid `git bisect` aborting with this error: The following untracked working tree files would be overwritten by checkout.
+ For example, this would happen in cases like this:
+ ```
+ cd $NIMDIGGER_CACHE/Nim
+ git checkout abaa42fd8a239ea62ddb39f6f58c3180137d750c
+ touch testament/testamenthtml.templ
+ cd -
+ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectCmd:'bin/nim -v | grep 0.19.0'
+ ```
+ so we handle cleaning untracked files via dry run (-n) followed by -f if user confirms.
+ ]#
+ let files = runCmdOutput fmt"git -C {dir.quoteShell} clean -n"
+ if files.len > 0:
+ var runClean = true
+ if requireConfirmation:
+ echo &"untracked files may prevent `git bisect` from working, `git -C {dir.quoteShell} clean -n` returned:\n{files}"
+ echo fmt"enter `yes` to proceed with `git clean -f` in: {dir.quoteShell}"
+ let answer = stdin.readLine()
+ runClean = answer == "yes"
+ if runClean:
+ runCmd fmt"git -C {dir.quoteShell} clean -f"
+proc gitFetch(dir: string) = runCmd fmt"git -C {dir.quoteShell} fetch"
+proc gitLatestTag(dir: string): string = runCmdOutput("git describe --abbrev=0 HEAD", dir)
+proc gitCurrentRev(dir: string): string = runCmdOutput("git rev-parse HEAD", dir)
+proc gitCheck(dir: string) =
+ # checks whether we're in a valid git repo; there may be better ways
+ discard runCmdOutput("git describe HEAD", dir)
+
+proc gitIsAncestorOf(dir: string, rev1, rev2: string): bool =
+ gitCheck(dir)
+ execShellCmd(fmt"git -C {dir.quoteShell} merge-base --is-ancestor {rev1} {rev2}") == 0
+
+import std/strscans
+
+proc parseNimGitTag(tag: string): (int, int, int) =
+ if not scanf(tag, "v$i.$i.$i$.", result[0], result[1], result[2]):
+ raise newException(ValueError, tag)
+
+proc isGitNimTag(tag: string): bool =
+ try:
+ discard parseNimGitTag(tag)
+ return true
+ except ValueError:
+ return false
+
+proc toNimCsourcesExe(binDir: string, name: string, rev: string): string =
+ let rev2 = rev.replace(".", "_")
+ result = binDir / fmt"nim_nimdigger_{name}_{rev2}{ExeExt2}"
+
+proc buildCsourcesRev(copt: CsourcesState) =
+ # sync with `_nimBuildCsourcesIfNeeded`
+ let csourcesExe = toNimCsourcesExe(copt.binDir, copt.name, copt.rev)
+ if csourcesExe.fileExists:
+ return
+ if verbose: dbg copt
+ if not copt.dir.dirExists: gitClone(copt.url, copt.dir)
+ if copt.fetch: gitFetch(copt.dir)
+ gitResetHard(copt.dir, copt.rev)
+ when defined(bsd):
+ let make = "gmake"
+ else:
+ let make = "make"
+ let oldNim = copt.binDir / "nim" & ExeExt2
+ removeFile(oldNim) # otherwise `make` may incorrectly decide there's notthing to build
+ let ncpu = countProcessors()
+ if copt.rev.isGitNimTag and copt.rev.parseNimGitTag < (0,15,2):
+ # avoids: make: *** No rule to make target `c_code/3_2/compiler_testability.o', needed by `../bin/nim'. Stop.
+ discard runCmdOutput(fmt"sh build.sh {copt.csourcesBuildArgs}", copt.dir)
+ else:
+ discard runCmdOutput(fmt"{make} -j {ncpu + 2} -l {ncpu} {copt.csourcesBuildArgs}", copt.dir)
+ if isSimulate():
+ dbg csourcesExe
+ else:
+ copyFile(oldNim, csourcesExe)
+
+proc buildCsourcesAnyRevs(copt: CsourcesState) =
+ for rev in copt.revs:
+ copt.rev = rev
+ buildCsourcesRev(copt)
+
+proc toCsourcesRev(rev: string): string =
+ let ver = rev.parseNimGitTag
+ if ver >= (1, 0, 0): return csourcesRevs[^1]
+ for a in csourcesRevs[1 ..< ^1].reversed:
+ if ver >= a.parseNimGitTag: return a
+ return csourcesRevs[1] # because v0.9.4 seems broken
+
+proc getCsourcesState(state: DiggerState): CsourcesState =
+ let file = state.nimDir/"config/build_config.txt" # for newer nim versions, this file specifies correct csources_v1 to use
+ if file.fileExists:
+ let tab = file.readFile.parseKeyVal
+ result = state.csourceV1
+ result.rev = tab["nim_csourcesHash"]
+ elif gitIsAncestorOf(state.nimDir, "a9b62de", state.rev): # commit that introduced csources_v1
+ result = state.csourceV1
+ result.rev = csourcesV1Revs[0]
+ else:
+ let tag = gitLatestTag(state.nimDir)
+ result = state.csourceV0
+ result.rev = tag.toCsourcesRev
+ result.nimCsourcesExe = toNimCsourcesExe(state.binDir, result.name, result.rev)
+
+proc main2(opt: DiggerOpt) =
+ let state = DiggerState(nimDir: opt.nimDir, rev: opt.rev)
+ if state.nimDir.len == 0:
+ let nimdiggerCache = getEnv(NimDiggerEnv, getCacheDir("nimdigger"))
+ state.nimDir = nimdiggerCache / "Nim"
+ if verbose: dbg state
+ let nimDir = state.nimDir
+ state.binDir = nimDir/"bin"
+
+ if nimDir.dirExists:
+ doAssert fileExists(nimDir / "lib/system.nim"), fmt"nimDir is not a nim repo: {nimDir}"
+ else:
+ createDir nimDir.parentDir
+ gitClone("https://github.com/nim-lang/Nim", nimDir)
+ state.csourceV0 = CsourcesState(dir: nimDir/"csources", url: "https://github.com/nim-lang/csources.git", name: "csources", revs: csourcesRevs)
+ state.csourceV1 = CsourcesState(dir: nimDir/"csources_v1", url: "https://github.com/nim-lang/csources_v1.git", name: "csources_v1", revs: csourcesV1Revs)
+ for copt in [state.csourceV0, state.csourceV1]:
+ copt.binDir = state.binDir
+ copt.fetch = opt.fetch
+ if opt.buildAllCsources:
+ buildCsourcesAnyRevs(copt)
+
+ if opt.fetch: gitFetch(nimDir)
+ if state.rev.len > 0:
+ gitResetHard(nimDir, state.rev)
+ state.rev = gitCurrentRev(state.nimDir)
+ let nimDiggerExe = state.binDir / fmt"nim_nimdigger_nim_{state.rev}{ExeExt2}"
+ if opt.compileNim:
+ let isCached = nimDiggerExe.fileExists
+ echo fmt"digger getting nim: {nimDiggerExe} cached: {isCached}"
+ if not isCached:
+ let copt = getCsourcesState(state)
+ buildCsourcesRev(copt)
+ discard runCmdOutput(fmt"{copt.nimCsourcesExe} c -o:{nimDiggerExe} -d:release --hints:off --skipUserCfg compiler/nim.nim", nimDir)
+ copyFile(nimDiggerExe, state.binDir / "nim" & ExeExt2)
+
+ if opt.oldnew.len > 0:
+ let oldnew2 = opt.oldnew.split("..")
+ doAssert oldnew2.len == 2, opt.oldnew
+ let oldrev = oldnew2[0]
+ let newrev = oldnew2[1]
+ doAssert oldrev.len > 0 # for regressions, aka goodrev
+ doAssert newrev.len > 0 # for a regressions, aka badrev
+ gitCleanDanger(state.nimDir, requireConfirmation = true)
+ proc bisectStart(old, new: string)=
+ runCmd(fmt"git -C {state.nimDir.quoteShell} bisect start --term-old {old} --term-new {new} {newrev} {oldrev}")
+ if opt.bisectBugfix: bisectStart("BROKEN", "BUGFIX")
+ else: bisectStart("WORKS", "REGRESSION")
+ let exe = getAppFileName()
+ var msg = opt.bisectCmd
+ if opt.bisectBugfix:
+ msg = fmt"! ({msg})" # negate exit code
+ let bisectCmd2 = fmt"{exe} --compileNim && ( {msg} )"
+ runCmd(fmt"git -C {state.nimDir.quoteShell} bisect run bash -c {bisectCmd2.quoteShell}")
+
+proc main(rev = "", nimDir = "", compileNim = false, fetch = false, oldnew = "", bisectBugfix = false, verbose = false, bisectCmd = "", args: seq[string]) =
+ nimdigger.verbose = verbose
+ var bisectCmd = bisectCmd
+ if bisectCmd.len == 0:
+ bisectCmd = args.quoteShellCommand
+ else:
+ doAssert args.len == 0
+ main2(DiggerOpt.construct(rev, nimDir, compileNim, fetch, bisectCmd, oldnew, bisectBugfix))
+
+when isMainModule:
+ import pkg/cligen
+ dispatch main