diff --git a/compiler/index.nim b/compiler/index.nim index 2c2a34fb57bbd..696d1e6cafe59 100644 --- a/compiler/index.nim +++ b/compiler/index.nim @@ -1,5 +1,5 @@ ##[ -This module only exists to generate docs for the compiler. +This module only exists to generate internal docs for the compiler. ## links * [main docs](../lib.html) diff --git a/config/nimdoc.cfg b/config/nimdoc.cfg index ed1b346a22f1c..fb7dfe0e5312b 100644 --- a/config/nimdoc.cfg +++ b/config/nimdoc.cfg @@ -139,6 +139,9 @@ doc.body_toc_group = """
  • Compiler docs
  • +
  • + Tools docs +
  • Fusion docs
  • diff --git a/tests/tools/tnimdigger.nim b/tests/tools/tnimdigger.nim new file mode 100644 index 0000000000000..d2d307277b2d9 --- /dev/null +++ b/tests/tools/tnimdigger.nim @@ -0,0 +1,11 @@ +import tools/nimdigger {.all.} + +block: # parseNimGitTag + doAssert parseNimGitTag("v1.4.2") == (1, 4, 2) + doAssertRaises(ValueError): discard parseNimGitTag("v1.4") + doAssertRaises(ValueError): discard parseNimGitTag("v1.4.2a") + doAssertRaises(ValueError): discard parseNimGitTag("av1.4.2") + +block: # isGitNimTag + doAssert isGitNimTag("v1.4.2") + doAssert not isGitNimTag("v1.4.2a") diff --git a/tools/ci_generate.nim b/tools/ci_generate.nim index 52b84f0d89553..add2041f7c7c2 100644 --- a/tools/ci_generate.nim +++ b/tools/ci_generate.nim @@ -5,7 +5,7 @@ duplication that could be removed. ## usage edit this file as needed and then re-generate via: -``` +```bash nim r tools/ci_generate.nim ``` ]## diff --git a/tools/index.nim b/tools/index.nim new file mode 100644 index 0000000000000..a7ac065c8934c --- /dev/null +++ b/tools/index.nim @@ -0,0 +1,15 @@ +##[ +This module only exists to generate internal docs for `tools/`. + +## links +* [main docs](../lib.html) +* [compiler user guide](../nimc.html) +* [Internals of the Nim Compiler](../intern.html) +]## + +#[ +* see also `compiler/index.nim` +* move src/fusion/docutils.nim to std/private/docutils so it can be reused here too +]# + +import nimdigger, ci_generate, nimgrep diff --git a/tools/kochdocs.nim b/tools/kochdocs.nim index f25564fad4d73..bd5e0780deb5b 100644 --- a/tools/kochdocs.nim +++ b/tools/kochdocs.nim @@ -243,6 +243,8 @@ proc buildDocPackages(nimArgs, destPath: string) = # xxx keep in sync with what's in $nim_prs_D/config/nimdoc.cfg, or, rather, # start using nims instead of nimdoc.cfg docProject(destPath/"compiler", extra, "compiler/index.nim") + docProject(destPath/"tools", extra & " --threads", "tools/index.nim") + # --threads needed for nimgrep proc buildDoc(nimArgs, destPath: string) = # call nim for the documentation: diff --git a/tools/nimdigger.nim b/tools/nimdigger.nim new file mode 100644 index 0000000000000..a7b1713ae2420 --- /dev/null +++ b/tools/nimdigger.nim @@ -0,0 +1,324 @@ +##[ +`nimdigger` is a tool to build nim at any revision (including custom branches), taking +care of details such as figuring out automatically the correct csources/csources_v1 revision to use. + +## design goals +* ease of use: 1 liner for running `git bisect` workflows, or to build nim at past revisions +* performance: via caching both csources built binaries, and intermediate nim binaries +* lazyness: build artifacts on demand +* go as far back as possible, currently oldest buildable nim version is v0.12.0~157 + +## examples +build at any revision >= v0.12.0~157 +```bash +$ nim r tools/nimdigger.nim --compileNim --rev:v0.15.2~10 +$ $NIMDIGGER_CACHE/Nim/bin/nim -v +Nim Compiler Version 0.15.2 (2021-05-28) [MacOSX: amd64] [...] +``` + +find a which commit introduced a regression +```bash +$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 \ + --bisectCmd:'bin/nim -v | grep 0.19.0' +66c0f7c3fb214485ca6cfd799af6e50798fcdf6d is the first REGRESSION commit +``` + +find a which commit introduced a bugfix +```bash +$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectBugfix \ + --bisectCmd:'bin/nim -v | grep 0.20.0' +be9c38d2659496f918fb39e129b9b5b055eafd88 is the first BUGFIX commit +``` +Note that this is fast (e.g. 3s) if intermediate nim binaries have already been built/cached in prior runs. + +find an actual regression, e.g. for https://github.com/nim-lang/Nim/issues/16376, +copy this snippet to /tmp/t16376.nim +```nim +type Matrix[T] = object + data: T +proc randMatrix*[T](m, n: int, max: T): Matrix[T] = discard +proc randMatrix*[T](m, n: int, x: Slice[T]): Matrix[T] = discard +template randMatrix*[T](m, n: int): Matrix[T] = randMatrix[T](m, n, T(1.0)) +let B = randMatrix[float32](20, 10) +``` +```bash +$ nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 -- \ + bin/nim c --hints:off --skipparentcfg --skipusercfg /tmp/t16376.nim +fd16875561634e3ef24072631cf85eeead6213f2 is the first REGRESSION commit +``` + +## notes +* this uses `git` (in particular `bisect`), `csources`, `csources_v`, `bash`, `make`/`gmake` +* Unstable API, subject to change +]## + +#[ +## TODO +allow a way to verify that oldnew revisions honor what's implied by bisectBugfix:true|false + +## note +we should give exit code = 125 to commits where nim won't build, to skip over, see also: +https://stackoverflow.com/a/22592593/1426932 (Magic exit statuses) +> anything above 127 makes the bisection fail with something like: +> 125 is magic and makes the run be skipped with git bisect skip. +]# + +import std/[os, osproc, strformat, macros, strutils, tables, algorithm] + +proc `$`(a: ref): string = + if a == nil: "nil" else: $a[] + +template dbg(args: varargs[untyped]): untyped = + # so users can swap in their own better logging until stdlib has one + echo args + +type + DiggerOpt = object ## nimdigger input + rev: string + nimDir: string + compileNim: bool + fetch: bool + csourcesBuildArgs: string + buildAllCsources: bool + verbose: bool + + # bisect cmds + # TODO: allow user to not compile nim, for cases where it's not needed + oldnew: string # eg: v0.20.0~10..v0.20.0 + bisectCmd: string # eg: bin/nim c --hints:off --skipparentcfg --skipusercfg $timn_D/tests/nim/all/t12329.nim 'arg1 bar' 'arg2' + bisectBugfix: bool + CsourcesState = ref object ## represents csources or csources_v1 repos + url: string + dir: string # e.g. /pathto/Nim/csources + rev: string + binDir: string + csourcesBuildArgs: string ## extra args to build csources + revs: seq[string] + fetch: bool + name: string + nimCsourcesExe: string + DiggerState = ref object ## nimdigger internal state + nimDir: string # e.g.: /pathto/Nim + binDir: string # e.g.: $nimDir/bin + rev: string # e.g.: hash obtained from `git rev-parse HEAD` + csourceV0, csourceV1: CsourcesState + +const + csourcesRevs = "v0.9.4 v0.13.0 v0.15.2 v0.16.0 v0.17.0 v0.17.2 v0.18.0 v0.19.0 v0.20.0".split & + "64e34778fa7e114b4afc753c7845dee250584167" + csourcesV1Revs = "a8a5241f9475099c823cfe1a5e0ca4022ac201ff".split + NimDiggerEnv = "NIMDIGGER_CACHE" + ExeExt2 = when ExeExt.len > 0: "." & ExeExt else: "" + +var verbose = false + +proc isSimulate(): bool = + defined(nimDiggerSimulate) + +proc runCmd(cmd: string) = + # TODO: allow `dir` param (or use `runCmdOutput`) + if isSimulate(): + dbg cmd + else: + if verbose: dbg cmd + doAssert execShellCmd(cmd) == 0, cmd + +proc runCmdOutput(cmd: string, dir = ""): string = + if verbose: dbg cmd, dir + let (outp, status) = execCmdEx(cmd, workingDir = dir) + doAssert status == 0, indent(&"status: {status}\ncmd: {cmd}\ndir: {dir}\noutput: {outp}", 2) + result = outp + stripLineEnd(result) + +macro construct(obj: untyped, a: varargs[untyped]): untyped = + ## Generates an object constructor call from a list of fields. + # xxx expose in std/sugar, factor with https://github.com/nim-lang/fusion/pull/32 + runnableExamples: + type Foo = object + a, b: int + doAssert Foo.construct(a,b) == Foo(a: a, b: b) + result = nnkObjConstr.newTree(obj) + for ai in a: result.add nnkExprColonExpr.newTree(ai, ai) + +proc parseKeyVal(a: string): OrderedTable[string, string] = + ## parse bash-like entries of the form key=val + for ai in a.splitLines: + if ai.len == 0 or ai.startsWith "#": continue + let kv = split(ai, "=", maxsplit = 1) + doAssert kv.len == 2, $(ai, kv) + result[kv[0]] = kv[1] + +# xxx move some of these to std/private/gitutils.nim +proc gitClone(url: string, dir: string) = runCmd fmt"git clone -q {url.quoteShell} {dir.quoteShell}" +proc gitResetHard(dir: string, rev: string) = runCmd fmt"git -C {dir.quoteShell} reset --hard {rev}" +proc gitCleanDanger(dir: string, requireConfirmation = true) = + #[ + This is needed to avoid `git bisect` aborting with this error: The following untracked working tree files would be overwritten by checkout. + For example, this would happen in cases like this: + ``` + cd $NIMDIGGER_CACHE/Nim + git checkout abaa42fd8a239ea62ddb39f6f58c3180137d750c + touch testament/testamenthtml.templ + cd - + nim r tools/nimdigger.nim --oldnew:v0.19.0..v0.20.0 --bisectCmd:'bin/nim -v | grep 0.19.0' + ``` + so we handle cleaning untracked files via dry run (-n) followed by -f if user confirms. + ]# + let files = runCmdOutput fmt"git -C {dir.quoteShell} clean -n" + if files.len > 0: + var runClean = true + if requireConfirmation: + echo &"untracked files may prevent `git bisect` from working, `git -C {dir.quoteShell} clean -n` returned:\n{files}" + echo fmt"enter `yes` to proceed with `git clean -f` in: {dir.quoteShell}" + let answer = stdin.readLine() + runClean = answer == "yes" + if runClean: + runCmd fmt"git -C {dir.quoteShell} clean -f" +proc gitFetch(dir: string) = runCmd fmt"git -C {dir.quoteShell} fetch" +proc gitLatestTag(dir: string): string = runCmdOutput("git describe --abbrev=0 HEAD", dir) +proc gitCurrentRev(dir: string): string = runCmdOutput("git rev-parse HEAD", dir) +proc gitCheck(dir: string) = + # checks whether we're in a valid git repo; there may be better ways + discard runCmdOutput("git describe HEAD", dir) + +proc gitIsAncestorOf(dir: string, rev1, rev2: string): bool = + gitCheck(dir) + execShellCmd(fmt"git -C {dir.quoteShell} merge-base --is-ancestor {rev1} {rev2}") == 0 + +import std/strscans + +proc parseNimGitTag(tag: string): (int, int, int) = + if not scanf(tag, "v$i.$i.$i$.", result[0], result[1], result[2]): + raise newException(ValueError, tag) + +proc isGitNimTag(tag: string): bool = + try: + discard parseNimGitTag(tag) + return true + except ValueError: + return false + +proc toNimCsourcesExe(binDir: string, name: string, rev: string): string = + let rev2 = rev.replace(".", "_") + result = binDir / fmt"nim_nimdigger_{name}_{rev2}{ExeExt2}" + +proc buildCsourcesRev(copt: CsourcesState) = + # sync with `_nimBuildCsourcesIfNeeded` + let csourcesExe = toNimCsourcesExe(copt.binDir, copt.name, copt.rev) + if csourcesExe.fileExists: + return + if verbose: dbg copt + if not copt.dir.dirExists: gitClone(copt.url, copt.dir) + if copt.fetch: gitFetch(copt.dir) + gitResetHard(copt.dir, copt.rev) + when defined(bsd): + let make = "gmake" + else: + let make = "make" + let oldNim = copt.binDir / "nim" & ExeExt2 + removeFile(oldNim) # otherwise `make` may incorrectly decide there's notthing to build + let ncpu = countProcessors() + if copt.rev.isGitNimTag and copt.rev.parseNimGitTag < (0,15,2): + # avoids: make: *** No rule to make target `c_code/3_2/compiler_testability.o', needed by `../bin/nim'. Stop. + discard runCmdOutput(fmt"sh build.sh {copt.csourcesBuildArgs}", copt.dir) + else: + discard runCmdOutput(fmt"{make} -j {ncpu + 2} -l {ncpu} {copt.csourcesBuildArgs}", copt.dir) + if isSimulate(): + dbg csourcesExe + else: + copyFile(oldNim, csourcesExe) + +proc buildCsourcesAnyRevs(copt: CsourcesState) = + for rev in copt.revs: + copt.rev = rev + buildCsourcesRev(copt) + +proc toCsourcesRev(rev: string): string = + let ver = rev.parseNimGitTag + if ver >= (1, 0, 0): return csourcesRevs[^1] + for a in csourcesRevs[1 ..< ^1].reversed: + if ver >= a.parseNimGitTag: return a + return csourcesRevs[1] # because v0.9.4 seems broken + +proc getCsourcesState(state: DiggerState): CsourcesState = + let file = state.nimDir/"config/build_config.txt" # for newer nim versions, this file specifies correct csources_v1 to use + if file.fileExists: + let tab = file.readFile.parseKeyVal + result = state.csourceV1 + result.rev = tab["nim_csourcesHash"] + elif gitIsAncestorOf(state.nimDir, "a9b62de", state.rev): # commit that introduced csources_v1 + result = state.csourceV1 + result.rev = csourcesV1Revs[0] + else: + let tag = gitLatestTag(state.nimDir) + result = state.csourceV0 + result.rev = tag.toCsourcesRev + result.nimCsourcesExe = toNimCsourcesExe(state.binDir, result.name, result.rev) + +proc main2(opt: DiggerOpt) = + let state = DiggerState(nimDir: opt.nimDir, rev: opt.rev) + if state.nimDir.len == 0: + let nimdiggerCache = getEnv(NimDiggerEnv, getCacheDir("nimdigger")) + state.nimDir = nimdiggerCache / "Nim" + if verbose: dbg state + let nimDir = state.nimDir + state.binDir = nimDir/"bin" + + if nimDir.dirExists: + doAssert fileExists(nimDir / "lib/system.nim"), fmt"nimDir is not a nim repo: {nimDir}" + else: + createDir nimDir.parentDir + gitClone("https://github.com/nim-lang/Nim", nimDir) + state.csourceV0 = CsourcesState(dir: nimDir/"csources", url: "https://github.com/nim-lang/csources.git", name: "csources", revs: csourcesRevs) + state.csourceV1 = CsourcesState(dir: nimDir/"csources_v1", url: "https://github.com/nim-lang/csources_v1.git", name: "csources_v1", revs: csourcesV1Revs) + for copt in [state.csourceV0, state.csourceV1]: + copt.binDir = state.binDir + copt.fetch = opt.fetch + if opt.buildAllCsources: + buildCsourcesAnyRevs(copt) + + if opt.fetch: gitFetch(nimDir) + if state.rev.len > 0: + gitResetHard(nimDir, state.rev) + state.rev = gitCurrentRev(state.nimDir) + let nimDiggerExe = state.binDir / fmt"nim_nimdigger_nim_{state.rev}{ExeExt2}" + if opt.compileNim: + let isCached = nimDiggerExe.fileExists + echo fmt"digger getting nim: {nimDiggerExe} cached: {isCached}" + if not isCached: + let copt = getCsourcesState(state) + buildCsourcesRev(copt) + discard runCmdOutput(fmt"{copt.nimCsourcesExe} c -o:{nimDiggerExe} -d:release --hints:off --skipUserCfg compiler/nim.nim", nimDir) + copyFile(nimDiggerExe, state.binDir / "nim" & ExeExt2) + + if opt.oldnew.len > 0: + let oldnew2 = opt.oldnew.split("..") + doAssert oldnew2.len == 2, opt.oldnew + let oldrev = oldnew2[0] + let newrev = oldnew2[1] + doAssert oldrev.len > 0 # for regressions, aka goodrev + doAssert newrev.len > 0 # for a regressions, aka badrev + gitCleanDanger(state.nimDir, requireConfirmation = true) + proc bisectStart(old, new: string)= + runCmd(fmt"git -C {state.nimDir.quoteShell} bisect start --term-old {old} --term-new {new} {newrev} {oldrev}") + if opt.bisectBugfix: bisectStart("BROKEN", "BUGFIX") + else: bisectStart("WORKS", "REGRESSION") + let exe = getAppFileName() + var msg = opt.bisectCmd + if opt.bisectBugfix: + msg = fmt"! ({msg})" # negate exit code + let bisectCmd2 = fmt"{exe} --compileNim && ( {msg} )" + runCmd(fmt"git -C {state.nimDir.quoteShell} bisect run bash -c {bisectCmd2.quoteShell}") + +proc main(rev = "", nimDir = "", compileNim = false, fetch = false, oldnew = "", bisectBugfix = false, verbose = false, bisectCmd = "", args: seq[string]) = + nimdigger.verbose = verbose + var bisectCmd = bisectCmd + if bisectCmd.len == 0: + bisectCmd = args.quoteShellCommand + else: + doAssert args.len == 0 + main2(DiggerOpt.construct(rev, nimDir, compileNim, fetch, bisectCmd, oldnew, bisectBugfix)) + +when isMainModule: + import pkg/cligen + dispatch main