From a0941b26fa15d7e69c28e4b90f5f4a1dae4247b5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 Jan 2024 17:54:58 -0500 Subject: [PATCH] Systematize `builtins.fetchTree` docs The renaming task is splatting everything together into markdown lists. The docs are temporily not rendered. --- src/libexpr/primops/fetchTree.cc | 125 ---------------------- src/libfetchers/fetchers.cc | 16 +-- src/libfetchers/fetchers.hh | 25 ++++- src/libfetchers/git.cc | 173 +++++++++++++++++++++++++++---- src/libfetchers/github.cc | 60 +++++++++-- src/libfetchers/indirect.cc | 28 ++++- src/libfetchers/mercurial.cc | 38 +++++-- src/libfetchers/path.cc | 33 ++++-- src/libfetchers/tarball.cc | 103 ++++++++++++++++-- src/nix/main.cc | 41 ++++++-- 10 files changed, 435 insertions(+), 207 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 5061e40fdf2..ede0f5d740f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -238,131 +238,6 @@ static RegisterPrimOp primop_fetchTree({ document) if `fetchTree` was a curried call with the first paramter for `type` or an attribute like `builtins.fetchTree.git`! --> - - `"file"` - - Place a plain file into the Nix store. - This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) - - - `url` (String, required) - - Supported protocols: - - - `https` - - > **Example** - > - > ```nix - > fetchTree { - > type = "file"; - > url = "https://example.com/index.html"; - > } - > ``` - - - `http` - - Insecure HTTP transfer for legacy sources. - - > **Warning** - > - > HTTP performs no encryption or authentication. - > Use a `narHash` known in advance to ensure the output has expected contents. - - - `file` - - A file on the local file system. - - > **Example** - > - > ```nix - > fetchTree { - > type = "file"; - > url = "file:///home/eelco/nix/README.md"; - > } - > ``` - - - `"tarball"` - - Download a tar archive and extract it into the Nix store. - This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball) - - - `url` (String, required) - - > **Example** - > - > ```nix - > fetchTree { - > type = "tarball"; - > url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11"; - > } - > ``` - - - `"git"` - - Fetch a Git tree and copy it to the Nix store. - This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). - - - `url` (String, required) - - The URL formats supported are the same as for Git itself. - - > **Example** - > - > ```nix - > fetchTree { - > type = "git"; - > url = "git@github.com:NixOS/nixpkgs.git"; - > } - > ``` - - > **Note** - > - > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory. - - - `ref` (String, optional) - - A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. - - Default: `"HEAD"` - - - `rev` (String, optional) - - A Git revision; a commit hash. - - Default: the tip of `ref` - - - `shallow` (Bool, optional) - - Make a shallow clone when fetching the Git tree. - - Default: `false` - - - `submodules` (Bool, optional) - - Also fetch submodules if available. - - Default: `false` - - - `allRefs` (Bool, optional) - - If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache. - Otherwise, only the latest commit is fetched if it is not already cached. - - Default: `false` - - - `lastModified` (Integer, optional) - - Unix timestamp of the fetched commit. - - If set, pass through the value to the output attribute set. - Otherwise, generated from the fetched Git tree. - - - `revCount` (Integer, optional) - - Number of revisions in the history of the Git repository before the fetched commit. - - If set, pass through the value to the output attribute set. - Otherwise, generated from the fetched Git tree. - The following input types are still subject to change: - `"path"` diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 483796f0b67..492add83b2d 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -8,8 +8,6 @@ namespace nix::fetchers { -using InputSchemeMap = std::map>; - std::unique_ptr inputSchemes = nullptr; void registerInputScheme(std::shared_ptr && inputScheme) @@ -22,17 +20,9 @@ void registerInputScheme(std::shared_ptr && inputScheme) inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); } -nlohmann::json dumpRegisterInputSchemeInfo() { - using nlohmann::json; - - auto res = json::object(); - - for (auto & [name, scheme] : *inputSchemes) { - auto & r = res[name] = json::object(); - r["allowedAttrs"] = scheme->allowedAttrs(); - } - - return res; +const InputSchemeMap & getAllInputSchemes() +{ + return *inputSchemes; } Input Input::fromURL(const std::string & url, bool requireTree) diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index cd11f9eae93..25cd4311bfe 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -158,14 +158,26 @@ struct InputScheme */ virtual std::string_view schemeName() const = 0; + /** + * Longform description of this scheme, for documentation purposes. + */ + virtual std::string schemeDescription() const = 0; + + // TODO remove these defaults + struct AttributeInfo { + const char * type = "String"; + bool required = true; + const char * doc = ""; + }; + /** * Allowed attributes in an attribute set that is converted to an - * input. + * input, and documentation for each attribute. * - * `type` is not included from this set, because the `type` field is + * `type` is not included from this map, because the `type` field is parsed first to choose which scheme; `type` is always required. */ - virtual StringSet allowedAttrs() const = 0; + virtual std::map allowedAttrs() const = 0; virtual ParsedURL toURL(const Input & input) const; @@ -223,7 +235,12 @@ struct InputScheme void registerInputScheme(std::shared_ptr && fetcher); -nlohmann::json dumpRegisterInputSchemeInfo(); +using InputSchemeMap = std::map>; + +/** + * Use this for docs, not for finding a specific scheme + */ +const InputSchemeMap & getAllInputSchemes(); struct PublicKey { diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 34cfd3f5bea..80499f248bd 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -197,26 +197,163 @@ struct GitInputScheme : InputScheme return "git"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + return stripIndentation(R"( + Fetch a Git tree and copy it to the Nix store. + This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). + )"); + } + + std::map allowedAttrs() const override { return { - "url", - "ref", - "rev", - "shallow", - "submodules", - "exportIgnore", - "lastModified", - "revCount", - "narHash", - "allRefs", - "name", - "dirtyRev", - "dirtyShortRev", - "verifyCommit", - "keytype", - "publicKey", - "publicKeys", + { + "url", + { + .type = "String", + .required = true, + .doc = R"( + The URL formats supported are the same as for Git itself. + + > **Example** + > + > ```nix + > fetchTree { + > type = "git"; + > url = "git@github.com:NixOS/nixpkgs.git"; + > } + > ``` + + > **Note** + > + > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory. + )", + }, + }, + { + "ref", + { + .type = "String", + .required = false, + .doc = R"( + A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. + + Default: `"HEAD"` + )", + }, + }, + { + "rev", + { + .type = "String", + .required = false, + .doc = R"( + A Git revision; a commit hash. + + Default: the tip of `ref` + )", + }, + }, + { + "shallow", + { + .type = "Bool", + .required = false, + .doc = R"( + Make a shallow clone when fetching the Git tree. + + Default: `false` + )", + }, + }, + { + "submodules", + { + .type = "Bool", + .required = false, + .doc = R"( + Also fetch submodules if available. + + Default: `false` + )", + }, + }, + { + "lastModified", + { + .type = "integer", + .required = false, + .doc = R"( + Unix timestamp of the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + )", + }, + }, + { + "revCount", + { + .type = "integer", + .required = false, + .doc = R"( + Number of revisions in the history of the Git repository before the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + )", + }, + }, + { + "narHash", + {}, + }, + { + "allRefs", + { + .type = "Bool", + .required = false, + .doc = R"( + If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache. + Otherwise, only the latest commit is fetched if it is not already cached. + + Default: `false` + )", + }, + }, + { + "name", + {}, + }, + { + "dirtyRev", + {}, + }, + { + "dirtyShortRev", + {}, + }, + { + "exportIgnore", + {}, + }, + { + "verifyCommit", + {}, + }, + { + "keytype", + {}, + }, + { + "publicKey", + {}, + }, + { + "publicKeys", + {}, + }, }; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8100afe4d74..68ba358f962 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -105,17 +105,41 @@ struct GitArchiveInputScheme : InputScheme return input; } - StringSet allowedAttrs() const override + std::map allowedAttrs() const override { return { - "owner", - "repo", - "ref", - "rev", - "narHash", - "lastModified", - "host", - "treeHash", + { + "owner", + {}, + }, + { + "repo", + {}, + }, + { + "ref", + {}, + }, + { + "rev", + {}, + }, + { + "narHash", + {}, + }, + { + "lastModified", + {}, + }, + { + "host", + {}, + }, + { + "treeHash", + {}, + }, }; } @@ -315,6 +339,12 @@ struct GitHubInputScheme : GitArchiveInputScheme { std::string_view schemeName() const override { return "github"; } + std::string schemeDescription() const override + { + // TODO + return ""; + } + std::optional> accessHeaderFromToken(const std::string & token) const override { // Github supports PAT/OAuth2 tokens and HTTP Basic @@ -398,6 +428,12 @@ struct GitLabInputScheme : GitArchiveInputScheme { std::string_view schemeName() const override { return "gitlab"; } + std::string schemeDescription() const override + { + // TODO + return ""; + } + std::optional> accessHeaderFromToken(const std::string & token) const override { // Gitlab supports 4 kinds of authorization, two of which are @@ -467,6 +503,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme { std::string_view schemeName() const override { return "sourcehut"; } + std::string schemeDescription() const override + { + // TODO + return ""; + } + std::optional> accessHeaderFromToken(const std::string & token) const override { // SourceHut supports both PAT and OAuth2. See diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 3f21445e101..ad8d614fc88 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -55,13 +55,31 @@ struct IndirectInputScheme : InputScheme return "indirect"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + // TODO + return ""; + } + + std::map allowedAttrs() const override { return { - "id", - "ref", - "rev", - "narHash", + { + "id", + {}, + }, + { + "ref", + {}, + }, + { + "rev", + {}, + }, + { + "narHash", + {}, + }, }; } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index a2702338ff8..efc0922d860 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -77,15 +77,39 @@ struct MercurialInputScheme : InputScheme return "hg"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + // TODO + return ""; + } + + std::map allowedAttrs() const override { return { - "url", - "ref", - "rev", - "revCount", - "narHash", - "name", + { + "url", + {}, + }, + { + "ref", + {}, + }, + { + "rev", + {}, + }, + { + "revCount", + {}, + }, + { + "narHash", + {}, + }, + { + "name", + {}, + }, }; } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 0af1bad7381..312e568ac1f 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -39,19 +39,40 @@ struct PathInputScheme : InputScheme return "path"; } - StringSet allowedAttrs() const override + std::string schemeDescription() const override + { + // TODO + return ""; + } + + std::map allowedAttrs() const override { return { - "path", + { + "path", + {}, + }, /* Allow the user to pass in "fake" tree info attributes. This is useful for making a pinned tree work the same as the repository from which is exported (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - "rev", - "revCount", - "lastModified", - "narHash", + { + "rev", + {}, + }, + { + "revCount", + {}, + }, + { + "lastModified", + {}, + }, + { + "narHash", + {}, + }, }; } std::optional inputFromAttrs(const Attrs & attrs) const override diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index f08509cb7ff..2cfefc3abe6 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -246,17 +246,85 @@ struct CurlInputScheme : InputScheme return input; } - StringSet allowedAttrs() const override + std::map allowedAttrs() const override { return { - "type", - "url", - "narHash", - "name", - "unpack", - "rev", - "revCount", - "lastModified", + { + "url", + { + .type = "String", + .required = true, + .doc = R"( + Supported protocols: + + - `https` + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "https://example.com/index.html"; + > } + > ``` + + - `http` + + Insecure HTTP transfer for legacy sources. + + > **Warning** + > + > HTTP performs no encryption or authentication. + > Use a `narHash` known in advance to ensure the output has expected contents. + + - `file` + + A file on the local file system. + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "file:///home/eelco/nix/README.md"; + > } + > ``` + + > **Example** + > + > ```nix + > fetchTree { + > type = "tarball"; + > url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11"; + > } + > ``` + )", + }, + }, + { + "narHash", + {}, + }, + { + "name", + {}, + }, + { + "unpack", + {}, + }, + { + "rev", + {}, + }, + { + "revCount", + {}, + }, + { + "lastModified", + {}, + }, }; } @@ -289,6 +357,14 @@ struct FileInputScheme : CurlInputScheme { std::string_view schemeName() const override { return "file"; } + std::string schemeDescription() const override + { + return stripIndentation(R"( + Place a plain file into the Nix store. + This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) + )"); + } + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); @@ -323,6 +399,15 @@ struct TarballInputScheme : CurlInputScheme { std::string_view schemeName() const override { return "tarball"; } + std::string schemeDescription() const override + { + return stripIndentation(R"( + Download a tar archive and extract it into the Nix store. + This has the same underyling implementation as [`builtins.fetchTarball`](@doc + root@/language/builtins.md#builtins-fetchTarball) + )"); + } + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); diff --git a/src/nix/main.cc b/src/nix/main.cc index 5af5f2e4174..b1716552442 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -199,21 +199,40 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs std::string dumpCli() { - auto res = nlohmann::json::object(); + using nlohmann::json; + + auto res = json::object(); res["args"] = toJSON(); - auto stores = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { - auto storeConfig = implem.getConfig(); - auto storeName = storeConfig->name(); - auto & j = stores[storeName]; - j["doc"] = storeConfig->doc(); - j["settings"] = storeConfig->toJSON(); - j["experimentalFeature"] = storeConfig->experimentalFeature(); + { + auto & stores = res["stores"] = json::object(); + for (const auto & implem : *Implementations::registered) { + auto storeConfig = implem.getConfig(); + auto storeName = storeConfig->name(); + auto & j = stores[storeName]; + j["doc"] = storeConfig->doc(); + j["settings"] = storeConfig->toJSON(); + j["experimentalFeature"] = storeConfig->experimentalFeature(); + } } - res["stores"] = std::move(stores); - res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); + + { + auto & fetchers = res["fetchers"] = json::object(); + + for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) { + auto & s = fetchers[schemeName] = json::object(); + s["description"] = scheme->schemeDescription(); + auto & attrs = s["allowedAttrs"] = json::object(); + for (auto & [fieldName, field] : scheme->allowedAttrs()) { + auto & f = attrs[fieldName] = json::object(); + f["type"] = field.type; + f["required"] = field.required; + f["doc"] = stripIndentation(field.doc); + } + } + + }; return res.dump(); }