NixOS/nix
NixOS/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2024-10-18 00:16:11 -04:00
Code Activity

Systematize builtins.fetchTree docs

Browse source 
The renaming task is splatting everything together into markdown lists.
The docs are temporily not rendered.
This commit is contained in:
John Ericson 2024-01-09 17:54:58 -05:00
parent f9d7c8021d
commit 448842ad67
10 changed files with 441 additions and 213 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

131
src/libexpr/primops/fetchTree.cc
View file

@ -251,137 +251,6 @@ static RegisterPrimOp primop_fetchTree({
document) if `fetchTree` was a curried call with the first parameter for
`type` or an attribute like `builtins.fetchTree.git`! -->
- `"file"`
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
- `url` (String, required)
Supported protocols:
- `https`
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- `http`
Insecure HTTP transfer for legacy sources.
> **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
- `file`
A file on the local file system.
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
- `"tarball"`
Download a tar archive and extract it into the Nix store.
This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
- `url` (String, required)
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
- `"git"`
Fetch a Git tree and copy it to the Nix store.
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
- `url` (String, required)
The URL formats supported are the same as for Git itself.
> **Example**
>
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
- `ref` (String, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
- `rev` (String, optional)
A Git revision; a commit hash.
Default: the tip of `ref`
- `shallow` (Bool, optional)
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
- `submodules` (Bool, optional)
Also fetch submodules if available.
Default: `false`
- `allRefs` (Bool, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
- `lastModified` (Integer, optional)
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `revCount` (Integer, optional)
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
The following input types are still subject to change:
- `"path"`

16
src/libfetchers/fetchers.cc
View file

@ -8,8 +8,6 @@
namespace nix::fetchers {
using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
std::unique_ptr<InputSchemeMap> inputSchemes = nullptr;
void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
@ -22,17 +20,9 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
inputSchemes->insert_or_assign(schemeName, std::move(inputScheme));
}
nlohmann::json dumpRegisterInputSchemeInfo() {
using nlohmann::json;
auto res = json::object();
for (auto & [name, scheme] : *inputSchemes) {
auto & r = res[name] = json::object();
r["allowedAttrs"] = scheme->allowedAttrs();
}
return res;
const InputSchemeMap & getAllInputSchemes()
{
return *inputSchemes;
}
Input Input::fromURL(

25
src/libfetchers/fetchers.hh
View file

@ -179,14 +179,26 @@ struct InputScheme
*/
virtual std::string_view schemeName() const = 0;
/**
* Longform description of this scheme, for documentation purposes.
*/
virtual std::string schemeDescription() const = 0;
// TODO remove these defaults
struct AttributeInfo {
const char * type = "String";
bool required = true;
const char * doc = "";
};
/**
* Allowed attributes in an attribute set that is converted to an
* input.
* input, and documentation for each attribute.
*
* `type` is not included from this set, because the `type` field is
* `type` is not included from this map, because the `type` field is
parsed first to choose which scheme; `type` is always required.
*/
virtual StringSet allowedAttrs() const = 0;
virtual std::map<std::string, AttributeInfo> allowedAttrs() const = 0;
virtual ParsedURL toURL(const Input & input) const;
@ -244,7 +256,12 @@ struct InputScheme
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
nlohmann::json dumpRegisterInputSchemeInfo();
using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
/**
* Use this for docs, not for finding a specific scheme
*/
const InputSchemeMap & getAllInputSchemes();
struct PublicKey
{

179
src/libfetchers/git.cc
View file

@ -201,26 +201,169 @@ struct GitInputScheme : InputScheme
return "git";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
return stripIndentation(R"(
Fetch a Git tree and copy it to the Nix store.
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
)");
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"url",
"ref",
"rev",
"shallow",
"submodules",
"exportIgnore",
"lastModified",
"revCount",
"narHash",
"allRefs",
"name",
"dirtyRev",
"dirtyShortRev",
"verifyCommit",
"keytype",
"publicKey",
"publicKeys",
{
"url",
{
.type = "String",
.required = true,
.doc = R"(
The URL formats supported are the same as for Git itself.
> **Example**
>
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
)",
},
},
{
"ref",
{
.type = "String",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
)",
},
},
{
"rev",
{
.type = "String",
.required = false,
.doc = R"(
A Git revision; a commit hash.
Default: the tip of `ref`
)",
},
},
{
"shallow",
{
.type = "Bool",
.required = false,
.doc = R"(
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
)",
},
},
{
"submodules",
{
.type = "Bool",
.required = false,
.doc = R"(
Also fetch submodules if available.
Default: `false`
)",
},
},
{
"lastModified",
{
.type = "integer",
.required = false,
.doc = R"(
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"revCount",
{
.type = "integer",
.required = false,
.doc = R"(
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"narHash",
{},
},
{
"allRefs",
{
.type = "Bool",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
)",
},
},
{
"name",
{},
},
{
"dirtyRev",
{},
},
{
"dirtyShortRev",
{},
},
{
"exportIgnore",
{},
},
{
"verifyCommit",
{},
},
{
"keytype",
{},
},
{
"publicKey",
{},
},
{
"publicKeys",
{},
},
};
}

60
src/libfetchers/github.cc
View file

@ -107,17 +107,41 @@ struct GitArchiveInputScheme : InputScheme
return input;
}
StringSet allowedAttrs() const override
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"owner",
"repo",
"ref",
"rev",
"narHash",
"lastModified",
"host",
"treeHash",
{
"owner",
{},
},
{
"repo",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
{
"lastModified",
{},
},
{
"host",
{},
},
{
"treeHash",
{},
},
};
}
@ -330,6 +354,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
{
std::string_view schemeName() const override { return "github"; }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// Github supports PAT/OAuth2 tokens and HTTP Basic
@ -413,6 +443,12 @@ struct GitLabInputScheme : GitArchiveInputScheme
{
std::string_view schemeName() const override { return "gitlab"; }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// Gitlab supports 4 kinds of authorization, two of which are
@ -488,6 +524,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme
{
std::string_view schemeName() const override { return "sourcehut"; }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// SourceHut supports both PAT and OAuth2. See

28
src/libfetchers/indirect.cc
View file

@ -57,13 +57,31 @@ struct IndirectInputScheme : InputScheme
return "indirect";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"id",
"ref",
"rev",
"narHash",
{
"id",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
};
}

38
src/libfetchers/mercurial.cc
View file

@ -78,15 +78,39 @@ struct MercurialInputScheme : InputScheme
return "hg";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"url",
"ref",
"rev",
"revCount",
"narHash",
"name",
{
"url",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"narHash",
{},
},
{
"name",
{},
},
};
}

33
src/libfetchers/path.cc
View file

@ -40,19 +40,40 @@ struct PathInputScheme : InputScheme
return "path";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"path",
{
"path",
{},
},
/* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree work
the same as the repository from which is exported (e.g.
path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
*/
"rev",
"revCount",
"lastModified",
"narHash",
{
"rev",
{},
},
{
"revCount",
{},
},
{
"lastModified",
{},
},
{
"narHash",
{},
},
};
}

103
src/libfetchers/tarball.cc
View file

@ -277,17 +277,85 @@ struct CurlInputScheme : InputScheme
return input;
}
StringSet allowedAttrs() const override
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"type",
"url",
"narHash",
"name",
"unpack",
"rev",
"revCount",
"lastModified",
{
"url",
{
.type = "String",
.required = true,
.doc = R"(
Supported protocols:
- `https`
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- `http`
Insecure HTTP transfer for legacy sources.
> **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
- `file`
A file on the local file system.
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
)",
},
},
{
"narHash",
{},
},
{
"name",
{},
},
{
"unpack",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"lastModified",
{},
},
};
}
@ -322,6 +390,14 @@ struct FileInputScheme : CurlInputScheme
{
std::string_view schemeName() const override { return "file"; }
std::string schemeDescription() const override
{
return stripIndentation(R"(
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
)");
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
auto parsedUrlScheme = parseUrlScheme(url.scheme);
@ -356,6 +432,15 @@ struct TarballInputScheme : CurlInputScheme
{
std::string_view schemeName() const override { return "tarball"; }
std::string schemeDescription() const override
{
return stripIndentation(R"(
Download a tar archive and extract it into the Nix store.
This has the same underyling implementation as [`builtins.fetchTarball`](@doc
root@/language/builtins.md#builtins-fetchTarball)
)");
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
auto parsedUrlScheme = parseUrlScheme(url.scheme);

41
src/nix/main.cc
View file

@ -217,21 +217,40 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
std::string dumpCli()
{
auto res = nlohmann::json::object();
using nlohmann::json;
auto res = json::object();
res["args"] = toJSON();
auto stores = nlohmann::json::object();
for (auto & implem : *Implementations::registered) {
auto storeConfig = implem.getConfig();
auto storeName = storeConfig->name();
auto & j = stores[storeName];
j["doc"] = storeConfig->doc();
j["settings"] = storeConfig->toJSON();
j["experimentalFeature"] = storeConfig->experimentalFeature();
{
auto & stores = res["stores"] = json::object();
for (const auto & implem : *Implementations::registered) {
auto storeConfig = implem.getConfig();
auto storeName = storeConfig->name();
auto & j = stores[storeName];
j["doc"] = storeConfig->doc();
j["settings"] = storeConfig->toJSON();
j["experimentalFeature"] = storeConfig->experimentalFeature();
}
}
res["stores"] = std::move(stores);
res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo();
{
auto & fetchers = res["fetchers"] = json::object();
for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
auto & s = fetchers[schemeName] = json::object();
s["description"] = scheme->schemeDescription();
auto & attrs = s["allowedAttrs"] = json::object();
for (auto & [fieldName, field] : scheme->allowedAttrs()) {
auto & f = attrs[fieldName] = json::object();
f["type"] = field.type;
f["required"] = field.required;
f["doc"] = stripIndentation(field.doc);
}
}
};
return res.dump();
}