NixOS/nix
NixOS/nix
1
0
Fork 0
mirror of https://github.com/NixOS/nix synced 2024-10-18 00:16:11 -04:00
Code Activity

Merge f66a76134d into 806a91f7bf

Browse source
This commit is contained in:
John Ericson 2024-10-15 17:57:34 +03:00 committed by GitHub
parent 806a91f7bf f66a76134d
commit 2c0e153c61
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 690 additions and 283 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Makefile.config.in
View file

@ -4,6 +4,7 @@ BOOST_LDFLAGS = @BOOST_LDFLAGS@
BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@
CC = @CC@
CFLAGS = @CFLAGS@
CMARK_LIBS = @CMARK_LIBS@
CXX = @CXX@
CXXFLAGS = @CXXFLAGS@
CXXLTO = @CXXLTO@

13
configure.ac
View file

@ -372,21 +372,24 @@ PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest])
PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9])
# Look for cmark library.
PKG_CHECK_MODULES([CMARK], [libcmark], [CXXFLAGS="$CMARK_CFLAGS $CXXFLAGS"])
# Look for lowdown library.
AC_ARG_ENABLE([markdown], AS_HELP_STRING([--enable-markdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]),
enable_markdown=$enableval, enable_markdown=auto)
AS_CASE(["$enable_markdown"],
AC_ARG_ENABLE([lowdown], AS_HELP_STRING([--enable-lowdown], [Enable Markdown rendering in the Nix binary (requires lowdown) [default=auto]]),
enable_lowdown=$enableval, enable_lowdown=auto)
AS_CASE(["$enable_lowdown"],
[yes | auto], [
PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [
CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"
have_lowdown=1
AC_DEFINE(HAVE_LOWDOWN, 1, [Whether lowdown is available and should be used for Markdown rendering.])
], [
AS_IF([test "x$enable_markdown" == "xyes"], [AC_MSG_ERROR([--enable-markdown was specified, but lowdown was not found.])])
AS_IF([test "x$enable_lowdown" == "xyes"], [AC_MSG_ERROR([--enable-lowdown was specified, but lowdown was not found.])])
])
],
[no], [have_lowdown=],
[AC_MSG_ERROR([bad value "$enable_markdown" for --enable-markdown, must be one of: yes, no, auto])])
[AC_MSG_ERROR([bad value "$enable_lowdown" for --enable-lowdown, must be one of: yes, no, auto])])
# Look for libgit2.

17
flake.lock
View file

@ -1,5 +1,21 @@
{
"nodes": {
"cmark": {
"flake": false,
"locked": {
"lastModified": 1706295831,
"narHash": "sha256-nEI85W8w49ZVr17ycO+7aZvcgA3U2QphNZGrfQl2mSk=",
"owner": "commonmark",
"repo": "cmark",
"rev": "cd37711b8a08da67ba4e21a42614b86dd8def929",
"type": "github"
},
"original": {
"owner": "commonmark",
"repo": "cmark",
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
@ -128,6 +144,7 @@
},
"root": {
"inputs": {
"cmark": "cmark",
"flake-compat": "flake-compat",
"flake-parts": "flake-parts",
"git-hooks-nix": "git-hooks-nix",

2
flake.nix
View file

@ -6,6 +6,8 @@
inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446";
inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; };
# Until https://github.com/commonmark/cmark/pull/524 is released
inputs.cmark = { url = "github:commonmark/cmark"; flake = false; };
# dev tooling
inputs.flake-parts.url = "github:hercules-ci/flake-parts";

8
package.nix
View file

@ -23,6 +23,7 @@
, libseccomp
, libsodium
, man
, cmark
, lowdown
, mdbook
, mdbook-linkcheck
@ -79,7 +80,7 @@
, enableGC ? !stdenv.hostPlatform.isWindows
# Whether to enable Markdown rendering in the Nix binary.
, enableMarkdown ? !stdenv.hostPlatform.isWindows
, enableLowdown ? !stdenv.hostPlatform.isWindows
# Which interactive line editor library to use for Nix's repl.
#
@ -228,7 +229,8 @@ in {
toml11
xz
({ inherit readline editline; }.${readlineFlavor})
] ++ lib.optionals enableMarkdown [
cmark
] ++ lib.optionals enableLowdown [
lowdown
] ++ lib.optionals buildUnitTests [
gtest
@ -255,7 +257,7 @@ in {
(lib.enableFeature doInstallCheck "functional-tests")
(lib.enableFeature enableManual "doc-gen")
(lib.enableFeature enableGC "gc")
(lib.enableFeature enableMarkdown "markdown")
(lib.enableFeature enableLowdown "lowdown")
(lib.enableFeature installUnitTests "install-unit-tests")
(lib.withFeatureAs true "readline-flavor" readlineFlavor)
] ++ lib.optionals (!forDevShell) [

5
packaging/dependencies.nix
View file

@ -166,6 +166,11 @@ scope: {
];
});
cmark = pkgs.cmark.overrideAttrs (_: {
src = inputs.cmark;
version = inputs.lastModifiedDate;
});
busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override {
useMusl = true;
enableStatic = true;

4
packaging/hydra.nix
View file

@ -90,9 +90,9 @@ in
# Toggles some settings for better coverage. Windows needs these
# library combinations, and Debian build Nix with GNU readline too.
buildReadlineNoMarkdown = forAllSystems (system:
buildReadlineNoLowdown = forAllSystems (system:
self.packages.${system}.nix.override {
enableMarkdown = false;
enableLowdown = false;
readlineFlavor = "readline";
}
);

7
src/libexpr/eval.cc
View file

@ -500,7 +500,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
v.mkApp(vPrimOp, vPrimOp);
return addConstant(primOp.name, v, {
.type = nThunk, // FIXME
.doc = primOp.doc,
.doc = primOp.doc.c_str(),
});
}
@ -527,13 +527,14 @@ std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
{
if (v.isPrimOp()) {
auto v2 = &v;
if (auto * doc = v2->primOp()->doc)
auto & doc = v2->primOp()->doc;
if (doc != "")
return Doc {
.pos = {},
.name = v2->primOp()->name,
.arity = v2->primOp()->arity,
.args = v2->primOp()->args,
.doc = doc,
.doc = doc.c_str(),
};
}
if (v.isLambda()) {

2
src/libexpr/eval.hh
View file

@ -88,7 +88,7 @@ struct PrimOp
/**
* Optional free-form documentation about the primop.
*/
const char * doc = nullptr;
const std::string doc = "";
/**
* Add a trace item, `while calling the '<name>' builtin`

2
src/libexpr/local.mk
View file

@ -20,7 +20,7 @@ libexpr_CXXFLAGS += \
libexpr_LIBS = libutil libstore libfetchers
libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS)
libexpr_LDFLAGS += -lboost_context $(THREAD_LDFLAGS) $(CMARK_LIBS)
ifdef HOST_LINUX
libexpr_LDFLAGS += -ldl
endif

292
src/libexpr/primops/fetchTree.cc
View file

@ -8,6 +8,7 @@
#include "registry.hh"
#include "tarball.hh"
#include "url.hh"
#include "cmark-cpp.hh"
#include "value-to-json.hh"
#include "fetch-to-store.hh"
@ -210,223 +211,144 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args,
static RegisterPrimOp primop_fetchTree({
.name = "fetchTree",
.args = {"input"},
.doc = R"(
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
.doc = []() -> std::string {
using namespace cmark;
- the resulting fixed-output [store path](@docroot@/store/store-path.md)
- the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
- backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
// Stores strings referenced by AST. Deallocate after rendering.
std::vector<std::string> textArena;
*input* must be an attribute set with the following attributes:
auto root = node_new(CMARK_NODE_DOCUMENT);
- `type` (String, required)
auto & before = textArena.emplace_back(stripIndentation(R"(
Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with:
One of the [supported source types](#source-types).
This determines other required and allowed input attributes.
- the resulting fixed-output [store path](@docroot@/store/store-path.md)
- the corresponding [NAR](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) hash
- backend-specific metadata (currently not documented). <!-- TODO: document output attributes -->
- `narHash` (String, optional)
*input* must be an attribute set with the following attributes:
The `narHash` parameter can be used to substitute the source of the tree.
It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
- `type` (String, required)
A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
That is, `fetchTree` is idempotent.
One of the [supported source types](#source-types).
This determines other required and allowed input attributes.
Downloads are cached in `$XDG_CACHE_HOME/nix`.
The remote source will be fetched from the network if both are true:
- A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
- `narHash` (String, optional)
> **Note**
>
> [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
The `narHash` parameter can be used to substitute the source of the tree.
It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism.
If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available.
- There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again.
That is, `fetchTree` is idempotent.
## Source types
Downloads are cached in `$XDG_CACHE_HOME/nix`.
The remote source will be fetched from the network if both are true:
- A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store
The following source types and associated input attributes are supported.
<!-- TODO: It would be soooo much more predictable to work with (and
document) if `fetchTree` was a curried call with the first parameter for
`type` or an attribute like `builtins.fetchTree.git`! -->
- `"file"`
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
- `url` (String, required)
Supported protocols:
- `https`
> **Example**
> **Note**
>
> ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
> [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching.
- `http`
- There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl)
Insecure HTTP transfer for legacy sources.
## Source types
> **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
The following source types and associated input attributes are supported.
- `file`
<!-- TODO: It would be soooo much more predictable to work with (and
document) if `fetchTree` was a curried call with the first parameter for
`type` or an attribute like `builtins.fetchTree.git`! -->
)"));
parse_document(*root, before, CMARK_OPT_DEFAULT);
A file on the local file system.
auto & schemes = node_append_child(*root, node_new(CMARK_NODE_LIST));
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
auto & s = node_append_child(schemes, node_new(CMARK_NODE_ITEM));
{
auto & name_p = node_append_child(s, node_new(CMARK_NODE_PARAGRAPH));
auto & name = node_append_child(name_p, node_new(CMARK_NODE_TEXT));
node_set_literal(name, schemeName.data());
}
parse_document(s, scheme->schemeDescription(), CMARK_OPT_DEFAULT);
- `"tarball"`
auto & attrs = node_append_child(s, node_new(CMARK_NODE_LIST));
for (const auto & [attrName, attribute] : scheme->allowedAttrs()) {
auto & a = node_append_child(attrs, node_new(CMARK_NODE_ITEM));
{
auto & name_info = node_append_child(a, node_new(CMARK_NODE_PARAGRAPH));
{
auto & name = node_append_child(name_info, node_new(CMARK_NODE_CODE));
auto & name_t = textArena.emplace_back(attrName);
node_set_literal(name, name_t.c_str());
}
auto & info = node_append_child(name_info, node_new(CMARK_NODE_TEXT));
auto & header = textArena.emplace_back(std::string { }
+ " (" + attribute.type
+ ", " + (attribute.required ? "required" : "optional")
+ ")");
node_set_literal(info, header.c_str());
}
{
auto & doc = textArena.emplace_back(stripIndentation(attribute.doc));
parse_document(a, doc, CMARK_OPT_DEFAULT);
}
}
}
Download a tar archive and extract it into the Nix store.
This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball)
auto & after = textArena.emplace_back(stripIndentation(R"(
The following input types are still subject to change:
- `url` (String, required)
- `"path"`
- `"github"`
- `"gitlab"`
- `"sourcehut"`
- `"mercurial"`
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
- `"git"`
Fetch a Git tree and copy it to the Nix store.
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
- `url` (String, required)
The URL formats supported are the same as for Git itself.
*input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
> **Example**
>
> Fetch a GitHub repository using the attribute set representation:
>
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> builtins.fetchTree {
> type = "github";
> owner = "NixOS";
> repo = "nixpkgs";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> }
> ```
>
> This evaluates to the following attribute set:
>
> ```nix
> {
> lastModified = 1686503798;
> lastModifiedDate = "20230611171638";
> narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
> outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> shortRev = "ae2e6b3";
> }
> ```
> **Note**
> **Example**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
> Fetch the same GitHub repository using the URL-like syntax:
>
> ```nix
> builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
> ```
)"));
parse_document(*root, after, CMARK_OPT_DEFAULT);
- `ref` (String, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
- `rev` (String, optional)
A Git revision; a commit hash.
Default: the tip of `ref`
- `shallow` (Bool, optional)
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
- `submodules` (Bool, optional)
Also fetch submodules if available.
Default: `false`
- `allRefs` (Bool, optional)
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
- `lastModified` (Integer, optional)
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
- `revCount` (Integer, optional)
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
The following input types are still subject to change:
- `"path"`
- `"github"`
- `"gitlab"`
- `"sourcehut"`
- `"mercurial"`
*input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references).
The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled.
> **Example**
>
> Fetch a GitHub repository using the attribute set representation:
>
> ```nix
> builtins.fetchTree {
> type = "github";
> owner = "NixOS";
> repo = "nixpkgs";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> }
> ```
>
> This evaluates to the following attribute set:
>
> ```nix
> {
> lastModified = 1686503798;
> lastModifiedDate = "20230611171638";
> narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc=";
> outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source";
> rev = "ae2e6b3958682513d28f7d633734571fb18285dd";
> shortRev = "ae2e6b3";
> }
> ```
> **Example**
>
> Fetch the same GitHub repository using the URL-like syntax:
>
> ```nix
> builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd"
> ```
)",
auto p = render_commonmark(*root, CMARK_OPT_DEFAULT, 0);
assert(p);
return { &*p };
}(),
.fun = prim_fetchTree,
.experimentalFeature = Xp::FetchTree,
});

16
src/libfetchers/fetchers.cc
View file

@ -8,8 +8,6 @@
namespace nix::fetchers {
using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
std::unique_ptr<InputSchemeMap> inputSchemes = nullptr;
void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
@ -22,17 +20,9 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
inputSchemes->insert_or_assign(schemeName, std::move(inputScheme));
}
nlohmann::json dumpRegisterInputSchemeInfo() {
using nlohmann::json;
auto res = json::object();
for (auto & [name, scheme] : *inputSchemes) {
auto & r = res[name] = json::object();
r["allowedAttrs"] = scheme->allowedAttrs();
}
return res;
const InputSchemeMap & getAllInputSchemes()
{
return *inputSchemes;
}
Input Input::fromURL(

25
src/libfetchers/fetchers.hh
View file

@ -179,14 +179,26 @@ struct InputScheme
*/
virtual std::string_view schemeName() const = 0;
/**
* Longform description of this scheme, for documentation purposes.
*/
virtual std::string schemeDescription() const = 0;
// TODO remove these defaults
struct AttributeInfo {
const char * type = "String";
bool required = true;
const char * doc = "";
};
/**
* Allowed attributes in an attribute set that is converted to an
* input.
* input, and documentation for each attribute.
*
* `type` is not included from this set, because the `type` field is
* `type` is not included from this map, because the `type` field is
parsed first to choose which scheme; `type` is always required.
*/
virtual StringSet allowedAttrs() const = 0;
virtual std::map<std::string, AttributeInfo> allowedAttrs() const = 0;
virtual ParsedURL toURL(const Input & input) const;
@ -244,7 +256,12 @@ struct InputScheme
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
nlohmann::json dumpRegisterInputSchemeInfo();
using InputSchemeMap = std::map<std::string_view, std::shared_ptr<InputScheme>>;
/**
* Use this for docs, not for finding a specific scheme
*/
const InputSchemeMap & getAllInputSchemes();
struct PublicKey
{

179
src/libfetchers/git.cc
View file

@ -201,26 +201,169 @@ struct GitInputScheme : InputScheme
return "git";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
return stripIndentation(R"(
Fetch a Git tree and copy it to the Nix store.
This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit).
)");
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"url",
"ref",
"rev",
"shallow",
"submodules",
"exportIgnore",
"lastModified",
"revCount",
"narHash",
"allRefs",
"name",
"dirtyRev",
"dirtyShortRev",
"verifyCommit",
"keytype",
"publicKey",
"publicKeys",
{
"url",
{
.type = "String",
.required = true,
.doc = R"(
The URL formats supported are the same as for Git itself.
> **Example**
>
> ```nix
> fetchTree {
> type = "git";
> url = "git@github.com:NixOS/nixpkgs.git";
> }
> ```
> **Note**
>
> If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory.
)",
},
},
{
"ref",
{
.type = "String",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name.
Default: `"HEAD"`
)",
},
},
{
"rev",
{
.type = "String",
.required = false,
.doc = R"(
A Git revision; a commit hash.
Default: the tip of `ref`
)",
},
},
{
"shallow",
{
.type = "Bool",
.required = false,
.doc = R"(
Make a shallow clone when fetching the Git tree.
When this is enabled, the options `ref` and `allRefs` have no effect anymore.
Default: `true`
)",
},
},
{
"submodules",
{
.type = "Bool",
.required = false,
.doc = R"(
Also fetch submodules if available.
Default: `false`
)",
},
},
{
"lastModified",
{
.type = "integer",
.required = false,
.doc = R"(
Unix timestamp of the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"revCount",
{
.type = "integer",
.required = false,
.doc = R"(
Number of revisions in the history of the Git repository before the fetched commit.
If set, pass through the value to the output attribute set.
Otherwise, generated from the fetched Git tree.
)",
},
},
{
"narHash",
{},
},
{
"allRefs",
{
.type = "Bool",
.required = false,
.doc = R"(
By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled.
Whether to fetch all references (eg. branches and tags) of the repository.
With this argument being true, it's possible to load a `rev` from *any* `ref`.
(Without setting this option, only `rev`s from the specified `ref` are supported).
Default: `false`
)",
},
},
{
"name",
{},
},
{
"dirtyRev",
{},
},
{
"dirtyShortRev",
{},
},
{
"exportIgnore",
{},
},
{
"verifyCommit",
{},
},
{
"keytype",
{},
},
{
"publicKey",
{},
},
{
"publicKeys",
{},
},
};
}

60
src/libfetchers/github.cc
View file

@ -107,17 +107,41 @@ struct GitArchiveInputScheme : InputScheme
return input;
}
StringSet allowedAttrs() const override
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"owner",
"repo",
"ref",
"rev",
"narHash",
"lastModified",
"host",
"treeHash",
{
"owner",
{},
},
{
"repo",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
{
"lastModified",
{},
},
{
"host",
{},
},
{
"treeHash",
{},
},
};
}
@ -330,6 +354,12 @@ struct GitHubInputScheme : GitArchiveInputScheme
{
std::string_view schemeName() const override { return "github"; }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// Github supports PAT/OAuth2 tokens and HTTP Basic
@ -413,6 +443,12 @@ struct GitLabInputScheme : GitArchiveInputScheme
{
std::string_view schemeName() const override { return "gitlab"; }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// Gitlab supports 4 kinds of authorization, two of which are
@ -488,6 +524,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme
{
std::string_view schemeName() const override { return "sourcehut"; }
std::string schemeDescription() const override
{
// TODO
return "";
}
std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const override
{
// SourceHut supports both PAT and OAuth2. See

28
src/libfetchers/indirect.cc
View file

@ -57,13 +57,31 @@ struct IndirectInputScheme : InputScheme
return "indirect";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"id",
"ref",
"rev",
"narHash",
{
"id",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"narHash",
{},
},
};
}

38
src/libfetchers/mercurial.cc
View file

@ -78,15 +78,39 @@ struct MercurialInputScheme : InputScheme
return "hg";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"url",
"ref",
"rev",
"revCount",
"narHash",
"name",
{
"url",
{},
},
{
"ref",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"narHash",
{},
},
{
"name",
{},
},
};
}

33
src/libfetchers/path.cc
View file

@ -40,19 +40,40 @@ struct PathInputScheme : InputScheme
return "path";
}
StringSet allowedAttrs() const override
std::string schemeDescription() const override
{
// TODO
return "";
}
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"path",
{
"path",
{},
},
/* Allow the user to pass in "fake" tree info
attributes. This is useful for making a pinned tree work
the same as the repository from which is exported (e.g.
path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...).
*/
"rev",
"revCount",
"lastModified",
"narHash",
{
"rev",
{},
},
{
"revCount",
{},
},
{
"lastModified",
{},
},
{
"narHash",
{},
},
};
}

103
src/libfetchers/tarball.cc
View file

@ -278,17 +278,85 @@ struct CurlInputScheme : InputScheme
return input;
}
StringSet allowedAttrs() const override
std::map<std::string, AttributeInfo> allowedAttrs() const override
{
return {
"type",
"url",
"narHash",
"name",
"unpack",
"rev",
"revCount",
"lastModified",
{
"url",
{
.type = "String",
.required = true,
.doc = R"(
Supported protocols:
- `https`
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "https://example.com/index.html";
> }
> ```
- `http`
Insecure HTTP transfer for legacy sources.
> **Warning**
>
> HTTP performs no encryption or authentication.
> Use a `narHash` known in advance to ensure the output has expected contents.
- `file`
A file on the local file system.
> **Example**
>
> ```nix
> fetchTree {
> type = "file";
> url = "file:///home/eelco/nix/README.md";
> }
> ```
> **Example**
>
> ```nix
> fetchTree {
> type = "tarball";
> url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11";
> }
> ```
)",
},
},
{
"narHash",
{},
},
{
"name",
{},
},
{
"unpack",
{},
},
{
"rev",
{},
},
{
"revCount",
{},
},
{
"lastModified",
{},
},
};
}
@ -323,6 +391,14 @@ struct FileInputScheme : CurlInputScheme
{
std::string_view schemeName() const override { return "file"; }
std::string schemeDescription() const override
{
return stripIndentation(R"(
Place a plain file into the Nix store.
This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl)
)");
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
auto parsedUrlScheme = parseUrlScheme(url.scheme);
@ -357,6 +433,15 @@ struct TarballInputScheme : CurlInputScheme
{
std::string_view schemeName() const override { return "tarball"; }
std::string schemeDescription() const override
{
return stripIndentation(R"(
Download a tar archive and extract it into the Nix store.
This has the same underyling implementation as [`builtins.fetchTarball`](@doc
root@/language/builtins.md#builtins-fetchTarball)
)");
}
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
auto parsedUrlScheme = parseUrlScheme(url.scheme);

85
src/libutil/cmark-cpp.hh Normal file
View file

@ -0,0 +1,85 @@
#pragma once
///@file
#include "types.hh"
#include "util.hh"
#include <cmark.h>
namespace nix::cmark {
using Node = struct cmark_node;
using NodeType = cmark_node_type;
using ListType = cmark_list_type;
using Iter = struct cmark_iter;
struct Deleter
{
void operator () (Node * ptr) { cmark_node_free(ptr); }
void operator () (Iter * ptr) { cmark_iter_free(ptr); }
};
template <typename T>
using UniquePtr = std::unique_ptr<Node, Deleter>;
static inline void parse_document(Node & root, std::string_view s, int options)
{
cmark_parser * parser = cmark_parser_new_with_mem_into_root(
options,
cmark_get_default_mem_allocator(),
&root);
cmark_parser_feed(parser, s.data(), s.size());
(void) cmark_parser_finish(parser);
cmark_parser_free(parser);
}
static inline UniquePtr<Node> parse_document(std::string_view s, int options)
{
return UniquePtr<Node> {
cmark_parse_document(s.data(), s.size(), options)
};
}
static inline std::unique_ptr<char, FreeDeleter> render_commonmark(Node & root, int options, int width)
{
return std::unique_ptr<char, FreeDeleter> {
cmark_render_commonmark(&root, options, width)
};
}
static inline std::unique_ptr<char, FreeDeleter> render_xml(Node & root, int options)
{
return std::unique_ptr<char, FreeDeleter> {
cmark_render_xml(&root, options)
};
}
static inline UniquePtr<Node> node_new(NodeType type)
{
return UniquePtr<Node> {
cmark_node_new(type)
};
}
/**
* The parent takes ownership
*/
static inline Node & node_append_child(Node & node, UniquePtr<Node> child)
{
auto status = (bool) cmark_node_append_child(&node, &*child);
assert(status);
return *child.release();
}
static inline bool node_set_literal(Node & node, const char * content)
{
return (bool) cmark_node_set_literal(&node, content);
}
static inline bool node_set_list_type(Node & node, ListType type)
{
return (bool) cmark_node_set_list_type(&node, type);
}
}

10
src/libutil/util.hh
View file

@ -332,6 +332,16 @@ template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
std::string showBytes(uint64_t bytes);
/**
* For using `std::unique` with C functions.
*/
struct FreeDeleter
{
template <typename T>
void operator()(T *p) const { std::free(p); }
};
/**
* Provide an addition operator between strings and string_views
* inexplicably omitted from the standard library.

43
src/nix/main.cc
View file

@ -217,21 +217,40 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs
std::string dumpCli()
{
auto res = nlohmann::json::object();
using nlohmann::json;
auto res = json::object();
res["args"] = toJSON();
auto stores = nlohmann::json::object();
for (auto & implem : *Implementations::registered) {
auto storeConfig = implem.getConfig();
auto storeName = storeConfig->name();
auto & j = stores[storeName];
j["doc"] = storeConfig->doc();
j["settings"] = storeConfig->toJSON();
j["experimentalFeature"] = storeConfig->experimentalFeature();
{
auto & stores = res["stores"] = json::object();
for (const auto & implem : *Implementations::registered) {
auto storeConfig = implem.getConfig();
auto storeName = storeConfig->name();
auto & j = stores[storeName];
j["doc"] = storeConfig->doc();
j["settings"] = storeConfig->toJSON();
j["experimentalFeature"] = storeConfig->experimentalFeature();
}
}
res["stores"] = std::move(stores);
res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo();
{
auto & fetchers = res["fetchers"] = json::object();
for (const auto & [schemeName, scheme] : fetchers::getAllInputSchemes()) {
auto & s = fetchers[schemeName] = json::object();
s["description"] = scheme->schemeDescription();
auto & attrs = s["allowedAttrs"] = json::object();
for (auto & [fieldName, field] : scheme->allowedAttrs()) {
auto & f = attrs[fieldName] = json::object();
f["type"] = field.type;
f["required"] = field.required;
f["doc"] = stripIndentation(field.doc);
}
}
};
return res.dump();
}
@ -439,7 +458,7 @@ void mainWrapped(int argc, char * * argv)
auto b = nlohmann::json::object();
if (!builtin.value->isPrimOp()) continue;
auto primOp = builtin.value->primOp();
if (!primOp->doc) continue;
if (primOp->doc == "") continue;
b["args"] = primOp->args;
b["doc"] = trim(stripIndentation(primOp->doc));
if (primOp->experimentalFeature)