From 0feeab755a19acd426cdae6887019f9886b016b4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 2 Jul 2024 15:15:17 -0400 Subject: [PATCH] Move plugins infra to `libnixmain` They are not actually part of the store layer, but instead part of the Nix executable infra (libraries don't need plugins, executables do). This is part of a larger project of moving all of our legacy settings infra to libmain, and having the underlying libraries just have plain configuration structs detached from any settings infra / UI layer. Progress on #5638 --- meson.build | 1 + packaging/components.nix | 1 + packaging/hydra.nix | 1 + src/build-remote/build-remote.cc | 1 + src/libmain-c/.version | 1 + src/libmain-c/build-utils-meson | 1 + src/libmain-c/meson.build | 84 ++++++++++++++++++++++ src/libmain-c/nix_api_main.cc | 16 +++++ src/libmain-c/nix_api_main.h | 40 +++++++++++ src/libmain-c/package.nix | 83 ++++++++++++++++++++++ src/libmain/common-args.cc | 1 + src/libmain/meson.build | 2 + src/libmain/plugin.cc | 117 +++++++++++++++++++++++++++++++ src/libmain/plugin.hh | 12 ++++ src/libstore-c/nix_api_store.cc | 10 --- src/libstore-c/nix_api_store.h | 11 --- src/libstore/globals.cc | 55 --------------- src/libstore/globals.hh | 50 ------------- 18 files changed, 361 insertions(+), 126 deletions(-) create mode 120000 src/libmain-c/.version create mode 120000 src/libmain-c/build-utils-meson create mode 100644 src/libmain-c/meson.build create mode 100644 src/libmain-c/nix_api_main.cc create mode 100644 src/libmain-c/nix_api_main.h create mode 100644 src/libmain-c/package.nix create mode 100644 src/libmain/plugin.cc create mode 100644 src/libmain/plugin.hh diff --git a/meson.build b/meson.build index e6bdc2eac..1c46c5c28 100644 --- a/meson.build +++ b/meson.build @@ -26,6 +26,7 @@ subproject('external-api-docs') subproject('libutil-c') subproject('libstore-c') subproject('libexpr-c') +subproject('libmain-c') # Language Bindings subproject('perl') diff --git a/packaging/components.nix b/packaging/components.nix index 0e369a055..870e9ae61 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -29,6 +29,7 @@ in nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; nix-cmd = callPackage ../src/libcmd/package.nix { }; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 4dfaf9bbf..dbe992476 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -52,6 +52,7 @@ let "nix-flake" "nix-flake-tests" "nix-main" + "nix-main-c" "nix-cmd" "nix-ng" ]; diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 600fc7ee2..a0a404e57 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,6 +11,7 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" diff --git a/src/libmain-c/.version b/src/libmain-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libmain-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libmain-c/build-utils-meson b/src/libmain-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libmain-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build new file mode 100644 index 000000000..1d6b2f959 --- /dev/null +++ b/src/libmain-c/meson.build @@ -0,0 +1,84 @@ +project('nix-main-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-main'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-main.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-main.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-main.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_main.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_main.h', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixmainc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libmain-c/nix_api_main.cc b/src/libmain-c/nix_api_main.cc new file mode 100644 index 000000000..692d53f47 --- /dev/null +++ b/src/libmain-c/nix_api_main.cc @@ -0,0 +1,16 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#include "plugin.hh" + +nix_err nix_init_plugins(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::initPlugins(); + } + NIXC_CATCH_ERRS +} diff --git a/src/libmain-c/nix_api_main.h b/src/libmain-c/nix_api_main.h new file mode 100644 index 000000000..3957b992f --- /dev/null +++ b/src/libmain-c/nix_api_main.h @@ -0,0 +1,40 @@ +#ifndef NIX_API_MAIN_H +#define NIX_API_MAIN_H +/** + * @defgroup libmain libmain + * @brief C bindings for nix libmain + * + * libmain has misc utilities for CLI commands + * @{ + */ +/** @file + * @brief Main entry for the libmain C bindings + */ + +#include "nix_api_util.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +/** + * @brief Loads the plugins specified in Nix's plugin-files setting. + * + * Call this once, after calling your desired init functions and setting + * relevant settings. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization was successful, an error code otherwise. + */ +nix_err nix_init_plugins(nix_c_context * context); + +// cffi end +#ifdef __cplusplus +} +#endif +/** + * @} + */ +#endif // NIX_API_MAIN_H diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix new file mode 100644 index 000000000..478e34a85 --- /dev/null +++ b/src/libmain-c/package.nix @@ -0,0 +1,83 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util-c +, nix-store +, nix-store-c +, nix-main + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-main-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util-c + nix-store + nix-store-c + nix-main + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index a94845ab8..768b2177c 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -5,6 +5,7 @@ #include "logging.hh" #include "loggers.hh" #include "util.hh" +#include "plugin.hh" namespace nix { diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 859ce22f8..fe6133596 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -64,6 +64,7 @@ subdir('build-utils-meson/diagnostics') sources = files( 'common-args.cc', 'loggers.cc', + 'plugin.cc', 'progress-bar.cc', 'shared.cc', ) @@ -79,6 +80,7 @@ include_dirs = [include_directories('.')] headers = [config_h] + files( 'common-args.hh', 'loggers.hh', + 'plugin.hh', 'progress-bar.hh', 'shared.hh', ) diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc new file mode 100644 index 000000000..52b5b60e4 --- /dev/null +++ b/src/libmain/plugin.cc @@ -0,0 +1,117 @@ +#ifndef _WIN32 +# include +#endif + +#include "config-global.hh" +#include "signals.hh" + +namespace nix { + +struct PluginFilesSetting : public BaseSetting +{ + bool pluginsLoaded = false; + + PluginFilesSetting( + Config * options, + const Paths & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, true, name, description, aliases) + { + options->addSetting(this); + } + + Paths parse(const std::string & str) const override; +}; + +Paths PluginFilesSetting::parse(const std::string & str) const +{ + if (pluginsLoaded) + throw UsageError( + "plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); + return BaseSetting::parse(str); +} + +struct PluginSettings : Config +{ + PluginFilesSetting pluginFiles{ + this, + {}, + "plugin-files", + R"( + A list of plugin files to be loaded by Nix. Each of these files will + be dlopened by Nix. If they contain the symbol `nix_plugin_entry()`, + this symbol will be called. Alternatively, they can affect execution + through static initialization. In particular, these plugins may construct + static instances of RegisterPrimOp to add new primops or constants to the + expression language, RegisterStoreImplementation to add new store + implementations, RegisterCommand to add new subcommands to the `nix` + command, and RegisterSetting to add new nix config settings. See the + constructors for those types for more details. + + Warning! These APIs are inherently unstable and may change from + release to release. + + Since these files are loaded into the same address space as Nix + itself, they must be DSOs compatible with the instance of Nix + running at the time (i.e. compiled against the same headers, not + linked to any incompatible libraries). They should not be linked to + any Nix libs directly, as those will be available already at load + time. + + If an entry in the list is a directory, all files in the directory + are loaded as plugins (non-recursively). + )"}; +}; + +static PluginSettings pluginSettings; + +static GlobalConfig::Register rPluginSettings(&pluginSettings); + +void initPlugins() +{ + assert(!pluginSettings.pluginFiles.pluginsLoaded); + for (const auto & pluginFile : pluginSettings.pluginFiles.get()) { + std::vector pluginFiles; + try { + auto ents = std::filesystem::directory_iterator{pluginFile}; + for (const auto & ent : ents) { + checkInterrupt(); + pluginFiles.emplace_back(ent.path()); + } + } catch (std::filesystem::filesystem_error & e) { + if (e.code() != std::errc::not_a_directory) + throw; + pluginFiles.emplace_back(pluginFile); + } + for (const auto & file : pluginFiles) { + checkInterrupt(); + /* handle is purposefully leaked as there may be state in the + DSO needed by the action of the plugin. */ +#ifndef _WIN32 // TODO implement via DLL loading on Windows + void * handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); + + /* Older plugins use a statically initialized object to run their code. + Newer plugins can also export nix_plugin_entry() */ + void (*nix_plugin_entry)() = (void (*)()) dlsym(handle, "nix_plugin_entry"); + if (nix_plugin_entry) + nix_plugin_entry(); +#else + throw Error("could not dynamically open plugin file '%s'", file); +#endif + } + } + + /* Since plugins can add settings, try to re-apply previously + unknown settings. */ + globalConfig.reapplyUnknownSettings(); + globalConfig.warnUnknownSettings(); + + /* Tell the user if they try to set plugin-files after we've already loaded */ + pluginSettings.pluginFiles.pluginsLoaded = true; +} + +} diff --git a/src/libmain/plugin.hh b/src/libmain/plugin.hh new file mode 100644 index 000000000..4221c1b17 --- /dev/null +++ b/src/libmain/plugin.hh @@ -0,0 +1,12 @@ +#pragma once +///@file + +namespace nix { + +/** + * This should be called after settings are initialized, but before + * anything else + */ +void initPlugins(); + +} diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 4fe25c7d4..79841ca49 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -29,16 +29,6 @@ nix_err nix_libstore_init_no_load_config(nix_c_context * context) NIXC_CATCH_ERRS } -nix_err nix_init_plugins(nix_c_context * context) -{ - if (context) - context->last_err_code = NIX_OK; - try { - nix::initPlugins(); - } - NIXC_CATCH_ERRS -} - Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params) { if (context) diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index d3cb8fab8..4b2134457 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -42,17 +42,6 @@ nix_err nix_libstore_init(nix_c_context * context); */ nix_err nix_libstore_init_no_load_config(nix_c_context * context); -/** - * @brief Loads the plugins specified in Nix's plugin-files setting. - * - * Call this once, after calling your desired init functions and setting - * relevant settings. - * - * @param[out] context Optional, stores error information - * @return NIX_OK if the initialization was successful, an error code otherwise. - */ -nix_err nix_init_plugins(nix_c_context * context); - /** * @brief Open a nix store. * diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 4eabf6054..fa4c0ba7f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -15,7 +15,6 @@ #include #ifndef _WIN32 -# include # include #endif @@ -335,60 +334,6 @@ unsigned int MaxBuildJobsSetting::parse(const std::string & str) const } -Paths PluginFilesSetting::parse(const std::string & str) const -{ - if (pluginsLoaded) - throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); - return BaseSetting::parse(str); -} - - -void initPlugins() -{ - assert(!settings.pluginFiles.pluginsLoaded); - for (const auto & pluginFile : settings.pluginFiles.get()) { - std::vector pluginFiles; - try { - auto ents = std::filesystem::directory_iterator{pluginFile}; - for (const auto & ent : ents) { - checkInterrupt(); - pluginFiles.emplace_back(ent.path()); - } - } catch (std::filesystem::filesystem_error & e) { - if (e.code() != std::errc::not_a_directory) - throw; - pluginFiles.emplace_back(pluginFile); - } - for (const auto & file : pluginFiles) { - checkInterrupt(); - /* handle is purposefully leaked as there may be state in the - DSO needed by the action of the plugin. */ -#ifndef _WIN32 // TODO implement via DLL loading on Windows - void *handle = - dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); - if (!handle) - throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); - - /* Older plugins use a statically initialized object to run their code. - Newer plugins can also export nix_plugin_entry() */ - void (*nix_plugin_entry)() = (void (*)())dlsym(handle, "nix_plugin_entry"); - if (nix_plugin_entry) - nix_plugin_entry(); -#else - throw Error("could not dynamically open plugin file '%s'", file); -#endif - } - } - - /* Since plugins can add settings, try to re-apply previously - unknown settings. */ - globalConfig.reapplyUnknownSettings(); - globalConfig.warnUnknownSettings(); - - /* Tell the user if they try to set plugin-files after we've already loaded */ - settings.pluginFiles.pluginsLoaded = true; -} - static void preloadNSS() { /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index dfe25f317..30d7537bd 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -31,23 +31,6 @@ struct MaxBuildJobsSetting : public BaseSetting unsigned int parse(const std::string & str) const override; }; -struct PluginFilesSetting : public BaseSetting -{ - bool pluginsLoaded = false; - - PluginFilesSetting(Config * options, - const Paths & def, - const std::string & name, - const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - { - options->addSetting(this); - } - - Paths parse(const std::string & str) const override; -}; - const uint32_t maxIdsPerBuild = #if __linux__ 1 << 16 @@ -1158,33 +1141,6 @@ public: Setting minFreeCheckInterval{this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - PluginFilesSetting pluginFiles{ - this, {}, "plugin-files", - R"( - A list of plugin files to be loaded by Nix. Each of these files will - be dlopened by Nix. If they contain the symbol `nix_plugin_entry()`, - this symbol will be called. Alternatively, they can affect execution - through static initialization. In particular, these plugins may construct - static instances of RegisterPrimOp to add new primops or constants to the - expression language, RegisterStoreImplementation to add new store - implementations, RegisterCommand to add new subcommands to the `nix` - command, and RegisterSetting to add new nix config settings. See the - constructors for those types for more details. - - Warning! These APIs are inherently unstable and may change from - release to release. - - Since these files are loaded into the same address space as Nix - itself, they must be DSOs compatible with the instance of Nix - running at the time (i.e. compiled against the same headers, not - linked to any incompatible libraries). They should not be linked to - any Nix libs directly, as those will be available already at load - time. - - If an entry in the list is a directory, all files in the directory - are loaded as plugins (non-recursively). - )"}; - Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; @@ -1278,12 +1234,6 @@ public: // FIXME: don't use a global variable. extern Settings settings; -/** - * This should be called after settings are initialized, but before - * anything else - */ -void initPlugins(); - /** * Load the configuration (from `nix.conf`, `NIX_CONFIG`, etc.) into the * given configuration object.