diff --git a/README.md b/README.md new file mode 100644 index 0000000..2e0a668 --- /dev/null +++ b/README.md @@ -0,0 +1,18 @@ +CI + + GitPod + + +# [nix.dev](https://nix.dev) + +An opinionated guide for developers wanting to get things done with the Nix ecosystem. + +## Contributing + +Run `./live` and open a browser at . + +As you make changes your browser should auto-reload within a few seconds. + +For syntax see [RST/Sphinx Cheatsheet](https://sphinx-tutorial.readthedocs.io/cheatsheet/). diff --git a/README.rst b/README.rst deleted file mode 100644 index 3d71bfb..0000000 --- a/README.rst +++ /dev/null @@ -1,23 +0,0 @@ -|CI| |GitPod| - -.. |Netlify Status| image:: https://api.netlify.com/api/v1/badges/269f7467-6afd-49ae-97f2-61a160e93a9a/deploy-status - :target: https://app.netlify.com/sites/nixdev/deploys -.. |CI| image:: https://github.com/nix-dot-dev/nix.dev/workflows/CI/badge.svg -.. |GitPod| image:: https://gitpod.io/button/open-in-gitpod.svg - :target: https://gitpod.io/#https://github.com/nix-dot-dev/nix.dev - - -`nix.dev `_ -============================ - -An opinionated guide for developers wanting to get things done with the Nix ecosystem. - - -Contributing ------------- - -Run ``./live`` and open a browser at http://localhost:5500. - -As you make changes your browser should auto-reload within a few seconds. - -For syntax see `RST/Sphinx Cheatsheet `_. diff --git a/poetry.lock b/poetry.lock index d68a26a..6d296fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -131,33 +131,6 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -[[package]] -name = "importlib-metadata" -version = "4.8.2" -description = "Read metadata from Python packages" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] - -[[package]] -name = "importlib-resources" -version = "3.3.1" -description = "Read resources from Python packages" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" - -[package.extras] -docs = ["sphinx", "rst.linker", "jaraco.packaging"] - [[package]] name = "jinja2" version = "3.0.3" @@ -228,85 +201,6 @@ category = "main" optional = false python-versions = ">=3.6" -[[package]] -name = "mdformat" -version = "0.7.11" -description = "CommonMark compliant Markdown formatter" -category = "main" -optional = false -python-versions = ">=3.7,<4.0" - -[package.dependencies] -importlib-metadata = {version = ">=3.6.0", markers = "python_version < \"3.10\""} -markdown-it-py = ">=1.0.0b2,<3.0.0" -tomli = ">=1.1.0" - -[[package]] -name = "mdformat-deflist" -version = "0.1.1" -description = "An mdformat plugin for markdown-it-deflist." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -mdformat = ">=0.7.0,<0.8.0" -mdit-py-plugins = ">=0.2.7,<0.3.0" - -[package.extras] -dev = ["pre-commit"] -test = ["pytest (>=6.0,<7.0)", "coverage", "pytest-cov"] - -[[package]] -name = "mdformat-frontmatter" -version = "0.4.1" -description = "An mdformat plugin for parsing / ignoring frontmatter." -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -mdformat = ">=0.7.0,<0.8.0" -mdit-py-plugins = "*" -"ruamel.yaml" = "*" - -[package.extras] -dev = ["pre-commit"] -test = ["pytest (>=6.0,<7.0)", "coverage", "pytest-cov"] - -[[package]] -name = "mdformat-myst" -version = "0.1.4" -description = "Mdformat plugin for MyST compatibility." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -mdformat = ">=0.7.0,<0.8.0" -mdformat-frontmatter = ">=0.3.2" -mdformat-tables = ">=0.4.0" -mdit-py-plugins = ">=0.2.7,<0.3.0" -"ruamel.yaml" = ">=0.16.0" - -[package.extras] -dev = ["pre-commit"] -test = ["pytest", "coverage", "pytest-cov"] - -[[package]] -name = "mdformat-tables" -version = "0.4.1" -description = "An mdformat plugin for rendering tables." -category = "main" -optional = false -python-versions = ">=3.6.1" - -[package.dependencies] -mdformat = ">=0.7.5,<0.8.0" - -[package.extras] -test = ["pytest (>=6.0,<7.0)", "coverage", "pytest-cov"] - [[package]] name = "mdit-py-plugins" version = "0.2.8" @@ -455,53 +349,6 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"] use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"] -[[package]] -name = "rst-to-myst" -version = "0.3.2" -description = "Convert RST to MyST-Markdown." -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -click = ">=7.1,<8.0" -docutils = ">=0.15,<0.18" -importlib_resources = {version = ">=3.1,<4.0", markers = "python_version < \"3.9\""} -markdown-it-py = ">=1.0,<2.0" -mdformat = ">=0.7.6,<0.8.0" -mdformat-deflist = ">=0.1.0,<0.2.0" -mdformat-myst = ">=0.1.4,<0.2.0" -pyyaml = "*" -sphinx = {version = ">=3.2,<5", optional = true, markers = "extra == \"sphinx\""} - -[package.extras] -docs = ["myst-parser (>=0.15.0,<0.16.0)", "sphinx-book-theme", "sphinx-click (>=2.6,<3.0)", "sphinx-panels"] -sphinx = ["sphinx (>=3.2,<5)"] -test = ["pytest (>=6.0,<7.0)", "coverage", "pytest-cov", "pytest-regressions"] - -[[package]] -name = "ruamel.yaml" -version = "0.17.17" -description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" -category = "main" -optional = false -python-versions = ">=3" - -[package.dependencies] -"ruamel.yaml.clib" = {version = ">=0.1.2", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.10\""} - -[package.extras] -docs = ["ryd"] -jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] - -[[package]] -name = "ruamel.yaml.clib" -version = "0.2.6" -description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" -category = "main" -optional = false -python-versions = ">=3.5" - [[package]] name = "six" version = "1.16.0" @@ -669,7 +516,7 @@ test = ["pytest"] name = "tomli" version = "1.2.2" description = "A lil' TOML parser" -category = "main" +category = "dev" optional = false python-versions = ">=3.6" @@ -713,22 +560,10 @@ brotli = ["brotlipy (>=0.6.0)"] secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] -[[package]] -name = "zipp" -version = "3.6.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] - [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "b26765d9b9405b2a01982637985b20c290823b420a523e95dcf0b90f4504bdc4" +content-hash = "1980bb86d8c90496548ccbcc5cf3966cf5cd7133dca34d8e733b7d388db4068c" [metadata.files] alabaster = [ @@ -779,14 +614,6 @@ imagesize = [ {file = "imagesize-1.3.0-py2.py3-none-any.whl", hash = "sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c"}, {file = "imagesize-1.3.0.tar.gz", hash = "sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d"}, ] -importlib-metadata = [ - {file = "importlib_metadata-4.8.2-py3-none-any.whl", hash = "sha256:53ccfd5c134223e497627b9815d5030edf77d2ed573922f7a0b8f8bb81a1c100"}, - {file = "importlib_metadata-4.8.2.tar.gz", hash = "sha256:75bdec14c397f528724c1bfd9709d660b33a4d2e77387a3358f20b848bb5e5fb"}, -] -importlib-resources = [ - {file = "importlib_resources-3.3.1-py2.py3-none-any.whl", hash = "sha256:42068585cc5e8c2bf0a17449817401102a5125cbfbb26bb0f43cde1568f6f2df"}, - {file = "importlib_resources-3.3.1.tar.gz", hash = "sha256:0ed250dbd291947d1a298e89f39afcc477d5a6624770503034b72588601bcc05"}, -] jinja2 = [ {file = "Jinja2-3.0.3-py3-none-any.whl", hash = "sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8"}, {file = "Jinja2-3.0.3.tar.gz", hash = "sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7"}, @@ -873,26 +700,6 @@ markupsafe = [ {file = "MarkupSafe-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8"}, {file = "MarkupSafe-2.0.1.tar.gz", hash = "sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a"}, ] -mdformat = [ - {file = "mdformat-0.7.11-py3-none-any.whl", hash = "sha256:5129f4d64b6f3a265c348affe51fd41e5eadbaf95dc0fda182830f9428c1e727"}, - {file = "mdformat-0.7.11.tar.gz", hash = "sha256:0e44b6402a7f80e9a707ef6886e63ea87bea7203627466dbe3898f9d16d51e1b"}, -] -mdformat-deflist = [ - {file = "mdformat_deflist-0.1.1-py3-none-any.whl", hash = "sha256:798399bad9e79084628e06f6b89c390dc6a32555fb74c0bd06d0e4fd5669dd51"}, - {file = "mdformat_deflist-0.1.1.tar.gz", hash = "sha256:187b5e4377422ede075ad7c1ed95a71ac8de2fa9fc12c3bef3c9315b6b7d697c"}, -] -mdformat-frontmatter = [ - {file = "mdformat_frontmatter-0.4.1-py3-none-any.whl", hash = "sha256:9c13f6b7a53de7b401af3c95e66735237545bd174e6619392153b296135ffd49"}, - {file = "mdformat_frontmatter-0.4.1.tar.gz", hash = "sha256:15d3eed1543849d4fe72b1f75b8dffd8b49750c5149186591a1b9617178e2aa2"}, -] -mdformat-myst = [ - {file = "mdformat_myst-0.1.4-py3-none-any.whl", hash = "sha256:d6c08220620adf9a8c002b9fd48d1514e0b23c7306a0797c79c392dbe0e8ad3d"}, - {file = "mdformat_myst-0.1.4.tar.gz", hash = "sha256:15310f428793c5d5ac62aea1b1b7d5c1acb075ed39266655ee76e1bc3434ae50"}, -] -mdformat-tables = [ - {file = "mdformat_tables-0.4.1-py3-none-any.whl", hash = "sha256:981f3dc7350027f78e3fd6a5fe8a16e123eec423af2d140e588d855751501019"}, - {file = "mdformat_tables-0.4.1.tar.gz", hash = "sha256:3024e88e9d29d7b8bb07fd6b59c9d5dcf14d2060122be29e30e72d27b65d7da9"}, -] mdit-py-plugins = [ {file = "mdit-py-plugins-0.2.8.tar.gz", hash = "sha256:5991cef645502e80a5388ec4fc20885d2313d4871e8b8e320ca2de14ac0c015f"}, {file = "mdit_py_plugins-0.2.8-py3-none-any.whl", hash = "sha256:1833bf738e038e35d89cb3a07eb0d227ed647ce7dd357579b65343740c6d249c"}, @@ -972,37 +779,6 @@ requests = [ {file = "requests-2.26.0-py2.py3-none-any.whl", hash = "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24"}, {file = "requests-2.26.0.tar.gz", hash = "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"}, ] -rst-to-myst = [ - {file = "rst-to-myst-0.3.2.tar.gz", hash = "sha256:7aed69c67734efb028d2d5c29b40cdddd3d52118ba3550825077b0ecd6af2bb5"}, - {file = "rst_to_myst-0.3.2-py3-none-any.whl", hash = "sha256:3a1a78a77186ba9d7ba41f7fcc6e5de32e3c98846f4c1f606b8bfa07eed5259c"}, -] -"ruamel.yaml" = [ - {file = "ruamel.yaml-0.17.17-py3-none-any.whl", hash = "sha256:9af3ec5d7f8065582f3aa841305465025d0afd26c5fb54e15b964e11838fc74f"}, - {file = "ruamel.yaml-0.17.17.tar.gz", hash = "sha256:9751de4cbb57d4bfbf8fc394e125ed4a2f170fbff3dc3d78abf50be85924f8be"}, -] -"ruamel.yaml.clib" = [ - {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751"}, - {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7b2927e92feb51d830f531de4ccb11b320255ee95e791022555971c466af4527"}, - {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win32.whl", hash = "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5"}, - {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win_amd64.whl", hash = "sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c"}, - {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d67f273097c368265a7b81e152e07fb90ed395df6e552b9fa858c6d2c9f42502"}, - {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:72a2b8b2ff0a627496aad76f37a652bcef400fd861721744201ef1b45199ab78"}, - {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win32.whl", hash = "sha256:9efef4aab5353387b07f6b22ace0867032b900d8e91674b5d8ea9150db5cae94"}, - {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win_amd64.whl", hash = "sha256:846fc8336443106fe23f9b6d6b8c14a53d38cef9a375149d61f99d78782ea468"}, - {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd"}, - {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:78988ed190206672da0f5d50c61afef8f67daa718d614377dcd5e3ed85ab4a99"}, - {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win32.whl", hash = "sha256:a49e0161897901d1ac9c4a79984b8410f450565bbad64dbfcbf76152743a0cdb"}, - {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win_amd64.whl", hash = "sha256:bf75d28fa071645c529b5474a550a44686821decebdd00e21127ef1fd566eabe"}, - {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a32f8d81ea0c6173ab1b3da956869114cae53ba1e9f72374032e33ba3118c233"}, - {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7f7ecb53ae6848f959db6ae93bdff1740e651809780822270eab111500842a84"}, - {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win32.whl", hash = "sha256:89221ec6d6026f8ae859c09b9718799fea22c0e8da8b766b0b2c9a9ba2db326b"}, - {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win_amd64.whl", hash = "sha256:31ea73e564a7b5fbbe8188ab8b334393e06d997914a4e184975348f204790277"}, - {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed"}, - {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0"}, - {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win32.whl", hash = "sha256:3fb9575a5acd13031c57a62cc7823e5d2ff8bc3835ba4d94b921b4e6ee664104"}, - {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win_amd64.whl", hash = "sha256:825d5fccef6da42f3c8eccd4281af399f21c02b32d98e113dbc631ea6a6ecbc7"}, - {file = "ruamel.yaml.clib-0.2.6.tar.gz", hash = "sha256:4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd"}, -] six = [ {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, @@ -1110,7 +886,3 @@ urllib3 = [ {file = "urllib3-1.26.7-py2.py3-none-any.whl", hash = "sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"}, {file = "urllib3-1.26.7.tar.gz", hash = "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece"}, ] -zipp = [ - {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, - {file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"}, -] diff --git a/pyproject.toml b/pyproject.toml index 102b43a..27209b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,9 +14,6 @@ sphinx = "*" sphinx-book-theme = "*" sphinx-copybutton = "*" -# remove when all .rst files have been converted to .md -rst-to-myst = {version = "*", extras = ["sphinx"]} - [tool.poetry.dev-dependencies] black = "==21.12b0" diff --git a/source/tutorials/ad-hoc-developer-environments.md b/source/tutorials/ad-hoc-developer-environments.md new file mode 100644 index 0000000..47f04b0 --- /dev/null +++ b/source/tutorials/ad-hoc-developer-environments.md @@ -0,0 +1,162 @@ +(ad-hoc-envs)= + +# Ad hoc developer environments + +Assuming you have {ref}`Nix installed `, you can use it +to download packages and create new **shell environments** that use these packages. + +This is a great way to play with Nix tooling and see some of its potential. + +## What is a shell environment? + +A shell environment gives you access to the exact versions of packages specified by Nix. + +A hello world example: + +```shell-session +$ hello +The program ‘hello’ is currently not installed. + +$ nix-shell -p hello + +[nix-shell:~]$ hello +Hello, world! + +[nix-shell:~]$ exit +exit + +$ hello +The program ‘hello’ is currently not installed. +``` + +Here we used the `-p` (packages) flag to specify that we needed the `hello` dependency. Nix found this, downloaded it, and made it available in a shell environment. + +## When are shell environments useful? + +Sometimes you'd like **to use a tool that you do not have installed**. You don't want to +bother installing the software, but you want to use it. + +Sometimes you'd like **to try a tool for a few minutes**. For example, there's a new shiny +tool for writing presentation slides. + +Sometimes you'd like **to give someone else a one-liner to install a set of tools** and you want this to work on all Linux distributions and MacOS. + +Sometimes you'd like **to provide a script that is reproducible**, meaning it will also provide any tools that it depends on. + +## Searching package attribute names + +What can you put in a shell environment?” + +To start, anything that's in the [official package list](https://nixos.org/nixos/packages.html) can become part of the shell environment. + +You can search the package list using: + +```shell-session +$ nix-env -qaP git +gitAndTools.gitFull git-2.25.0 +gitMinimal git-2.25.0 +``` + +The first column is the {term}`attribute name` and the second is the {term}`package name` and its version. + +Once you are comfortable doing this, you can add other things too. +For example, packages of your own or custom shell aliases. + +:::{note} +The query you use for searching packages is a regex, so be aware when it comes to special characters. +::: + +## Ad hoc shell environments + +Once you have the {term}`attribute name` for packages, you can start a shell: + +```shell-session +$ nix-shell -p gitMinimal vim nano joe +these paths will be fetched (44.16 MiB download, 236.37 MiB unpacked): +... +/nix/store/fsn35pc8njnimgn2sn26dlsyxya1wssb-vim-8.2.0013 +/nix/store/wdqjszpr5dlys53d79fym6rv9vyyz29h-joe-4.6 +/nix/store/hx63qkip16i4wifaqgxwrrmxj4az53h1-git-2.25.0 + +[nix-shell:~]$ git --version +git version 2.25.0 + +[nix-shell:~]$ which git +/nix/store/hx63qkip16i4wifaqgxwrrmxj4az53h1-git-2.25.0/bin/git +``` + +Note that even if you had git installed before, once in the shell only the exact version installed by Nix is used. + +Press `CTRL-D` to exit the shell and those packages won't be available anymore. + +## Beyond tooling: Python libraries + +`nix-shell` provides a bunch of other bash variables from packages specified. + +Let's try a quick example using Python and `$PYTHONPATH`: + +```shell-session +$ nix-shell -p 'python38.withPackages (packages: [ packages.django ])' +... + +[nix-shell:~]$ python -c 'import django; print(django)' + +``` + +We create an ad hoc environment with `$PYTHONPATH` set and `python` available, along with the `django` package as well. + +The `-p` argument can handle more than attribute names. You can use a full Nix expression, but we'll cover that in later tutorials. + +## Towards reproducibility + +Even running in these basic Nix shells, if you handed over these commands to another developer, they could get different results. + +These shell environments are **really convenient**, but they are not **perfectly reproducible** in this form. + +What do we mean by reproducible? A fully reproducible example would give exactly the same results no matter **when** or **on what machine** you run the command. +The environment provided would be identical each time. + +Nix also offers fully reproducible environments, which it calls pure environments. + +The following is a fully reproducible example and something that different colleagues with different machines, for example, could share. + +```shell-session +$ nix-shell --pure -p git -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/82b5f87fcc710a99c47c5ffe441589807a8202af.tar.gz + +[nix-shell:~]$ git --version +git version 2.25.4 +``` + +There are two things going on here: + +1. `--pure` flag makes sure that the bash environment from your system is not inherited. That means only the `git` that Nix installed is available inside the shell. + This is useful for one-liners and scripts that run for example within a CI environment. While developing, however, we'd like to have our editor around and a bunch of other things. Therefore we might skip the flag for development environments but use it in build ones. +2. The `-I` flag pins the nixpkgs revision to an **exact git revision**, leaving no doubt which exact version of Nix packages will be used. + +## Reproducible executables + +Finally, we can wrap scripts with Nix to provide a reproducible shell environment that we can commit to a git repository +and share with strangers online. As long as they have Nix installed, they'll be able to execute the script without worrying about manually installing and later uninstalling dependencies at all. + +```python +#! /usr/bin/env nix-shell +#! nix-shell --pure -i python -p "python38.withPackages (ps: [ ps.django ])" +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/82b5f87fcc710a99c47c5ffe441589807a8202af.tar.gz + +import django + +print(django) +``` + +This is essentially the same example as in the previous section, but this time declaratively source controlled! All of the required Nix commands are included as `#!` shebang headers in the scripts itself. + +## Next steps + +We've only covered the bare essentials of Nix here. Once you're comfortable with these examples, take a look at: + +- {ref}`pinning-nixpkgs` to see different ways to import nixpkgs +- {ref}`declarative-reproducible-envs` +- [Garbage Collection](https://nixos.org/manual/nix/stable/package-management/garbage-collection.html)- as when using `nix-shell`, packages are downloaded into `/nix/store`, but never removed. +- See `man nix-shell` for all of the options. +- To quickly setup a Nix project read through + [Getting started Nix template](https://github.com/nix-dot-dev/getting-started-nix-template). diff --git a/source/tutorials/ad-hoc-developer-environments.rst b/source/tutorials/ad-hoc-developer-environments.rst deleted file mode 100644 index 93488b4..0000000 --- a/source/tutorials/ad-hoc-developer-environments.rst +++ /dev/null @@ -1,185 +0,0 @@ -.. _ad-hoc-envs: - -Ad hoc developer environments -============================= - -Assuming you have :ref:`Nix installed `, you can use it -to download packages and create new **shell environments** that use these packages. - -This is a great way to play with Nix tooling and see some of its potential. - - -What is a shell environment? ----------------------------- - -A shell environment gives you access to the exact versions of packages specified by Nix. - -A hello world example: - -.. code:: shell-session - - $ hello - The program ‘hello’ is currently not installed. - - $ nix-shell -p hello - - [nix-shell:~]$ hello - Hello, world! - - [nix-shell:~]$ exit - exit - - $ hello - The program ‘hello’ is currently not installed. - -Here we used the ``-p`` (packages) flag to specify that we needed the ``hello`` dependency. Nix found this, downloaded it, and made it available in a shell environment. - - -When are shell environments useful? ------------------------------------ - -Sometimes you'd like **to use a tool that you do not have installed**. You don't want to -bother installing the software, but you want to use it. - -Sometimes you'd like **to try a tool for a few minutes**. For example, there's a new shiny -tool for writing presentation slides. - -Sometimes you'd like **to give someone else a one-liner to install a set of tools** and you want this to work on all Linux distributions and MacOS. - -Sometimes you'd like **to provide a script that is reproducible**, meaning it will also provide any tools that it depends on. - - -Searching package attribute names ---------------------------------- - -What can you put in a shell environment?” - -To start, anything that's in the `official package list `_ can become part of the shell environment. - -You can search the package list using: - -.. code:: shell-session - - $ nix-env -qaP git - gitAndTools.gitFull git-2.25.0 - gitMinimal git-2.25.0 - - -The first column is the :term:`attribute name` and the second is the :term:`package name` and its version. - -Once you are comfortable doing this, you can add other things too. -For example, packages of your own or custom shell aliases. - -.. note:: - - The query you use for searching packages is a regex, so be aware when it comes to special characters. - - -Ad hoc shell environments -------------------------- - -Once you have the :term:`attribute name` for packages, you can start a shell: - -.. code:: shell-session - - $ nix-shell -p gitMinimal vim nano joe - these paths will be fetched (44.16 MiB download, 236.37 MiB unpacked): - ... - /nix/store/fsn35pc8njnimgn2sn26dlsyxya1wssb-vim-8.2.0013 - /nix/store/wdqjszpr5dlys53d79fym6rv9vyyz29h-joe-4.6 - /nix/store/hx63qkip16i4wifaqgxwrrmxj4az53h1-git-2.25.0 - - [nix-shell:~]$ git --version - git version 2.25.0 - - [nix-shell:~]$ which git - /nix/store/hx63qkip16i4wifaqgxwrrmxj4az53h1-git-2.25.0/bin/git - -Note that even if you had git installed before, once in the shell only the exact version installed by Nix is used. - -Press ``CTRL-D`` to exit the shell and those packages won't be available anymore. - - -Beyond tooling: Python libraries --------------------------------- - -``nix-shell`` provides a bunch of other bash variables from packages specified. - -Let's try a quick example using Python and ``$PYTHONPATH``: - -.. code:: shell-session - - $ nix-shell -p 'python38.withPackages (packages: [ packages.django ])' - ... - - [nix-shell:~]$ python -c 'import django; print(django)' - - -We create an ad hoc environment with ``$PYTHONPATH`` set and ``python`` available, along with the ``django`` package as well. - -The ``-p`` argument can handle more than attribute names. You can use a full Nix expression, but we'll cover that in later tutorials. - - -Towards reproducibility ------------------------ - -Even running in these basic Nix shells, if you handed over these commands to another developer, they could get different results. - -These shell environments are **really convenient**, but they are not **perfectly reproducible** in this form. - -What do we mean by reproducible? A fully reproducible example would give exactly the same results no matter **when** or **on what machine** you run the command. -The environment provided would be identical each time. - -Nix also offers fully reproducible environments, which it calls pure environments. - -The following is a fully reproducible example and something that different colleagues with different machines, for example, could share. - -.. code:: shell-session - - $ nix-shell --pure -p git -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/82b5f87fcc710a99c47c5ffe441589807a8202af.tar.gz - - [nix-shell:~]$ git --version - git version 2.25.4 - -There are two things going on here: - -1. ``--pure`` flag makes sure that the bash environment from your system is not inherited. That means only the ``git`` that Nix installed is available inside the shell. - This is useful for one-liners and scripts that run for example within a CI environment. While developing, however, we'd like to have our editor around and a bunch of other things. Therefore we might skip the flag for development environments but use it in build ones. - -2. The ``-I`` flag pins the nixpkgs revision to an **exact git revision**, leaving no doubt which exact version of Nix packages will be used. - - -Reproducible executables ------------------------- - -Finally, we can wrap scripts with Nix to provide a reproducible shell environment that we can commit to a git repository -and share with strangers online. As long as they have Nix installed, they'll be able to execute the script without worrying about manually installing and later uninstalling dependencies at all. - -.. code:: python - - #! /usr/bin/env nix-shell - #! nix-shell --pure -i python -p "python38.withPackages (ps: [ ps.django ])" - #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/82b5f87fcc710a99c47c5ffe441589807a8202af.tar.gz - - import django - - print(django) - -This is essentially the same example as in the previous section, but this time declaratively source controlled! All of the required Nix commands are included as ``#!`` shebang headers in the scripts itself. - - -Next steps ----------- - -We've only covered the bare essentials of Nix here. Once you're comfortable with these examples, take a look at: - -- :ref:`pinning-nixpkgs` to see different ways to import nixpkgs - -- :ref:`declarative-reproducible-envs` - -- `Garbage Collection `_- as when using `nix-shell`, packages are downloaded into `/nix/store`, but never removed. - -- See ``man nix-shell`` for all of the options. - -- To quickly setup a Nix project read through - `Getting started Nix template `_. diff --git a/source/tutorials/building-and-running-docker-images.md b/source/tutorials/building-and-running-docker-images.md new file mode 100644 index 0000000..3f6c9fd --- /dev/null +++ b/source/tutorials/building-and-running-docker-images.md @@ -0,0 +1,136 @@ +--- +html_meta: + "description lang=en": "Building and running Docker images" + "keywords": "Docker, containers, Nix, reproducible, build, tutorial" +--- + +# Building and running Docker images + +[Docker](https://www.docker.com/) is a set of tools and services used to +build, manage and deploy containers. + +As many cloud platforms offer Docker-based +container hosting services, creating Docker containers for a given service is a +common task when building reproducible software. In this tutorial, you will +learn how to build Docker containers using Nix. + +## Prerequisites + +We assume you have both Nix and [Docker installed](https://docs.docker.com/get-docker/). Docker is available in +`nixpkgs`, which is the preferred way to install it on NixOS. However, you can +also use the native Docker installation of your OS, if you are on another Linux +distribution or MacOS. + +## Build your first container + +[Nixpkgs](https://github.com/NixOS/nixpkgs) provides `dockerTools` to create +Docker images: + +```nix +{ pkgs ? import { } +, pkgsLinux ? import { system = "x86_64-linux"; } +}: + +pkgs.dockerTools.buildImage { + name = "hello-docker"; + config = { + Cmd = [ "${pkgsLinux.hello}/bin/hello" ]; + }; +} +``` + +:::{note} +If you're running **macOS** or any platform other than `x86_64-linux`, you'll need to either: + +- [Set up a remote builder](https://github.com/nix-dot-dev/nix.dev/issues/157) to build on Linux +- {ref}`Cross compile to Linux ` by replacing `pkgsLinux.hello` with `pkgs.pkgsCross.musl64.hello` +::: + +We call the `dockerTools.buildImage` and pass in some parameters: + +- a `name` for our image +- the `config` including the command `Cmd` that should be run inside the container + once the image is started. Here we reference the GNU hello package from `nixpkgs` and run + its executable in the container. + +Save this in `hello-docker.nix` and build it: + +```shell-session +$ nix-build hello-docker.nix +these derivations will be built: + /nix/store/qpgdp0qpd8ddi1ld72w02zkmm7n87b92-docker-layer-hello-docker.drv + /nix/store/m4xyfyviwbi38sfplq3xx54j6k7mccfb-runtime-deps.drv + /nix/store/v0bvy9qxa79izc7s03fhpq5nqs2h4sr5-docker-image-hello-docker.tar.gz.drv +warning: unknown setting 'experimental-features' +building '/nix/store/qpgdp0qpd8ddi1ld72w02zkmm7n87b92-docker-layer-hello-docker.drv'... +No contents to add to layer. +Packing layer... +Computing layer checksum... +Finished building layer 'hello-docker' +building '/nix/store/m4xyfyviwbi38sfplq3xx54j6k7mccfb-runtime-deps.drv'... +building '/nix/store/v0bvy9qxa79izc7s03fhpq5nqs2h4sr5-docker-image-hello-docker.tar.gz.drv'... +Adding layer... +tar: Removing leading `/' from member names +Adding meta... +Cooking the image... +Finished. +/nix/store/y74sb4nrhxr975xs7h83izgm8z75x5fc-docker-image-hello-docker.tar.gz +``` + +The image tag (`y74sb4nrhxr975xs7h83izgm8z75x5fc`) refers to the Nix build hash +and makes sure that the Docker image corresponds to our Nix build. The store +path in the last line of the output references the Docker image. + +## Run the container + +To work with the container, load this image into +Docker's image registry from the default `result` symlink created by nix-build: + +```shell-session +$ docker load < result +Loaded image: hello-docker:y74sb4nrhxr975xs7h83izgm8z75x5fc +``` + +You can also use the store path to load the image in order to avoid depending on the presence of +`result` + +```shell-session +$ docker load < /nix/store/y74sb4nrhxr975xs7h83izgm8z75x5fc-docker-image-hello-docker.tar.gz +Loaded image: hello-docker:y74sb4nrhxr975xs7h83izgm8z75x5fc +``` + +Even more conveniently, you can do everything in one command. The advantage of this approach +is that `nix-build` will rebuild the image if there are any changes and pass the new store +path to `docker load`: + +```shell-session +$ docker load < $(nix-build hello-docker.nix) +Loaded image: hello-docker:y74sb4nrhxr975xs7h83izgm8z75x5fc +``` + +Now that you have loaded the image into Docker, it is time to run it: + +```shell-session +$ docker run -t hello-docker:y74sb4nrhxr975xs7h83izgm8z75x5fc +Hello, world! +``` + +## Working with Docker images + +A general introduction to working with Docker images is not part of this +tutorial. The [official Docker documentation](https://docs.docker.com/) is a +much better place for that. Note that when you build your +Docker images with Nix, you will probably not write a `Dockerfile` +as Nix replaces the Dockerfile functionality within the Docker ecosystem. + +Nonetheless, understanding the anatomy of a Dockerfile may still be useful to +follow along how Nix replaces each of its functions. Using the Docker CLI, +Docker Compose, Docker Swarm or Docker Hub on the other hand may still be +relevant depending on your use case. + +## Next steps + +- More details on how to use `dockerTools` can be found in the [reference documentation](https://nixos.org/nixpkgs/manual/#sec-pkgs-dockerTools). +- You will also want to [browse through more examples of Docker images built with Nix](https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/docker/examples.nix). +- [Arion](https://docs.hercules-ci.com/arion/), docker-compose wrapper with first-class support for Nix. +- Build docker images on a {ref}`CI with Github Actions ` diff --git a/source/tutorials/building-and-running-docker-images.rst b/source/tutorials/building-and-running-docker-images.rst deleted file mode 100644 index d317540..0000000 --- a/source/tutorials/building-and-running-docker-images.rst +++ /dev/null @@ -1,149 +0,0 @@ -Building and running Docker images -================================== - -.. meta:: - :description: Building and running Docker images - :keywords: Docker, containers, Nix, reproducible, build, tutorial - -`Docker `_ is a set of tools and services used to -build, manage and deploy containers. - -As many cloud platforms offer Docker-based -container hosting services, creating Docker containers for a given service is a -common task when building reproducible software. In this tutorial, you will -learn how to build Docker containers using Nix. - - -Prerequisites -------------- -We assume you have both Nix and `Docker installed `_. Docker is available in -``nixpkgs``, which is the preferred way to install it on NixOS. However, you can -also use the native Docker installation of your OS, if you are on another Linux -distribution or MacOS. - - -Build your first container --------------------------- - -`Nixpkgs `_ provides ``dockerTools`` to create -Docker images: - -.. code:: nix - - { pkgs ? import { } - , pkgsLinux ? import { system = "x86_64-linux"; } - }: - - pkgs.dockerTools.buildImage { - name = "hello-docker"; - config = { - Cmd = [ "${pkgsLinux.hello}/bin/hello" ]; - }; - } - -.. note:: - - If you're running **macOS** or any platform other than ``x86_64-linux``, you'll need to either: - - - `Set up a remote builder `_ to build on Linux - - :ref:`Cross compile to Linux ` by replacing ``pkgsLinux.hello`` with ``pkgs.pkgsCross.musl64.hello`` - -We call the ``dockerTools.buildImage`` and pass in some parameters: - -* a ``name`` for our image -* the ``config`` including the command ``Cmd`` that should be run inside the container - once the image is started. Here we reference the GNU hello package from ``nixpkgs`` and run - its executable in the container. - -Save this in ``hello-docker.nix`` and build it: - -.. code:: shell-session - - $ nix-build hello-docker.nix - these derivations will be built: - /nix/store/qpgdp0qpd8ddi1ld72w02zkmm7n87b92-docker-layer-hello-docker.drv - /nix/store/m4xyfyviwbi38sfplq3xx54j6k7mccfb-runtime-deps.drv - /nix/store/v0bvy9qxa79izc7s03fhpq5nqs2h4sr5-docker-image-hello-docker.tar.gz.drv - warning: unknown setting 'experimental-features' - building '/nix/store/qpgdp0qpd8ddi1ld72w02zkmm7n87b92-docker-layer-hello-docker.drv'... - No contents to add to layer. - Packing layer... - Computing layer checksum... - Finished building layer 'hello-docker' - building '/nix/store/m4xyfyviwbi38sfplq3xx54j6k7mccfb-runtime-deps.drv'... - building '/nix/store/v0bvy9qxa79izc7s03fhpq5nqs2h4sr5-docker-image-hello-docker.tar.gz.drv'... - Adding layer... - tar: Removing leading `/' from member names - Adding meta... - Cooking the image... - Finished. - /nix/store/y74sb4nrhxr975xs7h83izgm8z75x5fc-docker-image-hello-docker.tar.gz - -The image tag (``y74sb4nrhxr975xs7h83izgm8z75x5fc``) refers to the Nix build hash -and makes sure that the Docker image corresponds to our Nix build. The store -path in the last line of the output references the Docker image. - - -Run the container ------------------ - -To work with the container, load this image into -Docker's image registry from the default ``result`` symlink created by nix-build: - -.. code:: shell-session - - $ docker load < result - Loaded image: hello-docker:y74sb4nrhxr975xs7h83izgm8z75x5fc - -You can also use the store path to load the image in order to avoid depending on the presence of -``result`` - -.. code:: shell-session - - $ docker load < /nix/store/y74sb4nrhxr975xs7h83izgm8z75x5fc-docker-image-hello-docker.tar.gz - Loaded image: hello-docker:y74sb4nrhxr975xs7h83izgm8z75x5fc - -Even more conveniently, you can do everything in one command. The advantage of this approach -is that ``nix-build`` will rebuild the image if there are any changes and pass the new store -path to ``docker load``: - -.. code:: shell-session - - $ docker load < $(nix-build hello-docker.nix) - Loaded image: hello-docker:y74sb4nrhxr975xs7h83izgm8z75x5fc - -Now that you have loaded the image into Docker, it is time to run it: - -.. code:: shell-session - - $ docker run -t hello-docker:y74sb4nrhxr975xs7h83izgm8z75x5fc - Hello, world! - - -Working with Docker images --------------------------- - -A general introduction to working with Docker images is not part of this -tutorial. The `official Docker documentation `_ is a -much better place for that. Note that when you build your -Docker images with Nix, you will probably not write a ``Dockerfile`` -as Nix replaces the Dockerfile functionality within the Docker ecosystem. - -Nonetheless, understanding the anatomy of a Dockerfile may still be useful to -follow along how Nix replaces each of its functions. Using the Docker CLI, -Docker Compose, Docker Swarm or Docker Hub on the other hand may still be -relevant depending on your use case. - - -Next steps ----------- - -- More details on how to use ``dockerTools`` can be found in the `reference documentation - `_. - -- You will also want to `browse through more examples of Docker images built with Nix - `_. - -- `Arion `_, docker-compose wrapper with first-class support for Nix. - -- Build docker images on a :ref:`CI with Github Actions ` diff --git a/source/tutorials/building-bootable-iso-image.md b/source/tutorials/building-bootable-iso-image.md new file mode 100644 index 0000000..cf7f36d --- /dev/null +++ b/source/tutorials/building-bootable-iso-image.md @@ -0,0 +1,41 @@ +# Building bootable ISO image + +:::{note} +In case you'd like to build images for a different platform that you're on, see [Cross compiling](https://github.com/nix-community/nixos-generators#cross-compiling). +::: + +Often we're faced with the official installation image lacking some hardware support. + +Create `myimage.nix` that will point the kernel to the latest using the minimal installation iso: + +```nix +{ pkgs, modulesPath, lib, ... }: { + imports = [ + "${modulesPath}/installer/cd-dvd/installation-cd-minimal.nix" + ]; + + # use the latest Linux kernel + boot.kernelPackages = pkgs.linuxPackages_latest; + + # Needed for https://github.com/NixOS/nixpkgs/issues/58959 + boot.supportedFilesystems = lib.mkForce [ "btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs" ]; +} +``` + +Generate an ISO with the above configuration: + +```shell-session +$ NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs/archive/74e2faf5965a12e8fa5cff799b1b19c6cd26b0e3.tar.gz nix-shell -p nixos-generators --run "nixos-generate --format iso --configuration ./myimage.nix -o result" +``` + +Copy the new image to your USB stick by replacing `sdX` with the name of your device: + +```shell-session +$ dd if=result/iso/*.iso of=/dev/sdX status=progress +$ sync +``` + +## Next steps + +- There are a bunch of [other formats that generators support](https://github.com/nix-community/nixos-generators#supported-formats), + for example different cloud providers or virtualization technologies diff --git a/source/tutorials/building-bootable-iso-image.rst b/source/tutorials/building-bootable-iso-image.rst deleted file mode 100644 index d8f4448..0000000 --- a/source/tutorials/building-bootable-iso-image.rst +++ /dev/null @@ -1,44 +0,0 @@ -Building bootable ISO image -=========================== - -.. note:: - In case you'd like to build images for a different platform that you're on, see `Cross compiling `_. - -Often we're faced with the official installation image lacking some hardware support. - -Create ``myimage.nix`` that will point the kernel to the latest using the minimal installation iso: - -.. code:: nix - - { pkgs, modulesPath, lib, ... }: { - imports = [ - "${modulesPath}/installer/cd-dvd/installation-cd-minimal.nix" - ]; - - # use the latest Linux kernel - boot.kernelPackages = pkgs.linuxPackages_latest; - - # Needed for https://github.com/NixOS/nixpkgs/issues/58959 - boot.supportedFilesystems = lib.mkForce [ "btrfs" "reiserfs" "vfat" "f2fs" "xfs" "ntfs" "cifs" ]; - } - -Generate an ISO with the above configuration: - -.. code:: shell-session - - $ NIX_PATH=nixpkgs=https://github.com/NixOS/nixpkgs/archive/74e2faf5965a12e8fa5cff799b1b19c6cd26b0e3.tar.gz nix-shell -p nixos-generators --run "nixos-generate --format iso --configuration ./myimage.nix -o result" - -Copy the new image to your USB stick by replacing ``sdX`` with the name of your device: - -.. code:: shell-session - - $ dd if=result/iso/*.iso of=/dev/sdX status=progress - $ sync - - -Next steps ----------- - -- There are a bunch of `other formats that generators support `_, - for example different cloud providers or virtualization technologies - diff --git a/source/tutorials/continuous-integration-github-actions.md b/source/tutorials/continuous-integration-github-actions.md new file mode 100644 index 0000000..d430bf9 --- /dev/null +++ b/source/tutorials/continuous-integration-github-actions.md @@ -0,0 +1,76 @@ +--- +html_meta: + "description lang=en": "Continuous Integration with GitHub Actions and Cachix" + "keywords": "CI, Continuous Integration, GitHub Actions, Cachix, Binary Cache, Nix" +--- + +(github-actions)= + +# Continuous Integration with GitHub Actions + +In this tutorial, we'll show you **a few short steps** to get started using [GitHub Actions](https://github.com/features/actions) as your continuous integration (CI) workflow for commits and pull requests. + +## Caching builds using Cachix + +One benefit of Nix is that **CI can build and cache developer environments for every project** on every branch using binary caches. + +An important aspect of CI is the feedback loop of, **how many minutes does the build take to finish?** + +Using [Cachix](https://cachix.org/) you'll never have to waste time building a derivation twice, and you'll share built derivations with all your developers. + +After each job, just-built derivations are pushed to your binary cache. + +Before each job, derivations to be built are first substituted (if they exist) from your binary cache. + +### 1. Creating your first binary cache + +It's recommended to have different binary caches per team, depending who will have write/read access to it. + +Fill out the form on the [create binary cache](https://app.cachix.org/cache) page. + +On your freshly created binary cache, follow the **Push binaries** tab instructions. + +### 2. Setting up secrets + +On your GitHub repository or organization (for use across all repositories): + +1. Click on `Settings`. +2. Click on `Secrets`. +3. Add your previously generated secrets (`CACHIX_SIGNING_KEY` and/or `CACHIX_AUTH_TOKEN`). + +## Setting up GitHub Actions + +Create `.github/workflows/test.yml` with: + +```yaml +name: "Test" +on: + pull_request: + push: +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.4.0 + - uses: cachix/install-nix-action@v16 + with: + nix_path: nixpkgs=channel:nixos-unstable + - uses: cachix/cachix-action@v10 + with: + name: mycache + # If you chose signing key for write access + signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' + # If you chose API tokens for write access OR if you have a private cache + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - run: nix-build + - run: nix-shell --run "echo OK" +``` + +Once you commit and push to your GitHub repository, +you should see status checks appearing on commits and PRs. + +## Next steps + +- See [GitHub Actions workflow syntax](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions) +- To quickly setup a Nix project read through + [Getting started Nix template](https://github.com/nix-dot-dev/getting-started-nix-template). diff --git a/source/tutorials/continuous-integration-github-actions.rst b/source/tutorials/continuous-integration-github-actions.rst deleted file mode 100644 index 3917de5..0000000 --- a/source/tutorials/continuous-integration-github-actions.rst +++ /dev/null @@ -1,85 +0,0 @@ -.. _github-actions: - -.. meta:: - :description: Continuous Integration with GitHub Actions and Cachix - :keywords: CI, Continuous Integration, GitHub Actions, Cachix, Binary Cache, Nix - -Continuous Integration with GitHub Actions -========================================== - -In this tutorial, we'll show you **a few short steps** to get started using `GitHub Actions `_ as your continuous integration (CI) workflow for commits and pull requests. - -Caching builds using Cachix ---------------------------- - -One benefit of Nix is that **CI can build and cache developer environments for every project** on every branch using binary caches. - -An important aspect of CI is the feedback loop of, **how many minutes does the build take to finish?** - -Using `Cachix `_ you'll never have to waste time building a derivation twice, and you'll share built derivations with all your developers. - -After each job, just-built derivations are pushed to your binary cache. - -Before each job, derivations to be built are first substituted (if they exist) from your binary cache. - - -1. Creating your first binary cache -*********************************** - -It's recommended to have different binary caches per team, depending who will have write/read access to it. - -Fill out the form on the `create binary cache `_ page. - -On your freshly created binary cache, follow the **Push binaries** tab instructions. - - -2. Setting up secrets -********************* - -On your GitHub repository or organization (for use across all repositories): - -1. Click on ``Settings``. -2. Click on ``Secrets``. -3. Add your previously generated secrets (``CACHIX_SIGNING_KEY`` and/or ``CACHIX_AUTH_TOKEN``). - - -Setting up GitHub Actions -------------------------- - -Create ``.github/workflows/test.yml`` with: - -.. code:: yaml - - name: "Test" - on: - pull_request: - push: - jobs: - tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2.4.0 - - uses: cachix/install-nix-action@v16 - with: - nix_path: nixpkgs=channel:nixos-unstable - - uses: cachix/cachix-action@v10 - with: - name: mycache - # If you chose signing key for write access - signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' - # If you chose API tokens for write access OR if you have a private cache - authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - run: nix-build - - run: nix-shell --run "echo OK" - -Once you commit and push to your GitHub repository, -you should see status checks appearing on commits and PRs. - - -Next steps ----------- - -- See `GitHub Actions workflow syntax `_ - -- To quickly setup a Nix project read through - `Getting started Nix template `_. diff --git a/source/tutorials/contributing.md b/source/tutorials/contributing.md new file mode 100644 index 0000000..a071e6d --- /dev/null +++ b/source/tutorials/contributing.md @@ -0,0 +1,67 @@ +# How to Contribute + +This guide explains how you can contribute to Nix, Nix packages or +NixOS. + +## Report an issue + +We can only fix issues that we know of, so please report any issue you +encounter. + +Issues with the **package manager Nix** (including it's documentation) +are reported at . + +Issues with **specific packages or NixOS** (including it's modules and +documentation) are reported at . + +Make sure that there is not already an open issue for your problem. +Please follow the issue template and fill in all requested information +as they help us solve the problem. + +You need a [GitHub] account for that. + +## Contribute to Nix + +The package manager Nix is mostly written in C++. If you are a developer +and want to contribute to it's development, you can find information on +[how to setup a development environment] in the manual. + +You can find inspiration for things to improve in the [reported +issues][reported issues]. + +Feel free to [join our community] to get in +contact with other developers. + +## Contribute to Nix packages + +Packaging for Nix is simple when you have understood the basic concept. + +[The manual] explains step-by-step how to add new packages to the Nix +package collection. There are also [programming language specific +instructions][programming language specific instructions]. + +## Contribute to NixOS + +It’s pretty easy to contribute to NixOS compared to other linux +distributions. All the code is on GitHub in the repository [nixpkgs]. +Everyone can propose an improvement and most of them get merged after a +review of the maintainers. You will get feedback in the pull request. + +See the [NixOS manual] to get started and find all the details. + +You can find inspiration for things to improve in the [reported +issues](https://github.com/NixOS/nixpkgs/issues). There are also +[issues tagged with good-first-bug] that are a good start for new +contributors. + +Feel free to [join our community] of developers! + +[github]: https://github.com/ +[how to setup a development environment]: https://nixos.org/manual/nix/stable/contributing/hacking.html +[issues tagged with good-first-bug]: https://github.com/NixOS/nixpkgs/labels/3.skill%3A%20good-first-bug +[join our community]: https://github.com/NixOS/nixpkgs#community +[nixos manual]: https://nixos.org/manual/nixos/stable/index.html#ch-development +[nixpkgs]: https://github.com/NixOS/nixpkgs +[programming language specific instructions]: https://nixos.org/manual/nixpkgs/stable/#chap-language-support +[reported issues]: https://github.com/NixOS/nix/issues +[the manual]: https://nixos.org/manual/nix/stable/quick-start.html diff --git a/source/tutorials/contributing.rst b/source/tutorials/contributing.rst deleted file mode 100644 index 7dc2f38..0000000 --- a/source/tutorials/contributing.rst +++ /dev/null @@ -1,72 +0,0 @@ -How to Contribute -================= - -This guide explains how you can contribute to Nix, Nix packages or -NixOS. - -Report an issue ---------------- - -We can only fix issues that we know of, so please report any issue you -encounter. - -Issues with the **package manager Nix** (including it's documentation) -are reported at https://github.com/NixOS/nix/issues. - -Issues with **specific packages or NixOS** (including it's modules and -documentation) are reported at https://github.com/NixOS/nixpkgs/issues. - -Make sure that there is not already an open issue for your problem. -Please follow the issue template and fill in all requested information -as they help us solve the problem. - -You need a `GitHub`_ account for that. - -Contribute to Nix ------------------ - -The package manager Nix is mostly written in C++. If you are a developer -and want to contribute to it's development, you can find information on -`how to setup a development environment`_ in the manual. - -You can find inspiration for things to improve in the `reported -issues`_. - -Feel free to `join our community`_ to get in -contact with other developers. - -Contribute to Nix packages --------------------------- - -Packaging for Nix is simple when you have understood the basic concept. - -`The manual`_ explains step-by-step how to add new packages to the Nix -package collection. There are also `programming language specific -instructions`_. - -Contribute to NixOS -------------------- - -It’s pretty easy to contribute to NixOS compared to other linux -distributions. All the code is on GitHub in the repository `nixpkgs`_. -Everyone can propose an improvement and most of them get merged after a -review of the maintainers. You will get feedback in the pull request. - -See the `NixOS manual`_ to get started and find all the details. - -You can find inspiration for things to improve in the `reported -issues `__. There are also -`issues tagged with good-first-bug`_ that are a good start for new -contributors. - -Feel free to `join our community`_ of developers! - -.. _GitHub: https://github.com/ -.. _how to setup a development environment: https://nixos.org/manual/nix/stable/contributing/hacking.html -.. _reported issues: https://github.com/NixOS/nix/issues -.. _join our community: https://github.com/NixOS/nixpkgs#community -.. _The manual: https://nixos.org/manual/nix/stable/quick-start.html -.. _programming language specific instructions: https://nixos.org/manual/nixpkgs/stable/#chap-language-support -.. _nixpkgs: https://github.com/NixOS/nixpkgs -.. _NixOS manual: https://nixos.org/manual/nixos/stable/index.html#ch-development -.. _issues tagged with good-first-bug: https://github.com/NixOS/nixpkgs/labels/3.skill%3A%20good-first-bug diff --git a/source/tutorials/cross-compilation.md b/source/tutorials/cross-compilation.md new file mode 100644 index 0000000..6158db7 --- /dev/null +++ b/source/tutorials/cross-compilation.md @@ -0,0 +1,290 @@ +--- +html_meta: + "description lang=en": "Cross compilation tutorial using Nix" + "keywords": "Nix, cross compilation, cross-compile, Nix" +--- + + +(cross-compilation)= + +# Cross compilation + +When compiling code, we can distinguish between the **build platform**, where the executable +is *built*, and the **host platform**, where the compiled executable *runs*. [^id3] + +**Native compilation** is the special case where those two platforms are the same. +**Cross compilation** is the general case where those two platforms are not. + +Cross compilation needed when the host platform has limited resources (such as CPU) +or when it's not easily accessible for development. + +The Nix community has world-class support for cross compilation, +after many years of hard work. + +[^id3]: Terminology for cross compilation platforms differs between build systems. + We have chosen to follow + [autoconf terminology](https://www.gnu.org/software/autoconf/manual/autoconf-2.69/html_node/Hosts-and-Cross_002dCompilation.html). + +## What's a target platform? + +There's actually a third platform named the target platform. + +It matters in cases where you'd like to distribute a compiler binary, +as you'd then like to build a compiler on the build platform, compile code on the +host platform and run the final executable on the target platform. + +Since that's rarely needed, we'll treat the target platform the same as the host. + +## Pinning nixpkgs + +To ensure reproducibility of this tutorial as explained in {ref}`the pinning tutorial `: + +```shell-session +$ NIX_PATH=https://github.com/NixOS/nixpkgs/archive/9420363b95521e65a76eb5153de1eaee4a2e41c6.tar.gz +``` + +## Determining the host platform config + +The build platform is determined automatically by Nix +as it can just guess it during the configure phase. + +The host platform is best determined by running on the host platform: + +```shell-session +$ bash $(nix-build '' -A gnu-config)/config.guess +aarch64-unknown-linux-gnu +``` + +In case that's not possible (when the host platform is not easily accessible +for development), the platform config has to be constructed manually via the following template: + +``` +--- +``` + +Note that `` is often `unknown` and `` is optional. +There's also no unique identifier for a platform, for example `unknown` and +`pc` are interchangeable (hence it's called config.guess). + +If you can't install Nix, find a way to run `config.guess` (usually comes with + +: the autoconf package) from the OS you're able to run on the host platform. + +Some other common examples of platform configs: + +- aarch64-apple-darwin14 +- aarch64-pc-linux-gnu +- x86_64-w64-mingw32 +- aarch64-apple-ios + +:::{note} +macOS/Darwin is a special case, as not the whole OS is open-source. +It's only possible to cross compile between `aarch64-darwin` and `x86_64-darwin`. +`aarch64-darwin` support was recently added, so cross compilation is barely tested. +::: + +## Choosing the host platform with Nix + +Nixpkgs comes with a set of predefined host platforms applied to all packages. + +It's possible to explore predefined attribute sets via `nix repl`: + +```shell-session +$ nix repl '' +Welcome to Nix version 2.3.12. Type :? for help. + +Loading ''... +Added 14200 variables. + +nix-repl> pkgsCross. +pkgsCross.aarch64-android pkgsCross.musl-power +pkgsCross.aarch64-android-prebuilt pkgsCross.musl32 +pkgsCross.aarch64-darwin pkgsCross.musl64 +pkgsCross.aarch64-embedded pkgsCross.muslpi +pkgsCross.aarch64-multiplatform pkgsCross.or1k +pkgsCross.aarch64-multiplatform-musl pkgsCross.pogoplug4 +pkgsCross.aarch64be-embedded pkgsCross.powernv +pkgsCross.amd64-netbsd pkgsCross.ppc-embedded +pkgsCross.arm-embedded pkgsCross.ppc64 +pkgsCross.armhf-embedded pkgsCross.ppc64-musl +pkgsCross.armv7a-android-prebuilt pkgsCross.ppcle-embedded +pkgsCross.armv7l-hf-multiplatform pkgsCross.raspberryPi +pkgsCross.avr pkgsCross.remarkable1 +pkgsCross.ben-nanonote pkgsCross.remarkable2 +pkgsCross.fuloongminipc pkgsCross.riscv32 +pkgsCross.ghcjs pkgsCross.riscv32-embedded +pkgsCross.gnu32 pkgsCross.riscv64 +pkgsCross.gnu64 pkgsCross.riscv64-embedded +pkgsCross.i686-embedded pkgsCross.scaleway-c1 +pkgsCross.iphone32 pkgsCross.sheevaplug +pkgsCross.iphone32-simulator pkgsCross.vc4 +pkgsCross.iphone64 pkgsCross.wasi32 +pkgsCross.iphone64-simulator pkgsCross.x86_64-embedded +pkgsCross.mingw32 pkgsCross.x86_64-netbsd +pkgsCross.mingwW64 pkgsCross.x86_64-netbsd-llvm +pkgsCross.mmix pkgsCross.x86_64-unknown-redox +pkgsCross.msp430 +``` + +Cross compilation package attribute names are made up, so it isn't always clear +what is the corresponding platform config. + +It's possible to query the platform config using: + +``` +nix-repl> pkgsCross.aarch64-multiplatform.stdenv.hostPlatform.config +"aarch64-unknown-linux-gnu" +``` + +In case the host platform you seek hasn't been defined yet: + +1. [Contribute it upstream](https://github.com/NixOS/nixpkgs/blob/master/lib/systems/examples.nix). + +2. Pass the host platforms to `crossSystem` when importing ``: + + ``` + nix-repl> (import { crossSystem = { config = "aarch64-unknown-linux-gnu"; }; }).hello + «derivation /nix/store/qjj23s25kg4vjqq19vxs4dg7k7h214ns-hello-aarch64-unknown-linux-gnu-2.10.drv» + ``` + + Or using passing it as an argument to `nix-build`: + + ``` + $ nix-build '' -A hello --arg crossSystem '{ config = "aarch64-unknown-linux-gnu"; }' + ``` + +## Cross compiling for the first time! + +To cross compile a package like [hello](https://www.gnu.org/software/hello/), +pick the platform attribute - `aarch64-multiplatform` in our case - and run: + +```shell-session +$ nix-build '' -A pkgsCross.aarch64-multiplatform.hello +... +/nix/store/pzi2h0d60nb4ydcl3nn7cbxxdnibw3sy-hello-aarch64-unknown-linux-gnu-2.10 +``` + +[Search for a package](https://search.nixos.org/packages) attribute name to find the +one that you're interested in building. + +## Real-world cross compiling of a Hello World example + +To show off the power of cross compilation in Nix, let's build our own Hello World program +by cross compiling it as static executables to `armv6l-unknown-linux-gnueabihf` +and `x86_64-w64-mingw32` (Windows) platforms and run the resulting executable +with [an emulator](https://en.wikipedia.org/wiki/Emulator). + +```nix +{ pkgs ? import {} +}: + +let + # Create a C program that prints Hello World + helloWorld = pkgs.writeText "hello.c" '' + #include + + int main (void) + { + printf ("Hello, world!\n"); + return 0; + } + ''; + + # A function that takes host platform packages + crossCompileFor = hostPkgs: + # Run a simple command with the compiler available + hostPkgs.runCommandCC "hello-world-cross-test" {} '' + # Wine requires home directory + HOME=$PWD + + # Compile our example using the compiler specific to our host platform + $CC ${helloWorld} -o hello + + # Run the compiled program using user mode emulation (Qemu/Wine) + # buildPackages is passed so that emulation is built for the build platform + ${hostPkgs.stdenv.hostPlatform.emulator hostPkgs.buildPackages} hello > $out + + # print to stdout + cat $out + ''; +in { + # Statically compile our example using the two platform hosts + rpi = crossCompileFor pkgs.pkgsCross.raspberryPi; + windows = crossCompileFor pkgs.pkgsCross.mingwW64; +} +``` + +If we build this example and print both resulting derivations, we should see "Hello, world!" for each: + +```shell-session +$ cat $(nix-build cross-compile.nix) +Hello, world! +Hello, world! +``` + +## Developer environment with a cross compiler + +In the {ref}`tutorial for declarative reproducible environments `, +we looked at how Nix helps us provide tooling and system libraries for our project. + +It's also possible to provide an environment with a compiler configured for **cross-compilation +to static binaries using musl**. + +Given we have a `shell.nix`: + +```nix +{ nixpkgs ? fetchTarball "https://github.com/NixOS/nixpkgs/archive/bba3474a5798b5a3a87e10102d1a55f19ec3fca5.tar.gz" +, pkgs ? (import nixpkgs {}).pkgsCross.aarch64-multiplatform +}: + +# callPackage is needed due to https://github.com/NixOS/nixpkgs/pull/126844 +pkgs.pkgsStatic.callPackage ({ mkShell, zlib, pkg-config, file }: mkShell { + # these tools run on the build platform, but are configured to target the host platform + nativeBuildInputs = [ pkg-config file ]; + # libraries needed for the host platform + buildInputs = [ zlib ]; +}) {} +``` + +And `hello.c`: + +```c +#include + +int main (void) +{ + printf ("Hello, world!\n"); + return 0; +} +``` + +We can cross compile it: + +```shell-session +$ nix-shell --run '$CC hello.c -o hello' cross-compile-shell.nix +``` + +And confirm it's aarch64: + +```shell-session +$ nix-shell --run 'file hello' cross-compile-shell.nix +hello: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), statically linked, with debug_info, not stripped +``` + +## Next steps + +- The [official binary cache](https://cache.nixos.org) has very limited number of binaries + for packages that are cross compiled, so to save time recompiling, configure + {ref}`a binary cache and CI (GitHub Actions and Cachix) `. + +- While many compilers in nixpkgs support cross compilation, + not all of them do. + + On top of that, supporting cross compilation is not trivial + work and due to many possible combinations of what would + need to be tested, some packages might not build. + + [A detailed explanation how of cross compilation is implemented in Nix](https://nixos.org/manual/nixpkgs/stable/#chap-cross) can help with fixing those issues. + +- The Nix community has a [dedicated Matrix room](https://matrix.to/#/#cross-compiling:nixos.org) + for help around cross compiling. diff --git a/source/tutorials/cross-compilation.rst b/source/tutorials/cross-compilation.rst deleted file mode 100644 index f1eb40c..0000000 --- a/source/tutorials/cross-compilation.rst +++ /dev/null @@ -1,295 +0,0 @@ -.. _cross-compilation: - -.. meta:: - :description: Cross compilation tutorial using Nix - :keywords: Nix, cross compilation, cross-compile, Nix - -Cross compilation -================= - -When compiling code, we can distinguish between the **build platform**, where the executable -is *built*, and the **host platform**, where the compiled executable *runs*. [#]_ - -**Native compilation** is the special case where those two platforms are the same. -**Cross compilation** is the general case where those two platforms are not. - -Cross compilation needed when the host platform has limited resources (such as CPU) -or when it's not easily accessible for development. - -The Nix community has world-class support for cross compilation, -after many years of hard work. - -.. [#] Terminology for cross compilation platforms differs between build systems. - We have chosen to follow - `autoconf terminology `_. - - -What's a target platform? -------------------------- - -There's actually a third platform named the target platform. - -It matters in cases where you'd like to distribute a compiler binary, -as you'd then like to build a compiler on the build platform, compile code on the -host platform and run the final executable on the target platform. - -Since that's rarely needed, we'll treat the target platform the same as the host. - - -Pinning nixpkgs ---------------- - -To ensure reproducibility of this tutorial as explained in :ref:`the pinning tutorial `: - -.. code:: shell-session - - $ NIX_PATH=https://github.com/NixOS/nixpkgs/archive/9420363b95521e65a76eb5153de1eaee4a2e41c6.tar.gz - - -Determining the host platform config ------------------------------------- - -The build platform is determined automatically by Nix -as it can just guess it during the configure phase. - -The host platform is best determined by running on the host platform: - -.. code:: shell-session - - $ bash $(nix-build '' -A gnu-config)/config.guess - aarch64-unknown-linux-gnu - -In case that's not possible (when the host platform is not easily accessible -for development), the platform config has to be constructed manually via the following template: - -.. code:: - - --- - -Note that ```` is often ``unknown`` and ```` is optional. -There's also no unique identifier for a platform, for example ``unknown`` and -``pc`` are interchangeable (hence it's called config.guess). - -If you can't install Nix, find a way to run ``config.guess`` (usually comes with - the autoconf package) from the OS you're able to run on the host platform. - -Some other common examples of platform configs: - -- aarch64-apple-darwin14 -- aarch64-pc-linux-gnu -- x86_64-w64-mingw32 -- aarch64-apple-ios - -.. note:: macOS/Darwin is a special case, as not the whole OS is open-source. - It's only possible to cross compile between ``aarch64-darwin`` and ``x86_64-darwin``. - ``aarch64-darwin`` support was recently added, so cross compilation is barely tested. - - -Choosing the host platform with Nix ------------------------------------ - -Nixpkgs comes with a set of predefined host platforms applied to all packages. - -It's possible to explore predefined attribute sets via ``nix repl``: - -.. code:: shell-session - - $ nix repl '' - Welcome to Nix version 2.3.12. Type :? for help. - - Loading ''... - Added 14200 variables. - - nix-repl> pkgsCross. - pkgsCross.aarch64-android pkgsCross.musl-power - pkgsCross.aarch64-android-prebuilt pkgsCross.musl32 - pkgsCross.aarch64-darwin pkgsCross.musl64 - pkgsCross.aarch64-embedded pkgsCross.muslpi - pkgsCross.aarch64-multiplatform pkgsCross.or1k - pkgsCross.aarch64-multiplatform-musl pkgsCross.pogoplug4 - pkgsCross.aarch64be-embedded pkgsCross.powernv - pkgsCross.amd64-netbsd pkgsCross.ppc-embedded - pkgsCross.arm-embedded pkgsCross.ppc64 - pkgsCross.armhf-embedded pkgsCross.ppc64-musl - pkgsCross.armv7a-android-prebuilt pkgsCross.ppcle-embedded - pkgsCross.armv7l-hf-multiplatform pkgsCross.raspberryPi - pkgsCross.avr pkgsCross.remarkable1 - pkgsCross.ben-nanonote pkgsCross.remarkable2 - pkgsCross.fuloongminipc pkgsCross.riscv32 - pkgsCross.ghcjs pkgsCross.riscv32-embedded - pkgsCross.gnu32 pkgsCross.riscv64 - pkgsCross.gnu64 pkgsCross.riscv64-embedded - pkgsCross.i686-embedded pkgsCross.scaleway-c1 - pkgsCross.iphone32 pkgsCross.sheevaplug - pkgsCross.iphone32-simulator pkgsCross.vc4 - pkgsCross.iphone64 pkgsCross.wasi32 - pkgsCross.iphone64-simulator pkgsCross.x86_64-embedded - pkgsCross.mingw32 pkgsCross.x86_64-netbsd - pkgsCross.mingwW64 pkgsCross.x86_64-netbsd-llvm - pkgsCross.mmix pkgsCross.x86_64-unknown-redox - pkgsCross.msp430 - -Cross compilation package attribute names are made up, so it isn't always clear -what is the corresponding platform config. - -It's possible to query the platform config using:: - - nix-repl> pkgsCross.aarch64-multiplatform.stdenv.hostPlatform.config - "aarch64-unknown-linux-gnu" - -In case the host platform you seek hasn't been defined yet: - -a) `Contribute it upstream `_. - -b) Pass the host platforms to ``crossSystem`` when importing ````:: - - nix-repl> (import { crossSystem = { config = "aarch64-unknown-linux-gnu"; }; }).hello - «derivation /nix/store/qjj23s25kg4vjqq19vxs4dg7k7h214ns-hello-aarch64-unknown-linux-gnu-2.10.drv» - - Or using passing it as an argument to ``nix-build``:: - - $ nix-build '' -A hello --arg crossSystem '{ config = "aarch64-unknown-linux-gnu"; }' - - -Cross compiling for the first time! ------------------------------------ - -To cross compile a package like `hello `_, -pick the platform attribute - ``aarch64-multiplatform`` in our case - and run: - -.. code:: shell-session - - $ nix-build '' -A pkgsCross.aarch64-multiplatform.hello - ... - /nix/store/pzi2h0d60nb4ydcl3nn7cbxxdnibw3sy-hello-aarch64-unknown-linux-gnu-2.10 - -`Search for a package `_ attribute name to find the -one that you're interested in building. - - -Real-world cross compiling of a Hello World example ---------------------------------------------------- - -To show off the power of cross compilation in Nix, let's build our own Hello World program -by cross compiling it as static executables to ``armv6l-unknown-linux-gnueabihf`` -and ``x86_64-w64-mingw32`` (Windows) platforms and run the resulting executable -with `an emulator `_. - -.. code:: nix - - { pkgs ? import {} - }: - - let - # Create a C program that prints Hello World - helloWorld = pkgs.writeText "hello.c" '' - #include - - int main (void) - { - printf ("Hello, world!\n"); - return 0; - } - ''; - - # A function that takes host platform packages - crossCompileFor = hostPkgs: - # Run a simple command with the compiler available - hostPkgs.runCommandCC "hello-world-cross-test" {} '' - # Wine requires home directory - HOME=$PWD - - # Compile our example using the compiler specific to our host platform - $CC ${helloWorld} -o hello - - # Run the compiled program using user mode emulation (Qemu/Wine) - # buildPackages is passed so that emulation is built for the build platform - ${hostPkgs.stdenv.hostPlatform.emulator hostPkgs.buildPackages} hello > $out - - # print to stdout - cat $out - ''; - in { - # Statically compile our example using the two platform hosts - rpi = crossCompileFor pkgs.pkgsCross.raspberryPi; - windows = crossCompileFor pkgs.pkgsCross.mingwW64; - } - -If we build this example and print both resulting derivations, we should see "Hello, world!" for each: - -.. code:: shell-session - - $ cat $(nix-build cross-compile.nix) - Hello, world! - Hello, world! - - -Developer environment with a cross compiler -------------------------------------------- - -In the :ref:`tutorial for declarative reproducible environments `, -we looked at how Nix helps us provide tooling and system libraries for our project. - -It's also possible to provide an environment with a compiler configured for **cross-compilation -to static binaries using musl**. - -Given we have a ``shell.nix``: - -.. code:: nix - - { nixpkgs ? fetchTarball "https://github.com/NixOS/nixpkgs/archive/bba3474a5798b5a3a87e10102d1a55f19ec3fca5.tar.gz" - , pkgs ? (import nixpkgs {}).pkgsCross.aarch64-multiplatform - }: - - # callPackage is needed due to https://github.com/NixOS/nixpkgs/pull/126844 - pkgs.pkgsStatic.callPackage ({ mkShell, zlib, pkg-config, file }: mkShell { - # these tools run on the build platform, but are configured to target the host platform - nativeBuildInputs = [ pkg-config file ]; - # libraries needed for the host platform - buildInputs = [ zlib ]; - }) {} - -And ``hello.c``: - -.. code:: c - - #include - - int main (void) - { - printf ("Hello, world!\n"); - return 0; - } - -We can cross compile it: - -.. code:: shell-session - - $ nix-shell --run '$CC hello.c -o hello' cross-compile-shell.nix - -And confirm it's aarch64: - -.. code:: shell-session - - $ nix-shell --run 'file hello' cross-compile-shell.nix - hello: ELF 64-bit LSB executable, ARM aarch64, version 1 (SYSV), statically linked, with debug_info, not stripped - - -Next steps ----------- - -- The `official binary cache `_ has very limited number of binaries - for packages that are cross compiled, so to save time recompiling, configure - :ref:`a binary cache and CI (GitHub Actions and Cachix) `. - -- While many compilers in nixpkgs support cross compilation, - not all of them do. - - On top of that, supporting cross compilation is not trivial - work and due to many possible combinations of what would - need to be tested, some packages might not build. - - `A detailed explanation how of cross compilation is implemented in Nix `_ can help with fixing those issues. - -- The Nix community has a `dedicated Matrix room `_ - for help around cross compiling. diff --git a/source/tutorials/declarative-and-reproducible-developer-environments.md b/source/tutorials/declarative-and-reproducible-developer-environments.md new file mode 100644 index 0000000..5b51ffa --- /dev/null +++ b/source/tutorials/declarative-and-reproducible-developer-environments.md @@ -0,0 +1,143 @@ +(declarative-reproducible-envs)= + +# Declarative and reproducible developer environments + +In the {ref}`ad-hoc-envs` tutorial we looked at providing shell +environments for when we need a quick'n'dirty way of getting hold +of some tools. + +In this tutorial we'll take a look how to create {term}`reproducible` +shell environments given a declarative configuration file called a Nix expression. + +## When are declarative shell environments useful? + +This is the quickest approach to getting started with Nix: + +- use single command to invoke it via `nix-shell` +- it works across different operating systems (Linux / MacOS) +- you share the exact same environment with all developers + +Developer environments allow you to: + +- provide CLI tools, such as `psql`, `jq`, `tmux`, etc +- provide developer libraries, such as `zlib`, `openssl`, etc +- set shell environment variables +- execute bash during environment activation + +## Getting started + +At the top-level of your project create `shell.nix` with the following contents: + +```nix +{ pkgs ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/3590f02e7d5760e52072c1a729ee2250b5560746.tar.gz") {} }: + +pkgs.mkShell { + buildInputs = [ + pkgs.which + pkgs.htop + pkgs.zlib + ]; +} +``` + +:::{note} +To understand the first line, read through {ref}`pinning nixpkgs tutorial `. +::: + +We import `nixpkgs` and make a shell with `which` and `htop` available in `$PATH`. +`zlib` provides libraries and headers in case we're compiling something against it. +To enter the environment: + +```shell-session +$ nix-shell +these paths will be fetched (0.07 MiB download, 0.20 MiB unpacked): + /nix/store/072a6x7rwv5f8wr6f5s1rq8nnm767cfp-htop-2.2.0 +copying path '/nix/store/072a6x7rwv5f8wr6f5s1rq8nnm767cfp-htop-2.2.0' from 'https://cache.nixos.org'... + +[nix-shell:~]$ +``` + +The command will start downloading the missing packages from the binary cache. + +Once it's done, you are dropped into a new +shell. This shell provides the packages specified in `shell.nix`. + +Run `htop` to confirm that it is present. Quit the program by hitting +`q`. + +Now, try `which htop` to check where the `htop` command is on disk. +You should see something similar to this: + +```shell-session +[nix-shell:~]$ which htop +/nix/store/y3w2i8kfdbfj9rx287ad52rahjpgv423-htop-2.2.0/bin/htop +``` + +## Customizing your developer environment + +Given the following `shell.nix`: + +```nix +{ pkgs ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/3590f02e7d5760e52072c1a729ee2250b5560746.tar.gz") {} }: + +pkgs.mkShell { + buildInputs = [ + pkgs.which + pkgs.htop + pkgs.zlib + ]; + + shellHook = '' + echo hello + ''; + + MY_ENVIRONMENT_VARIABLE = "world"; +} +``` + +Running `nix-shell` we observe: + +```shell-session +$ nix-shell +hello + +[nix-shell:~]$ echo $MY_ENVIRONMENT_VARIABLE +world +``` + +- The `shellHook` section allows you to execute bash while entering the shell environment. +- Any attributes passed to `mkShell` function are available once the shell environment is active. + +## `direnv`: Automatically activating the environment on directory change + +Besides activating the environment for each project, every time you change +`shell.nix` you need to re-enter the shell. + +You can use `direnv` to automate this process for you, with the downside that each developer needs +to install it globally. + +### Setting up `direnv` + +1. [Install direnv with your OS package manager](https://direnv.net/docs/installation.html#from-system-packages) +2. [Hook it into your shell](https://direnv.net/docs/hook.html) + +At the top-level of your project run: + +``` +echo "use nix" > .envrc && direnv allow +``` + +The next time your launch your terminal and enter the top-level of your project direnv will check for changes. + +```shell-session +$ cd myproject +direnv: loading myproject/.envrc +direnv: using nix +hello +``` + +## Next steps + +- {ref}`pinning-nixpkgs` to see different ways to import nixpkgs +- To quickly set up a Nix project read through + [Getting started Nix template](https://github.com/nix-dot-dev/getting-started-nix-template). diff --git a/source/tutorials/declarative-and-reproducible-developer-environments.rst b/source/tutorials/declarative-and-reproducible-developer-environments.rst deleted file mode 100644 index c22f4b5..0000000 --- a/source/tutorials/declarative-and-reproducible-developer-environments.rst +++ /dev/null @@ -1,156 +0,0 @@ -.. _declarative-reproducible-envs: - -Declarative and reproducible developer environments -=================================================== -In the :ref:`ad-hoc-envs` tutorial we looked at providing shell -environments for when we need a quick'n'dirty way of getting hold -of some tools. - -In this tutorial we'll take a look how to create :term:`reproducible` -shell environments given a declarative configuration file called a Nix expression. - - -When are declarative shell environments useful? ------------------------------------------------ - -This is the quickest approach to getting started with Nix: - -- use single command to invoke it via ``nix-shell`` -- it works across different operating systems (Linux / MacOS) -- you share the exact same environment with all developers - -Developer environments allow you to: - -- provide CLI tools, such as ``psql``, ``jq``, ``tmux``, etc -- provide developer libraries, such as ``zlib``, ``openssl``, etc -- set shell environment variables -- execute bash during environment activation - - -Getting started ---------------- - -At the top-level of your project create ``shell.nix`` with the following contents: - -.. code:: nix - - { pkgs ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/3590f02e7d5760e52072c1a729ee2250b5560746.tar.gz") {} }: - - pkgs.mkShell { - buildInputs = [ - pkgs.which - pkgs.htop - pkgs.zlib - ]; - } - -.. note:: To understand the first line, read through :ref:`pinning nixpkgs tutorial `. - - -We import ``nixpkgs`` and make a shell with ``which`` and ``htop`` available in ``$PATH``. -``zlib`` provides libraries and headers in case we're compiling something against it. -To enter the environment: - -.. code:: shell-session - - $ nix-shell - these paths will be fetched (0.07 MiB download, 0.20 MiB unpacked): - /nix/store/072a6x7rwv5f8wr6f5s1rq8nnm767cfp-htop-2.2.0 - copying path '/nix/store/072a6x7rwv5f8wr6f5s1rq8nnm767cfp-htop-2.2.0' from 'https://cache.nixos.org'... - - [nix-shell:~]$ - - -The command will start downloading the missing packages from the https://cache.nixos.org binary cache. - -Once it's done, you are dropped into a new -shell. This shell provides the packages specified in ``shell.nix``. - -Run ``htop`` to confirm that it is present. Quit the program by hitting -``q``. - -Now, try ``which htop`` to check where the ``htop`` command is on disk. -You should see something similar to this: - -.. code:: shell-session - - [nix-shell:~]$ which htop - /nix/store/y3w2i8kfdbfj9rx287ad52rahjpgv423-htop-2.2.0/bin/htop - - -Customizing your developer environment --------------------------------------- - -Given the following ``shell.nix``: - -.. code:: nix - - { pkgs ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/3590f02e7d5760e52072c1a729ee2250b5560746.tar.gz") {} }: - - pkgs.mkShell { - buildInputs = [ - pkgs.which - pkgs.htop - pkgs.zlib - ]; - - shellHook = '' - echo hello - ''; - - MY_ENVIRONMENT_VARIABLE = "world"; - } - -Running ``nix-shell`` we observe: - -.. code:: shell-session - - $ nix-shell - hello - - [nix-shell:~]$ echo $MY_ENVIRONMENT_VARIABLE - world - - -- The ``shellHook`` section allows you to execute bash while entering the shell environment. -- Any attributes passed to ``mkShell`` function are available once the shell environment is active. - - -``direnv``: Automatically activating the environment on directory change ------------------------------------------------------------------------- - -Besides activating the environment for each project, every time you change -``shell.nix`` you need to re-enter the shell. - -You can use ``direnv`` to automate this process for you, with the downside that each developer needs -to install it globally. - - -Setting up ``direnv`` -********************* - -1. `Install direnv with your OS package manager `_ - -2. `Hook it into your shell `_ - -At the top-level of your project run:: - - echo "use nix" > .envrc && direnv allow - -The next time your launch your terminal and enter the top-level of your project direnv will check for changes. - -.. code:: shell-session - - $ cd myproject - direnv: loading myproject/.envrc - direnv: using nix - hello - - -Next steps ----------- - -- :ref:`pinning-nixpkgs` to see different ways to import nixpkgs - -- To quickly set up a Nix project read through - `Getting started Nix template `_. diff --git a/source/tutorials/deploying-nixos-using-terraform.md b/source/tutorials/deploying-nixos-using-terraform.md new file mode 100644 index 0000000..7cdc947 --- /dev/null +++ b/source/tutorials/deploying-nixos-using-terraform.md @@ -0,0 +1,169 @@ +--- +html_meta: + "description lang=en": "Continuous Integration with GitHub Actions and Cachix" + "keywords": "NixOS, deployment, Terraform, AWS" +--- + +(deploying-nixos-using-terraform)= + +# Deploying NixOS using Terraform + +Assuming you're [familiar with the basics of Terraform](https://www.terraform.io/intro/index.html), +by the end of tutorial you will have provisioned an Amazon AWS instance with Terraform +and will be able to use Nix to deploy incremental changes to NixOS, running on the instance. + +We'll look at how to boot a NixOS machine and how to deploy the incremental changes: + +## Booting NixOS image + +1. Start by providing the terraform executable: + +```shell +nix-shell -p terraform +``` + +2. We are using [Terraform Cloud](https://app.terraform.io) as a [state/locking backend](https://www.terraform.io/docs/state/purpose.html): + +```shell +terraform login +``` + +3. Make sure to [create an organization](https://app.terraform.io/app/organizations/new) like `myorganization` in your Terraform Cloud account. +4. Inside `myorganization` [create a workspace](https://app.terraform.io/app/cachix/workspaces/new) by choosing **CLI-driven workflow** and pick a name like `myapp`. +5. Inside your workspace, under `Settings` / `General` change Execution Mode to `Local`. +6. Inside a new directory create a `main.tf` file with the following contents. This will start an AWS instance with the NixOS image using one SSH keypair and an SSH security group: + +``` +terraform { + backend "remote" { + organization = "myorganization" + + workspaces { + name = "myapp" + } + } +} + +provider "aws" { + region = "eu-central-1" +} + +module "nixos_image" { + source = "git::https://github.com/tweag/terraform-nixos.git//aws_image_nixos?ref=5f5a0408b299874d6a29d1271e9bffeee4c9ca71" + release = "20.09" +} + +resource "aws_security_group" "ssh_and_egress" { + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = [ "0.0.0.0/0" ] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "tls_private_key" "state_ssh_key" { + algorithm = "RSA" +} + +resource "local_file" "machine_ssh_key" { + sensitive_content = tls_private_key.state_ssh_key.private_key_pem + filename = "${path.module}/id_rsa.pem" + file_permission = "0600" +} + +resource "aws_key_pair" "generated_key" { + key_name = "generated-key-${sha256(tls_private_key.state_ssh_key.public_key_openssh)}" + public_key = tls_private_key.state_ssh_key.public_key_openssh +} + +resource "aws_instance" "machine" { + ami = module.nixos_image.ami + instance_type = "t3.micro" + security_groups = [ aws_security_group.ssh_and_egress.name ] + key_name = aws_key_pair.generated_key.key_name + + root_block_device { + volume_size = 50 # GiB + } +} + +output "public_dns" { + value = aws_instance.machine.public_dns +} +``` + +The only NixOS specific snippet is: + +``` +module "nixos_image" { + source = "git::https://github.com/tweag/terraform-nixos.git/aws_image_nixos?ref=5f5a0408b299874d6a29d1271e9bffeee4c9ca71" + release = "20.09" +} +``` + +:::{note} +The `aws_image_nixos` module will return an NixOS AMI given a [NixOS release number](https://status.nixos.org) +so that `aws_instance` resource can reference the AMI in [instance_type](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/instance#instance_type) argument. +::: + +5. Make sure to [configure AWS credentials](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication). +6. Applying the Terraform configuration should get you a running NixOS: + +```shell +terraform init +terraform apply +``` + +## Deploying NixOS changes + +Once the AWS instance is running an NixOS image via Terraform, we can teach Terraform to always build +the latest NixOS configuration and apply those changes to your instance. + +1. Create `configuration.nix` with the following contents: + +```nix +{ config, lib, pkgs, ... }: { + imports = [ ]; + + # Open https://search.nixos.org/options for all options +} +``` + +2. Append the following snippet to your `main.tf`: + +``` +module "deploy_nixos" { + source = "git::https://github.com/tweag/terraform-nixos.git//deploy_nixos?ref=5f5a0408b299874d6a29d1271e9bffeee4c9ca71" + nixos_config = "${path.module}/configuration.nix" + target_host = aws_instance.machine.public_ip + ssh_private_key_file = local_file.machine_ssh_key.filename + ssh_agent = false +} +``` + +3. Deploy: + +```shell +terraform init +terraform apply +``` + +## Caveats + +- The `deploy_nixos` module requires NixOS to be installed on the target machine and Nix on the host machine. +- The `deploy_nixos` module doesn't work when the client and target architectures are different (unless you use [distributed builds](https://nixos.org/manual/nix/unstable/advanced-topics/distributed-builds.html)). +- If you need to inject a value into Nix, there is no elegant solution. +- Each machine is evaluated separately, so note that your memory requirements will grow linearly with the number of machines. + +## Next steps + +- It's possible to [switch to use Google Compute Engine provider](https://github.com/tweag/terraform-nixos/tree/master/google_image_nixos#readme). +- [deploy_nixos module](https://github.com/tweag/terraform-nixos/tree/master/deploy_nixos#readme) supports a number arguments, for example to upload keys, etc. diff --git a/source/tutorials/deploying-nixos-using-terraform.rst b/source/tutorials/deploying-nixos-using-terraform.rst deleted file mode 100644 index b0cedda..0000000 --- a/source/tutorials/deploying-nixos-using-terraform.rst +++ /dev/null @@ -1,186 +0,0 @@ -.. _deploying-nixos-using-terraform: - - -.. meta:: - :description: Continuous Integration with GitHub Actions and Cachix - :keywords: NixOS, deployment, Terraform, AWS - - -Deploying NixOS using Terraform -=============================== - -Assuming you're `familiar with the basics of Terraform `_, -by the end of tutorial you will have provisioned an Amazon AWS instance with Terraform -and will be able to use Nix to deploy incremental changes to NixOS, running on the instance. - -We'll look at how to boot a NixOS machine and how to deploy the incremental changes: - - -Booting NixOS image -------------------- - -1. Start by providing the terraform executable: - -.. code:: shell - - nix-shell -p terraform - -2. We are using `Terraform Cloud `_ as a `state/locking backend `_: - -.. code:: shell - - terraform login - -3. Make sure to `create an organization `_ like ``myorganization`` in your Terraform Cloud account. - -4. Inside ``myorganization`` `create a workspace `_ by choosing **CLI-driven workflow** and pick a name like ``myapp``. - -5. Inside your workspace, under ``Settings`` / ``General`` change Execution Mode to ``Local``. - -6. Inside a new directory create a ``main.tf`` file with the following contents. This will start an AWS instance with the NixOS image using one SSH keypair and an SSH security group: - -.. code:: - - terraform { - backend "remote" { - organization = "myorganization" - - workspaces { - name = "myapp" - } - } - } - - provider "aws" { - region = "eu-central-1" - } - - module "nixos_image" { - source = "git::https://github.com/tweag/terraform-nixos.git//aws_image_nixos?ref=5f5a0408b299874d6a29d1271e9bffeee4c9ca71" - release = "20.09" - } - - resource "aws_security_group" "ssh_and_egress" { - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = [ "0.0.0.0/0" ] - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } - } - - resource "tls_private_key" "state_ssh_key" { - algorithm = "RSA" - } - - resource "local_file" "machine_ssh_key" { - sensitive_content = tls_private_key.state_ssh_key.private_key_pem - filename = "${path.module}/id_rsa.pem" - file_permission = "0600" - } - - resource "aws_key_pair" "generated_key" { - key_name = "generated-key-${sha256(tls_private_key.state_ssh_key.public_key_openssh)}" - public_key = tls_private_key.state_ssh_key.public_key_openssh - } - - resource "aws_instance" "machine" { - ami = module.nixos_image.ami - instance_type = "t3.micro" - security_groups = [ aws_security_group.ssh_and_egress.name ] - key_name = aws_key_pair.generated_key.key_name - - root_block_device { - volume_size = 50 # GiB - } - } - - output "public_dns" { - value = aws_instance.machine.public_dns - } - -The only NixOS specific snippet is: - -.. code:: - - module "nixos_image" { - source = "git::https://github.com/tweag/terraform-nixos.git/aws_image_nixos?ref=5f5a0408b299874d6a29d1271e9bffeee4c9ca71" - release = "20.09" - } - -.. note:: - - The ``aws_image_nixos`` module will return an NixOS AMI given a `NixOS release number `_ - so that ``aws_instance`` resource can reference the AMI in `instance_type `_ argument. - -5. Make sure to `configure AWS credentials `_. - -6. Applying the Terraform configuration should get you a running NixOS: - -.. code:: shell - - terraform init - terraform apply - - -Deploying NixOS changes ------------------------ - -Once the AWS instance is running an NixOS image via Terraform, we can teach Terraform to always build -the latest NixOS configuration and apply those changes to your instance. - -1. Create ``configuration.nix`` with the following contents: - -.. code:: nix - - { config, lib, pkgs, ... }: { - imports = [ ]; - - # Open https://search.nixos.org/options for all options - } - -2. Append the following snippet to your ``main.tf``: - -.. code:: - - module "deploy_nixos" { - source = "git::https://github.com/tweag/terraform-nixos.git//deploy_nixos?ref=5f5a0408b299874d6a29d1271e9bffeee4c9ca71" - nixos_config = "${path.module}/configuration.nix" - target_host = aws_instance.machine.public_ip - ssh_private_key_file = local_file.machine_ssh_key.filename - ssh_agent = false - } - -3. Deploy: - -.. code:: shell - - terraform init - terraform apply - - -Caveats -------- - -- The ``deploy_nixos`` module requires NixOS to be installed on the target machine and Nix on the host machine. - -- The ``deploy_nixos`` module doesn't work when the client and target architectures are different (unless you use `distributed builds `_). - -- If you need to inject a value into Nix, there is no elegant solution. - -- Each machine is evaluated separately, so note that your memory requirements will grow linearly with the number of machines. - - -Next steps ----------- - -- It's possible to `switch to use Google Compute Engine provider `_. - -- `deploy_nixos module `_ supports a number arguments, for example to upload keys, etc. diff --git a/source/tutorials/dev-environment.md b/source/tutorials/dev-environment.md new file mode 100644 index 0000000..c40d9df --- /dev/null +++ b/source/tutorials/dev-environment.md @@ -0,0 +1,82 @@ +# Setup a development environment + +As an exercise, let us build a Python web application using the Flask +web framework. + +Create a new file `default.nix`. This file is conventionally used for +specifying packages: + +```nix +{ pkgs ? import {} }: + +pkgs.python3Packages.buildPythonApplication { + pname = "myapp"; + src = ./.; + version = "0.1"; + propagatedBuildInputs = [ pkgs.python3Packages.flask ]; +} +``` + +You will also need a simple Flask app as `myapp.py`: + +```python +#! /usr/bin/env python + +from flask import Flask + +app = Flask(__name__) + +@app.route("/") +def hello(): + return "Hello, Nix!" + +def run(): + app.run(host="0.0.0.0") + +if __name__ == "__main__": + run() +``` + +and a `setup.py` script: + +```python +from setuptools import setup + +setup( + name='myapp', + version='0.1', + py_modules=['myapp'], + entry_points={ + 'console_scripts': ['myapp = myapp:run'] + }, +) +``` + +Now build the package with: + +```bash +nix-build +``` + +This will create a symbolic link `result` to our package's path in the +Nix store, which looks like +`/nix/store/6i4l781jwk5vbia8as32637207kgkllj-myapp-0.1`. Look around +to see what is inside. + +You may notice we can run the application from the package like +`./result/bin/myapp.py`. We can still use the `default.nix` as a +shell environment to get the same result: + +```bash +nix-shell default.nix +python3 myapp.py +``` + +In this context, Nix takes on the role that you would otherwise use pip +or virtualenv for. Nix installs required dependencies and separates the +environment from others on your system. + +You can check this Nix configuration into version control and share it +with others to make sure you are all running the same software. +Especially with many dependencies this is a great way to prevent +configuration drift between different team members & contributors. diff --git a/source/tutorials/dev-environment.rst b/source/tutorials/dev-environment.rst deleted file mode 100644 index 38fb27c..0000000 --- a/source/tutorials/dev-environment.rst +++ /dev/null @@ -1,83 +0,0 @@ -Setup a development environment -=============================== - -As an exercise, let us build a Python web application using the Flask -web framework. - -Create a new file ``default.nix``. This file is conventionally used for -specifying packages: - -.. code:: nix - - { pkgs ? import {} }: - - pkgs.python3Packages.buildPythonApplication { - pname = "myapp"; - src = ./.; - version = "0.1"; - propagatedBuildInputs = [ pkgs.python3Packages.flask ]; - } - -You will also need a simple Flask app as ``myapp.py``: - -.. code:: python - - #! /usr/bin/env python - - from flask import Flask - - app = Flask(__name__) - - @app.route("/") - def hello(): - return "Hello, Nix!" - - def run(): - app.run(host="0.0.0.0") - - if __name__ == "__main__": - run() - -and a ``setup.py`` script: - -.. code:: python - - from setuptools import setup - - setup( - name='myapp', - version='0.1', - py_modules=['myapp'], - entry_points={ - 'console_scripts': ['myapp = myapp:run'] - }, - ) - -Now build the package with: - -.. code:: bash - - nix-build - -This will create a symbolic link ``result`` to our package's path in the -Nix store, which looks like -``/nix/store/6i4l781jwk5vbia8as32637207kgkllj-myapp-0.1``. Look around -to see what is inside. - -You may notice we can run the application from the package like -``./result/bin/myapp.py``. We can still use the ``default.nix`` as a -shell environment to get the same result: - -.. code:: bash - - nix-shell default.nix - python3 myapp.py - -In this context, Nix takes on the role that you would otherwise use pip -or virtualenv for. Nix installs required dependencies and separates the -environment from others on your system. - -You can check this Nix configuration into version control and share it -with others to make sure you are all running the same software. -Especially with many dependencies this is a great way to prevent -configuration drift between different team members & contributors. diff --git a/source/tutorials/index.md b/source/tutorials/index.md new file mode 100644 index 0000000..13faaa6 --- /dev/null +++ b/source/tutorials/index.md @@ -0,0 +1,19 @@ +# Tutorials + +```{toctree} +:glob: true + +install-nix.rst +ad-hoc-developer-environments.rst +towards-reproducibility-pinning-nixpkgs.rst +declarative-and-reproducible-developer-environments.rst +continuous-integration-github-actions.rst +dev-environment.rst +building-and-running-docker-images.rst +building-bootable-iso-image.rst +deploying-nixos-using-terraform.rst +installing-nixos-on-a-raspberry-pi.rst +integration-testing-using-virtual-machines.rst +cross-compilation.rst +contributing.rst +``` diff --git a/source/tutorials/index.rst b/source/tutorials/index.rst deleted file mode 100644 index c5e3d50..0000000 --- a/source/tutorials/index.rst +++ /dev/null @@ -1,19 +0,0 @@ -Tutorials -========= - -.. toctree:: - :glob: - - install-nix.rst - ad-hoc-developer-environments.rst - towards-reproducibility-pinning-nixpkgs.rst - declarative-and-reproducible-developer-environments.rst - continuous-integration-github-actions.rst - dev-environment.rst - building-and-running-docker-images.rst - building-bootable-iso-image.rst - deploying-nixos-using-terraform.rst - installing-nixos-on-a-raspberry-pi.rst - integration-testing-using-virtual-machines.rst - cross-compilation.rst - contributing.rst diff --git a/source/tutorials/install-nix.md b/source/tutorials/install-nix.md new file mode 100644 index 0000000..1b601f3 --- /dev/null +++ b/source/tutorials/install-nix.md @@ -0,0 +1,74 @@ +(install-nix)= + +# Install Nix + +## Linux + +Install Nix on via the recommended [multi-user installation](https://nixos.org/manual/nix/stable/installation/multi-user.html): + +```bash +sh <(curl -L https://nixos.org/nix/install) --daemon +``` + +:::{note} +For security you may want to [verify the installation script] using GPG signatures. +::: + +## macOS + +Install Nix via the recommended [multi-user installation](https://nixos.org/manual/nix/stable/installation/multi-user.html): + +```bash +sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume --daemon +``` + +:::{note} +For security you may want to [verify the installation script] using GPG signatures. +::: + +## Windows (WSL2) + +Install Nix via the recommended [single-user installation](https://nixos.org/manual/nix/stable/installation/single-user.html): + +```bash +sh <(curl -L https://nixos.org/nix/install) --no-daemon +``` + +:::{note} +For security you may want to [verify the installation script] using GPG signatures. +::: + +## Docker + +Start a Docker shell with Nix: + +```bash +$ docker run -it nixos/nix +``` + +Or start a Docker shell with Nix exposing a `workdir` directory: + +```bash +$ mkdir workdir +$ docker run -it -v $(pwd)/workdir:/workdir nixos/nix +``` + +The `workdir` example from above can be also used to start hacking on nixpkgs: + +```bash +$ git clone git@github.com:NixOS/nixpkgs +$ docker run -it -v $(pwd)/nixpkgs:/nixpkgs nixos/nix +docker> nix-build -I nixpkgs=/nixpkgs -A hello +docker> find ./result # this symlink points to the build package +``` + +## Verify installation + +Check that the installation by opening **a new terminal** and typing: + +```bash +$ nix-env --version +nix-env (Nix) 2.3.15 +``` + +[verify the installation script]: https://nixos.org/download.html#nix-verify-installation diff --git a/source/tutorials/install-nix.rst b/source/tutorials/install-nix.rst deleted file mode 100644 index dd48680..0000000 --- a/source/tutorials/install-nix.rst +++ /dev/null @@ -1,85 +0,0 @@ -.. _install-nix: - -Install Nix -=========== - -Linux ------ - -Install Nix on via the recommended `multi-user installation `_: - -.. code:: bash - - sh <(curl -L https://nixos.org/nix/install) --daemon - -.. note:: - - For security you may want to `verify the installation script`_ using GPG signatures. - - -macOS ------ - -Install Nix via the recommended `multi-user installation `_: - -.. code:: bash - - sh <(curl -L https://nixos.org/nix/install) --darwin-use-unencrypted-nix-store-volume --daemon - - -.. note:: - - For security you may want to `verify the installation script`_ using GPG signatures. - - -Windows (WSL2) --------------- - -Install Nix via the recommended `single-user installation `_: - -.. code:: bash - - sh <(curl -L https://nixos.org/nix/install) --no-daemon - -.. note:: - - For security you may want to `verify the installation script`_ using GPG signatures. - - -Docker ------- - -Start a Docker shell with Nix: - -.. code:: bash - - $ docker run -it nixos/nix - -Or start a Docker shell with Nix exposing a ``workdir`` directory: - -.. code:: bash - - $ mkdir workdir - $ docker run -it -v $(pwd)/workdir:/workdir nixos/nix - -The ``workdir`` example from above can be also used to start hacking on nixpkgs: - -.. code:: bash - - $ git clone git@github.com:NixOS/nixpkgs - $ docker run -it -v $(pwd)/nixpkgs:/nixpkgs nixos/nix - docker> nix-build -I nixpkgs=/nixpkgs -A hello - docker> find ./result # this symlink points to the build package - -Verify installation -------------------- - -Check that the installation by opening **a new terminal** and typing: - - -.. code:: bash - - $ nix-env --version - nix-env (Nix) 2.3.15 - -.. _verify the installation script: https://nixos.org/download.html#nix-verify-installation diff --git a/source/tutorials/installing-nixos-on-a-raspberry-pi.md b/source/tutorials/installing-nixos-on-a-raspberry-pi.md new file mode 100644 index 0000000..111c71f --- /dev/null +++ b/source/tutorials/installing-nixos-on-a-raspberry-pi.md @@ -0,0 +1,189 @@ +--- +html_meta: + "description lang=en": "Installing NixOS on a Raspberry Pi" + "keywords": "Raspberry Pi, rpi, NixOS, installation, image, tutorial" +--- + + +# Installing NixOS on a Raspberry Pi + +This tutorial assumes [Raspberry P 4 Model B with 4GB RAM](https://www.raspberrypi.org/products/raspberry-pi-4-model-b/). + +Before starting this tutorial, make sure you have +[all necessary hardware](https://projects.raspberrypi.org/en/projects/raspberry-pi-setting-up/1): + +- HDMI cable/adapter. +- 8GB+ SD card. +- SD card reader in case your machine doesn't have an SD slot. +- Power cable for your Raspberry Pi. +- USB keyboard. + +:::{note} +This tutorial was written for the Raspberry Pi 4B. Using a previous supported revision, like the 3B or 3B+, is possible with tweaks. +::: + +## Booting NixOS live image + +:::{note} +Booting from USB may require an EEPROM firmware upgrade. This tutorial boots from an SD card to avoid such hiccups. +::: + +Prepare the AArch64 image on your laptop: + +```shell-session +$ nix-shell -p wget zstd +$ wget https://hydra.nixos.org/build/160738647/download/1/nixos-sd-image-22.05pre335501.c71f061c68b-aarch64-linux.img.zst +$ unzstd -d nixos-sd-image-22.05pre335501.c71f061c68b-aarch64-linux.img.zst +$ dmesg --follow +``` + +:::{note} +You can pick a newer image by going to [Hydra job](https://hydra.nixos.org/job/nixos/trunk-combined/nixos.sd_image.aarch64-linux), +clicking on a build and copying the link to the build product image. +::: + +Your terminal should be printing kernel messages as they come in. + +Plug in your SD card and your terminal should print what device it got assigned, for example `/dev/sdX`. + +Press `ctrl-c` to stop `dmesg --follow`. + +Copy NixOS to your SD card by replacing `sdX` with the name of your device: + +```shell-session +sudo dd if=nixos-sd-image-22.05pre335501.c71f061c68b-aarch64-linux.img of=/dev/sdX bs=4096 conv=fsync status=progress +``` + +Once that command exits, **move the SD card into your Raspberry Pi and power it on**. + +You should be greeted with a fresh shell! + +In case the image doesn't boot, it's worth [updating the firmware](https://www.raspberrypi.org/documentation/computers/raspberry-pi.html#updating-the-bootloader) +and retry booting the image again. + +## Getting internet connection + +Run `sudo -i` to get a root shell for the rest of the tutorial. + +At this point we'll need internet connection. If you can use an ethernet cable, plug it in. + +In case you're connecting to a wifi run `iwconfig` to see what is the name of your wireless +network interface. In case it's `wlan0` replace `SSID` and `passphrase` with your data and run: + +```shell-session +# wpa_supplicant -B -i wlan0 -c <(wpa_passphrase 'SSID' 'passphrase') & +``` + +Once you see in your terminal that connection is established, run `host nixos.org` to +check that DNS resolves correctly. + +In case you've made a typo, run `pkill wpa_supplicant` and start over. + +## Updating firmware + +To benefit from updates and bug fixes from the vendor, we'll start by updating Raspberry Pi firmware: + +```shell-session +# nix-shell -p raspberrypi-eeprom +# mount /dev/disk/by-label/FIRMWARE /mnt +# BOOTFS=/mnt FIRMWARE_RELEASE_STATUS=stable rpi-eeprom-update -d -a +``` + +## Installing NixOS + +For initial installation we'll install [XFCE](https://www.xfce.org/) desktop environment +with user `guest` and SSH daemon. + +```nix +{ config, pkgs, lib, ... }: + +let + user = "guest"; + password = "guest"; + SSID = "mywifi"; + SSIDpassword = "mypassword"; + interface = "wlan0"; + hostname = "myhostname"; +in { + imports = ["${fetchTarball "https://github.com/NixOS/nixos-hardware/archive/936e4649098d6a5e0762058cb7687be1b2d90550.tar.gz" }/raspberry-pi/4"]; + + fileSystems = { + "/" = { + device = "/dev/disk/by-label/NIXOS_SD"; + fsType = "ext4"; + options = [ "noatime" ]; + }; + }; + + networking = { + hostName = hostname; + wireless = { + enable = true; + networks."${SSID}".psk = SSIDpassword; + interfaces = [ interface ]; + }; + }; + + environment.systemPackages = with pkgs; [ vim ]; + + services.openssh.enable = true; + + users = { + mutableUsers = false; + users."${user}" = { + isNormalUser = true; + password = password; + extraGroups = [ "wheel" ]; + }; + }; + + # Enable GPU acceleration + hardware.raspberry-pi."4".fkms-3d.enable = true; + + services.xserver = { + enable = true; + displayManager.lightdm.enable = true; + desktopManager.xfce.enable = true; + }; + + hardware.pulseaudio.enable = true; +} +``` + +To save time on typing the whole configuration, download it: + +```shell-session +# curl -L https://tinyurl.com/nixos-rpi4-tutorial > /etc/nixos/configuration.nix +``` + +At the top of `/etc/nixos/configuration.nix` there are a few variables that you want to configure, +most important being your wifi connection details, this time specified in declarative way. + +Once you're ready to install NixOS: + +```shell-session +# nixos-install --root / +# reboot +``` + +In case your system doesn't boot, select the oldest configuration in the bootloader menu to get back to live image and start over. + +## Making changes + +It booted, congratulations! + +To make further changes to the configuration, [search through NixOS options](https://search.nixos.org/options), +edit `/etc/nixos/configuration.nix` and update your system: + +```shell-session +$ sudo -i +# nixos-rebuild switch +``` + +## Next steps + +- Once you have successfully running OS, try upgrading it with `nixos-rebuild switch --upgrade` + and reboot to the old configuration if something broke. +- To tweak bootloader options affecting hardware, [see config.txt options](https://www.raspberrypi.org/documentation/configuration/config-txt/) + and change the options by running `mount /dev/disk/by-label/FIRMWARE /mnt` and opening `/mnt/config.txt`. +- To see the power of declarative configuration, try replacing `xfce` with `kodi` in `/etc/nixos/configuration.nix` and `reboot`. diff --git a/source/tutorials/installing-nixos-on-a-raspberry-pi.rst b/source/tutorials/installing-nixos-on-a-raspberry-pi.rst deleted file mode 100644 index 590e093..0000000 --- a/source/tutorials/installing-nixos-on-a-raspberry-pi.rst +++ /dev/null @@ -1,199 +0,0 @@ -Installing NixOS on a Raspberry Pi -================================== - -.. meta:: - :description: Installing NixOS on a Raspberry Pi - :keywords: Raspberry Pi, rpi, NixOS, installation, image, tutorial - -This tutorial assumes `Raspberry P 4 Model B with 4GB RAM `_. - -Before starting this tutorial, make sure you have -`all necessary hardware `_: - -- HDMI cable/adapter. -- 8GB+ SD card. -- SD card reader in case your machine doesn't have an SD slot. -- Power cable for your Raspberry Pi. -- USB keyboard. - -.. note:: - - This tutorial was written for the Raspberry Pi 4B. Using a previous supported revision, like the 3B or 3B+, is possible with tweaks. - - -Booting NixOS live image ------------------------- - -.. note:: Booting from USB may require an EEPROM firmware upgrade. This tutorial boots from an SD card to avoid such hiccups. - -Prepare the AArch64 image on your laptop: - -.. code:: shell-session - - $ nix-shell -p wget zstd - $ wget https://hydra.nixos.org/build/160738647/download/1/nixos-sd-image-22.05pre335501.c71f061c68b-aarch64-linux.img.zst - $ unzstd -d nixos-sd-image-22.05pre335501.c71f061c68b-aarch64-linux.img.zst - $ dmesg --follow - -.. note:: - You can pick a newer image by going to `Hydra job `_, - clicking on a build and copying the link to the build product image. - -Your terminal should be printing kernel messages as they come in. - -Plug in your SD card and your terminal should print what device it got assigned, for example ``/dev/sdX``. - -Press ``ctrl-c`` to stop ``dmesg --follow``. - -Copy NixOS to your SD card by replacing ``sdX`` with the name of your device: - -.. code:: shell-session - - sudo dd if=nixos-sd-image-22.05pre335501.c71f061c68b-aarch64-linux.img of=/dev/sdX bs=4096 conv=fsync status=progress - -Once that command exits, **move the SD card into your Raspberry Pi and power it on**. - -You should be greeted with a fresh shell! - -In case the image doesn't boot, it's worth `updating the firmware `_ -and retry booting the image again. - - -Getting internet connection ---------------------------- - -Run ``sudo -i`` to get a root shell for the rest of the tutorial. - -At this point we'll need internet connection. If you can use an ethernet cable, plug it in. - -In case you're connecting to a wifi run ``iwconfig`` to see what is the name of your wireless -network interface. In case it's ``wlan0`` replace ``SSID`` and ``passphrase`` with your data and run: - -.. code:: shell-session - - # wpa_supplicant -B -i wlan0 -c <(wpa_passphrase 'SSID' 'passphrase') & - - -Once you see in your terminal that connection is established, run ``host nixos.org`` to -check that DNS resolves correctly. - -In case you've made a typo, run ``pkill wpa_supplicant`` and start over. - - -Updating firmware ------------------ - -To benefit from updates and bug fixes from the vendor, we'll start by updating Raspberry Pi firmware: - -.. code:: shell-session - - # nix-shell -p raspberrypi-eeprom - # mount /dev/disk/by-label/FIRMWARE /mnt - # BOOTFS=/mnt FIRMWARE_RELEASE_STATUS=stable rpi-eeprom-update -d -a - - -Installing NixOS ----------------- - -For initial installation we'll install `XFCE `_ desktop environment -with user ``guest`` and SSH daemon. - -.. code:: nix - - { config, pkgs, lib, ... }: - - let - user = "guest"; - password = "guest"; - SSID = "mywifi"; - SSIDpassword = "mypassword"; - interface = "wlan0"; - hostname = "myhostname"; - in { - imports = ["${fetchTarball "https://github.com/NixOS/nixos-hardware/archive/936e4649098d6a5e0762058cb7687be1b2d90550.tar.gz" }/raspberry-pi/4"]; - - fileSystems = { - "/" = { - device = "/dev/disk/by-label/NIXOS_SD"; - fsType = "ext4"; - options = [ "noatime" ]; - }; - }; - - networking = { - hostName = hostname; - wireless = { - enable = true; - networks."${SSID}".psk = SSIDpassword; - interfaces = [ interface ]; - }; - }; - - environment.systemPackages = with pkgs; [ vim ]; - - services.openssh.enable = true; - - users = { - mutableUsers = false; - users."${user}" = { - isNormalUser = true; - password = password; - extraGroups = [ "wheel" ]; - }; - }; - - # Enable GPU acceleration - hardware.raspberry-pi."4".fkms-3d.enable = true; - - services.xserver = { - enable = true; - displayManager.lightdm.enable = true; - desktopManager.xfce.enable = true; - }; - - hardware.pulseaudio.enable = true; - } - -To save time on typing the whole configuration, download it: - -.. code:: shell-session - - # curl -L https://tinyurl.com/nixos-rpi4-tutorial > /etc/nixos/configuration.nix - -At the top of `/etc/nixos/configuration.nix` there are a few variables that you want to configure, -most important being your wifi connection details, this time specified in declarative way. - -Once you're ready to install NixOS: - -.. code:: shell-session - - # nixos-install --root / - # reboot - -In case your system doesn't boot, select the oldest configuration in the bootloader menu to get back to live image and start over. - - -Making changes --------------- - -It booted, congratulations! - -To make further changes to the configuration, `search through NixOS options `_, -edit ``/etc/nixos/configuration.nix`` and update your system: - -.. code:: shell-session - - $ sudo -i - # nixos-rebuild switch - - -Next steps ----------- - -- Once you have successfully running OS, try upgrading it with `nixos-rebuild switch --upgrade` - and reboot to the old configuration if something broke. - -- To tweak bootloader options affecting hardware, `see config.txt options `_ - and change the options by running ``mount /dev/disk/by-label/FIRMWARE /mnt`` and opening ``/mnt/config.txt``. - -- To see the power of declarative configuration, try replacing ``xfce`` with ``kodi`` in ``/etc/nixos/configuration.nix`` and ``reboot``. diff --git a/source/tutorials/integration-testing-using-virtual-machines.md b/source/tutorials/integration-testing-using-virtual-machines.md new file mode 100644 index 0000000..3d99440 --- /dev/null +++ b/source/tutorials/integration-testing-using-virtual-machines.md @@ -0,0 +1,218 @@ +(integration-testing-vms)= + +# Integration testing using virtual machines (VMs) + +One of the most powerful features in the Nix ecosystem is **the ability +to provide a set of declarative NixOS configurations and use a simple +Python interface** to interact with them using [QEMU](https://www.qemu.org/) +as the backend. + +Those tests are widely used to ensure that NixOS works as intended, so in general they are called **NixOS tests**. +They can be written and launched outside of NixOS, on any Linux machine (with +[MacOS support coming soon](https://github.com/NixOS/nixpkgs/issues/108984)). + +Integration tests are reproducible due to the design properties of Nix, +making them a valuable part of a Continuous Integration (CI) pipeline. + +## Testing a typical web application backed by PostgreSQL + +This tutorial follows [PostgREST tutorial](https://postgrest.org/en/stable/tutorials/tut0.html), +a generic [RESTful API](https://restfulapi.net/) for PostgreSQL. + +If you skim over the official tutorial, you'll notice there's quite a bit of setup +in order to test if all the steps work. + +We are going to set up: + +- A VM named `server` running postgreSQL and postgREST. +- A VM named `client` running HTTP client queries using `curl`. +- A `testScript` orchestrating testing logic between `client` and `server`. + +## Writing the test + +Create `postgrest.nix`: + +% TODO: highlight nix https://github.com/pygments/pygments/issues/1793 + +```{code-block} +:linenos: true + +let + # Pin nixpkgs, see pinning tutorial for more details + nixpkgs = fetchTarball "https://github.com/NixOS/nixpkgs/archive/0f8f64b54ed07966b83db2f20c888d5e035012ef.tar.gz"; + pkgs = import nixpkgs {}; + + # Single source of truth for all tutorial constants + database = "postgres"; + schema = "api"; + table = "todos"; + username = "authenticator"; + password = "mysecretpassword"; + webRole = "web_anon"; + postgrestPort = 3000; + + # NixOS module shared between server and client + sharedModule = { + # Since it's common for CI not to have $DISPLAY available, we have to explicitly tell the tests "please don't expect any screen available" + virtualisation.graphics = false; + }; + +in pkgs.nixosTest ({ + # NixOS tests are run inside a virtual machine, and here we specify system of the machine. + system = "x86_64-linux"; + + nodes = { + server = { config, pkgs, ... }: { + imports = [ sharedModule ]; + + networking.firewall.allowedTCPPorts = [ postgrestPort ]; + + services.postgresql = { + enable = true; + + initialScript = pkgs.writeText "initialScript.sql" '' + create schema ${schema}; + + create table ${schema}.${table} ( + id serial primary key, + done boolean not null default false, + task text not null, + due timestamptz + ); + + insert into ${schema}.${table} (task) values ('finish tutorial 0'), ('pat self on back'); + + create role ${webRole} nologin; + grant usage on schema ${schema} to ${webRole}; + grant select on ${schema}.${table} to ${webRole}; + + create role ${username} inherit login password '${password}'; + grant ${webRole} to ${username}; + ''; + }; + + users = { + mutableUsers = false; + users = { + # For ease of debugging the VM as the `root` user + root.password = ""; + + # Create a system user that matches the database user so that we + # can use peer authentication. The tutorial defines a password, + # but it's not necessary. + "${username}".isSystemUser = true; + }; + }; + + systemd.services.postgrest = { + wantedBy = [ "multi-user.target" ]; + after = [ "postgresql.service" ]; + script = + let + configuration = pkgs.writeText "tutorial.conf" '' + db-uri = "postgres://${username}:${password}@localhost:${toString config.services.postgresql.port}/${database}" + db-schema = "${schema}" + db-anon-role = "${username}" + ''; + in "${pkgs.haskellPackages.postgrest}/bin/postgrest ${configuration}"; + serviceConfig.User = username; + }; + }; + + client = { + imports = [ sharedModule ]; + }; + }; + + # Disable linting for simpler debugging of the testScript + skipLint = true; + + testScript = '' + import json + import sys + + start_all() + + server.wait_for_open_port(${toString postgrestPort}) + + expected = [ + {"id": 1, "done": False, "task": "finish tutorial 0", "due": None}, + {"id": 2, "done": False, "task": "pat self on back", "due": None}, + ] + + actual = json.loads( + client.succeed( + "${pkgs.curl}/bin/curl http://server:${toString postgrestPort}/${table}" + ) + ) + + assert expected == actual, "table query returns expected content" + ''; +}) +``` + +A few notes: + +- Between the machines defined inside the `nodes` attribute, hostnames + are resolved based on their attribute names. In this case we have `client` and `server`. +- The testing framework exposes a wide set of operations used inside the `testScript`. + A full set of testing operations is part of + [VM testing operations API Reference](https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests). + +## Running tests + +To set up all machines and execute the test script: + +```shell-session +$ nix-build postgrest.nix +``` + +You'll notice an error message if something goes wrong. + +In case the tests succeed, you should see at the end: + +```shell-session +... +test script finished in 10.96s +cleaning up +killing client (pid 10) +killing server (pid 22) +(0.00 seconds) +/nix/store/bx7z3imvxxpwkkza10vb23czhw7873w2-vm-test-run-unnamed +``` + +## Developing and debugging tests + +When developing tests or when something breaks, it's useful to interactively fiddle +with the script or access a terminal for a machine. + +To interactively start a Python session with a testing framework: + +```shell-session +$ $(nix-build -A driver postgrest.nix)/bin/nixos-test-driver +... +starting VDE switch for network 1 +>>> +``` + +You can run [any of the testing operations](https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests). +The `testScript` attribute from our `postgrest.nix` definition can be executed with `test_script()` function. + +To start all machines and enter a telnet terminal to a specific machine: + +```shell-session +>>> start_all() +... +>>> server.shell_interact() +server: Terminal is ready (there is no prompt): + +uname -a +Linux server 5.10.37 #1-NixOS SMP Fri May 14 07:50:46 UTC 2021 x86_64 GNU/Linux +``` + +## Next steps + +- Running integration tests on CI requires hardware acceleration, which many CIs do not support. + To run integration tests on {ref}`GitHub Actions ` see + [how to disable hardware acceleration](https://github.com/cachix/install-nix-action#how-can-i-run-nixos-tests). +- NixOS comes with a large set of tests that serve also as educational examples. A good inspiration is [Matrix bridging with an IRC](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/matrix-appservice-irc.nix). diff --git a/source/tutorials/integration-testing-using-virtual-machines.rst b/source/tutorials/integration-testing-using-virtual-machines.rst deleted file mode 100644 index 318b56c..0000000 --- a/source/tutorials/integration-testing-using-virtual-machines.rst +++ /dev/null @@ -1,233 +0,0 @@ -.. _integration-testing-vms: - -Integration testing using virtual machines (VMs) -================================================ - -One of the most powerful features in the Nix ecosystem is **the ability -to provide a set of declarative NixOS configurations and use a simple -Python interface** to interact with them using `QEMU `_ -as the backend. - -Those tests are widely used to ensure that NixOS works as intended, so in general they are called **NixOS tests**. -They can be written and launched outside of NixOS, on any Linux machine (with -`MacOS support coming soon `_). - -Integration tests are reproducible due to the design properties of Nix, -making them a valuable part of a Continuous Integration (CI) pipeline. - - -Testing a typical web application backed by PostgreSQL ------------------------------------------------------- - -This tutorial follows `PostgREST tutorial `_, -a generic `RESTful API `_ for PostgreSQL. - -If you skim over the official tutorial, you'll notice there's quite a bit of setup -in order to test if all the steps work. - -We are going to set up: - -- A VM named ``server`` running postgreSQL and postgREST. - -- A VM named ``client`` running HTTP client queries using ``curl``. - -- A ``testScript`` orchestrating testing logic between ``client`` and ``server``. - - -Writing the test ----------------- - -Create ``postgrest.nix``: - -.. TODO: highlight nix https://github.com/pygments/pygments/issues/1793 - - -.. code-block:: - :linenos: - - let - # Pin nixpkgs, see pinning tutorial for more details - nixpkgs = fetchTarball "https://github.com/NixOS/nixpkgs/archive/0f8f64b54ed07966b83db2f20c888d5e035012ef.tar.gz"; - pkgs = import nixpkgs {}; - - # Single source of truth for all tutorial constants - database = "postgres"; - schema = "api"; - table = "todos"; - username = "authenticator"; - password = "mysecretpassword"; - webRole = "web_anon"; - postgrestPort = 3000; - - # NixOS module shared between server and client - sharedModule = { - # Since it's common for CI not to have $DISPLAY available, we have to explicitly tell the tests "please don't expect any screen available" - virtualisation.graphics = false; - }; - - in pkgs.nixosTest ({ - # NixOS tests are run inside a virtual machine, and here we specify system of the machine. - system = "x86_64-linux"; - - nodes = { - server = { config, pkgs, ... }: { - imports = [ sharedModule ]; - - networking.firewall.allowedTCPPorts = [ postgrestPort ]; - - services.postgresql = { - enable = true; - - initialScript = pkgs.writeText "initialScript.sql" '' - create schema ${schema}; - - create table ${schema}.${table} ( - id serial primary key, - done boolean not null default false, - task text not null, - due timestamptz - ); - - insert into ${schema}.${table} (task) values ('finish tutorial 0'), ('pat self on back'); - - create role ${webRole} nologin; - grant usage on schema ${schema} to ${webRole}; - grant select on ${schema}.${table} to ${webRole}; - - create role ${username} inherit login password '${password}'; - grant ${webRole} to ${username}; - ''; - }; - - users = { - mutableUsers = false; - users = { - # For ease of debugging the VM as the `root` user - root.password = ""; - - # Create a system user that matches the database user so that we - # can use peer authentication. The tutorial defines a password, - # but it's not necessary. - "${username}".isSystemUser = true; - }; - }; - - systemd.services.postgrest = { - wantedBy = [ "multi-user.target" ]; - after = [ "postgresql.service" ]; - script = - let - configuration = pkgs.writeText "tutorial.conf" '' - db-uri = "postgres://${username}:${password}@localhost:${toString config.services.postgresql.port}/${database}" - db-schema = "${schema}" - db-anon-role = "${username}" - ''; - in "${pkgs.haskellPackages.postgrest}/bin/postgrest ${configuration}"; - serviceConfig.User = username; - }; - }; - - client = { - imports = [ sharedModule ]; - }; - }; - - # Disable linting for simpler debugging of the testScript - skipLint = true; - - testScript = '' - import json - import sys - - start_all() - - server.wait_for_open_port(${toString postgrestPort}) - - expected = [ - {"id": 1, "done": False, "task": "finish tutorial 0", "due": None}, - {"id": 2, "done": False, "task": "pat self on back", "due": None}, - ] - - actual = json.loads( - client.succeed( - "${pkgs.curl}/bin/curl http://server:${toString postgrestPort}/${table}" - ) - ) - - assert expected == actual, "table query returns expected content" - ''; - }) - -A few notes: - -- Between the machines defined inside the ``nodes`` attribute, hostnames - are resolved based on their attribute names. In this case we have ``client`` and ``server``. - -- The testing framework exposes a wide set of operations used inside the ``testScript``. - A full set of testing operations is part of - `VM testing operations API Reference `_. - - -Running tests -------------- - -To set up all machines and execute the test script: - -.. code:: shell-session - - $ nix-build postgrest.nix - -You'll notice an error message if something goes wrong. - -In case the tests succeed, you should see at the end: - -.. code:: shell-session - - ... - test script finished in 10.96s - cleaning up - killing client (pid 10) - killing server (pid 22) - (0.00 seconds) - /nix/store/bx7z3imvxxpwkkza10vb23czhw7873w2-vm-test-run-unnamed - - -Developing and debugging tests ------------------------------- - -When developing tests or when something breaks, it's useful to interactively fiddle -with the script or access a terminal for a machine. - -To interactively start a Python session with a testing framework: - -.. code:: shell-session - - $ $(nix-build -A driver postgrest.nix)/bin/nixos-test-driver - ... - starting VDE switch for network 1 - >>> - -You can run `any of the testing operations `_. -The ``testScript`` attribute from our ``postgrest.nix`` definition can be executed with ``test_script()`` function. - -To start all machines and enter a telnet terminal to a specific machine: - -.. code:: shell-session - - >>> start_all() - ... - >>> server.shell_interact() - server: Terminal is ready (there is no prompt): - - uname -a - Linux server 5.10.37 #1-NixOS SMP Fri May 14 07:50:46 UTC 2021 x86_64 GNU/Linux - - -Next steps ----------- - -- Running integration tests on CI requires hardware acceleration, which many CIs do not support. - To run integration tests on :ref:`GitHub Actions ` see - `how to disable hardware acceleration `_. - -- NixOS comes with a large set of tests that serve also as educational examples. A good inspiration is `Matrix bridging with an IRC `_. diff --git a/source/tutorials/towards-reproducibility-pinning-nixpkgs.md b/source/tutorials/towards-reproducibility-pinning-nixpkgs.md new file mode 100644 index 0000000..0e4b770 --- /dev/null +++ b/source/tutorials/towards-reproducibility-pinning-nixpkgs.md @@ -0,0 +1,90 @@ +(pinning-nixpkgs)= + +# Towards reproducibility: Pinning nixpkgs + +In various Nix examples, you'll often see references to [\](https://github.com/NixOS/nixpkgs), as follows. + +```nix +{ pkgs ? import {} +}: + +... +``` + +This is **convenient** to quickly demonstrate a Nix expression and get it working by importing Nix packages. + +However, the resulting Nix expression **is not fully reproducible**. The `` reference +is set from the **local** `$NIX_PATH` environment variable. In most cases, this is set at the time Nix is installed +to the `nixpkgs-unstable` channel, and therefore it is likely to differ from machine to machine. + +:::{note} +[Channels](https://nixos.wiki/wiki/Nix_channels) are a way of distributing Nix software, but they are being phased out. +Even though they are still used by default, it is recommended to avoid channels +and `` by always setting `NIX_PATH=` to be empty. +::: + +## Pinning packages with URLs inside a Nix expression + +To create **fully reproducible** Nix expressions, we can pin an exact versions of nixpkgs. + +The simplest way to do this is to fetch the required nixpkgs version as a tarball specified via the relevant git commit hash: + +```nix +{ pkgs ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/3590f02e7d5760e52072c1a729ee2250b5560746.tar.gz") {} +}: + +... +``` + +Picking the commit can be done via [status.nixos.org](https://status.nixos.org/), +which lists all the releases and the latest commit that has passed all tests. + +When choosing a commit, it is recommended to follow either + +- the **latest stable NixOS** release by using a specific version, such as `nixos-21.05`, **or** +- the latest **unstable release** via `nixos-unstable`. + +## Dependency management with niv + +If you'd like a bit more automation around bumping dependencies, including nixpkgs, +[niv](https://github.com/nmattia/niv/) is made for exactly that. Niv itself is available +in `nixpkgs` so using it is simple: + +``` +$ nix-shell -p niv --run "niv init" +``` + +This command will generate `nix/sources.json` with information about how and where +dependencies are fetched. It will also create `nix/sources.nix` which glues the sources together in Nix. + +By default `niv` will use the **latest stable** NixOS release. However, you should check to see which version is currently specified in [the niv repository](https://github.com/nmattia/niv) if you require a specific release, as it might lag behind. + +You can see which version `niv` is tracking as follow: + +> \$ niv show + +And you can change the tracking branch to the one you want like this: + +> \$ niv modify nixpkgs --branch nixos-21.05 + +You can use the generated `nix/sources.nix` with a top-level `default.nix`: + +```nix +{ sources ? import ./nix/sources.nix +, pkgs ? import sources.nixpkgs {} +}: + +... +``` + +And you can update all the dependencies by running: + +``` +$ nix-shell -p niv --run "niv update" +``` + +## Next steps + +- For more examples and details of the different ways to pin `nixpkgs`, see {ref}`ref-pinning-nixpkgs`. +- To quickly setup a Nix project read through + [Getting started Nix template](https://github.com/nix-dot-dev/getting-started-nix-template). diff --git a/source/tutorials/towards-reproducibility-pinning-nixpkgs.rst b/source/tutorials/towards-reproducibility-pinning-nixpkgs.rst deleted file mode 100644 index 29c2a8b..0000000 --- a/source/tutorials/towards-reproducibility-pinning-nixpkgs.rst +++ /dev/null @@ -1,93 +0,0 @@ -.. _pinning-nixpkgs: - -Towards reproducibility: Pinning nixpkgs -======================================== - -In various Nix examples, you'll often see references to ` `_, as follows. - -.. code:: nix - - { pkgs ? import {} - }: - - ... - -This is **convenient** to quickly demonstrate a Nix expression and get it working by importing Nix packages. - -However, the resulting Nix expression **is not fully reproducible**. The ```` reference -is set from the **local** ``$NIX_PATH`` environment variable. In most cases, this is set at the time Nix is installed -to the ``nixpkgs-unstable`` channel, and therefore it is likely to differ from machine to machine. - -.. note:: - `Channels `_ are a way of distributing Nix software, but they are being phased out. - Even though they are still used by default, it is recommended to avoid channels - and ```` by always setting ``NIX_PATH=`` to be empty. - -Pinning packages with URLs inside a Nix expression --------------------------------------------------- - -To create **fully reproducible** Nix expressions, we can pin an exact versions of nixpkgs. - -The simplest way to do this is to fetch the required nixpkgs version as a tarball specified via the relevant git commit hash: - -.. code:: nix - - { pkgs ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/3590f02e7d5760e52072c1a729ee2250b5560746.tar.gz") {} - }: - - ... - -Picking the commit can be done via `status.nixos.org `_, -which lists all the releases and the latest commit that has passed all tests. - -When choosing a commit, it is recommended to follow either - -* the **latest stable NixOS** release by using a specific version, such as ``nixos-21.05``, **or** -* the latest **unstable release** via ``nixos-unstable``. - -Dependency management with niv ------------------------------- - -If you'd like a bit more automation around bumping dependencies, including nixpkgs, -`niv `_ is made for exactly that. Niv itself is available -in ``nixpkgs`` so using it is simple:: - - $ nix-shell -p niv --run "niv init" - -This command will generate ``nix/sources.json`` with information about how and where -dependencies are fetched. It will also create ``nix/sources.nix`` which glues the sources together in Nix. - -By default ``niv`` will use the **latest stable** NixOS release. However, you should check to see which version is currently specified in `the niv repository `_ if you require a specific release, as it might lag behind. - -You can see which version ``niv`` is tracking as follow: - - $ niv show - -And you can change the tracking branch to the one you want like this: - - $ niv modify nixpkgs --branch nixos-21.05 - - - -You can use the generated ``nix/sources.nix`` with a top-level ``default.nix``: - -.. code:: nix - - { sources ? import ./nix/sources.nix - , pkgs ? import sources.nixpkgs {} - }: - - ... - -And you can update all the dependencies by running:: - - $ nix-shell -p niv --run "niv update" - - -Next steps ----------- - -- For more examples and details of the different ways to pin ``nixpkgs``, see :ref:`ref-pinning-nixpkgs`. - -- To quickly setup a Nix project read through - `Getting started Nix template `_.