From a4e708522768a30cd5120a65c979c87abd525336 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 20 Apr 2021 13:46:53 +0200 Subject: [PATCH] stdenv.mkDerivation: Allow overriding of recursive definitions See updated manual for further explanation. --- doc/stdenv/meta.chapter.md | 30 +++++++++++ doc/stdenv/stdenv.chapter.md | 52 +++++++++++++++++++ pkgs/stdenv/generic/make-derivation.nix | 67 +++++++++++++++++++++++-- 3 files changed, 146 insertions(+), 3 deletions(-) diff --git a/doc/stdenv/meta.chapter.md b/doc/stdenv/meta.chapter.md index d3e1dd5b27d8..5f3e12ba0041 100644 --- a/doc/stdenv/meta.chapter.md +++ b/doc/stdenv/meta.chapter.md @@ -175,6 +175,36 @@ The NixOS tests are available as `nixosTests` in parameters of derivations. For NixOS tests run in a VM, so they are slower than regular package tests. For more information see [NixOS module tests](https://nixos.org/manual/nixos/stable/#sec-nixos-tests). +Alternatively, you can specify other derivations as tests. You can make use of +the optional parameter (here: `self`) to inject the correct package without +relying on non-local definitions, even in the presence of `overrideAttrs`. This +definition of `tests` does not rely on the original `mypkg` or overrides it in +all places. + +```nix +# my-package/default.nix +{ stdenv, callPackage }: +stdenv.mkDerivation (self: { + # ... + passthru.tests.example = callPackage ./example.nix { my-package = self; }; +}) +``` + +```nix +# my-package/example.nix +{ runCommand, lib, my-package, ... }: +runCommand "my-package-test" { + nativeBuildInputs = [ my-package ]; + src = lib.sources.sourcesByRegex ./. [ ".*.in" ".*.expected" ]; +} '' + my-package --help + my-package example.actual + diff -U3 --color=auto example.expected example.actual + mkdir $out +'' +``` + + ### `timeout` {#var-meta-timeout} A timeout (in seconds) for building the derivation. If the derivation takes longer than this time to build, it can fail due to breaking the timeout. However, all computers do not have the same computing power, hence some builders may decide to apply a multiplicative factor to this value. When filling this value in, try to keep it approximately consistent with other values already present in `nixpkgs`. diff --git a/doc/stdenv/stdenv.chapter.md b/doc/stdenv/stdenv.chapter.md index 40f295b178bb..2994f7020b19 100644 --- a/doc/stdenv/stdenv.chapter.md +++ b/doc/stdenv/stdenv.chapter.md @@ -317,6 +317,58 @@ The script will be usually run from the root of the Nixpkgs repository but you s For information about how to run the updates, execute `nix-shell maintainers/scripts/update.nix`. +### Recursive attributes in `mkDerivation` + +If you pass a function to `mkDerivation`, it will receive as its argument the final output of the same `mkDerivation` call. For example: + +```nix +mkDerivation (self: { + pname = "hello"; + withFeature = true; + configureFlags = + lib.optionals self.withFeature ["--with-feature"]; +}) +``` + +Note that this does not use the `rec` keyword to reuse `withFeature` in `configureFlags`. +Instead, the definition references `self`, allowing users to change `withFeature` +consistently with `overrideAttrs`. + +Let's look at a more elaborate example to understand the differences between +various bindings: + +```nix +# `pkg` is the _original_ definition (for illustration purposes) +let pkg = + mkDerivation (self: { # self is the final package + # ... + + # An example attribute + packages = []; + + # `passthru.tests` is a commonly defined attribute. + passthru.tests.simple = f self; + + # An example of an attribute containing a function + passthru.appendPackages = packages': + self.overrideAttrs (newSelf: super: { + packages = super.packages ++ packages'; + }); + + # For illustration purposes; referenced as + # `(pkg.overrideAttrs(x)).self` etc in the text below. + passthru.self = self; + passthru.original = pkg; + }); +in pkg +``` + +Unlike the `pkg` binding in the above example, the `self` parameter always references the final package. For instance `(pkg.overrideAttrs(x)).self` is identical to `pkg.overrideAttrs(x)`, whereas `(pkg.overrideAttrs(x)).original` is the same as `pkg`. + +This is also different from `mkDerivation rec { ..... }`, which binds the recursive references immediately, so it allows you to reference original _inputs_ only. + +See also the section about [`passthru.tests`](#var-meta-tests). + ## Phases {#sec-stdenv-phases} `stdenv.mkDerivation` sets the Nix [derivation](https://nixos.org/manual/nix/stable/expressions/derivations.html#derivations)'s builder to a script that loads the stdenv `setup.sh` bash library and calls `genericBuild`. Most packaging functions rely on this default builder. diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index d1b93874a25a..2a47ea993839 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -9,8 +9,68 @@ let # to build it. This is a bit confusing for cross compilation. inherit (stdenv) hostPlatform; }; + + makeOverlayable = mkDerivationSimple: # TODO(@robert): turn mkDerivationSimple into let binding. + fnOrAttrs: + if builtins.isFunction fnOrAttrs + then makeDerivationExtensible mkDerivationSimple fnOrAttrs + else makeDerivationExtensibleConst mkDerivationSimple fnOrAttrs; + + # Based off lib.makeExtensible, with modifications: + # - lib.fix' -> lib.fix ∘ mkDerivationSimple; then inline fix + # - convert `f` to an overlay + # - inline overrideAttrs and make it positional instead of // to reduce allocs + makeDerivationExtensible = mkDerivationSimple: rattrs: + let + r = mkDerivationSimple + (f0: + let + f = self: super: + # Convert f0 to an overlay. Legacy is: + # overrideAttrs (super: {}) + # We want to introduce self. We follow the convention of overlays: + # overrideAttrs (self: super: {}) + # Which means the first parameter can be either self or super. + # This is surprising, but far better than the confusion that would + # arise from flipping an overlay's parameters in some cases. + let x = f0 super; + in + if builtins.isFunction x + then + # Can't reuse `x`, because `self` comes first. + # Looks inefficient, but `f0 super` was a cheap thunk. + f0 self super + else x; + in + makeDerivationExtensible mkDerivationSimple + (self: let super = rattrs self; in super // f self super)) + (rattrs r); + in r; + + # makeDerivationExtensibleConst == makeDerivationExtensible (_: attrs), + # but pre-evaluated for a slight improvement in performance. + makeDerivationExtensibleConst = mkDerivationSimple: attrs: + mkDerivationSimple (f0: + let + f = self: super: + let x = f0 super; + in + if builtins.isFunction x + then + # Can't reuse `x`, because `self` comes first. + # Looks inefficient, but `f0 super` was a cheap thunk. + f0 self super + else x; + in + makeDerivationExtensible mkDerivationSimple (self: attrs // f self attrs)) + attrs; + in +# TODO(@roberth): inline makeOverlayable; reindenting whole rest of this file. +makeOverlayable (overrideAttrs: + + # `mkDerivation` wraps the builtin `derivation` function to # produce derivations that use this stdenv and its shell. # @@ -70,6 +130,7 @@ in , # TODO(@Ericson2314): Make always true and remove strictDeps ? if config.strictDepsByDefault then true else stdenv.hostPlatform != stdenv.buildPlatform + , meta ? {} , passthru ? {} , pos ? # position used in error messages and for meta.position @@ -381,8 +442,6 @@ in lib.extendDerivation validity.handled ({ - overrideAttrs = f: stdenv.mkDerivation (attrs // (f attrs)); - # A derivation that always builds successfully and whose runtime # dependencies are the original derivations build time dependencies # This allows easy building and distributing of all derivations @@ -408,10 +467,12 @@ lib.extendDerivation args = [ "-c" "export > $out" ]; }); - inherit meta passthru; + inherit meta passthru overrideAttrs; } // # Pass through extra attributes that are not inputs, but # should be made available to Nix expressions using the # derivation (e.g., in assertions). passthru) (derivation derivationArg) + +)