From 85d792a92894ded8346ae3b775cca47d23d37b73 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Thu, 12 May 2022 15:17:51 +0000 Subject: [PATCH 1/4] nixos: don't put non-firmware in hardware.firmware These two packages don't have a lib/firmware directory, so putting them in hardware.firmware has no effect. This will become a hard error once firmware compression is implemented. (In the case of Linux, the firmware was all moved to linux-firmware.) --- nixos/modules/hardware/all-firmware.nix | 1 - nixos/modules/system/boot/kernel.nix | 3 --- 2 files changed, 4 deletions(-) diff --git a/nixos/modules/hardware/all-firmware.nix b/nixos/modules/hardware/all-firmware.nix index da2bc8ffef4b..89a1217dfb31 100644 --- a/nixos/modules/hardware/all-firmware.nix +++ b/nixos/modules/hardware/all-firmware.nix @@ -83,7 +83,6 @@ in { broadcom-bt-firmware b43Firmware_5_1_138 b43Firmware_6_30_163_46 - b43FirmwareCutter xow_dongle-firmware ] ++ optionals pkgs.stdenv.hostPlatform.isx86 [ facetimehd-calibration diff --git a/nixos/modules/system/boot/kernel.nix b/nixos/modules/system/boot/kernel.nix index fad00e39497d..b2c92a85f7ab 100644 --- a/nixos/modules/system/boot/kernel.nix +++ b/nixos/modules/system/boot/kernel.nix @@ -273,9 +273,6 @@ in boot.kernelModules = [ "loop" "atkbd" ]; - # The Linux kernel >= 2.6.27 provides firmware. - hardware.firmware = [ kernel ]; - # Create /etc/modules-load.d/nixos.conf, which is read by # systemd-modules-load.service to load required kernel modules. environment.etc = From 7b488d1fc95def61e1da768d4a2c78676d57cdb0 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Thu, 12 May 2022 15:19:04 +0000 Subject: [PATCH 2/4] b43Firmware_6_30_163_46: use the right location hardware.firmware only considers lib/firmware, so this firmware was being completely ignored. --- pkgs/os-specific/linux/firmware/b43-firmware/6.30.163.46.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/os-specific/linux/firmware/b43-firmware/6.30.163.46.nix b/pkgs/os-specific/linux/firmware/b43-firmware/6.30.163.46.nix index 622b89b1baea..92c179726d25 100644 --- a/pkgs/os-specific/linux/firmware/b43-firmware/6.30.163.46.nix +++ b/pkgs/os-specific/linux/firmware/b43-firmware/6.30.163.46.nix @@ -14,8 +14,8 @@ stdenv.mkDerivation rec { sourceRoot = "."; installPhase = '' - mkdir $out - b43-fwcutter -w $out *.wl_apsta.o + mkdir -p $out/lib/firmware + b43-fwcutter -w $out/lib/firmware *.wl_apsta.o ''; meta = with lib; { From 94880867466073f28eecd6a7858dab6ec67b6a0b Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Thu, 12 May 2022 15:21:04 +0000 Subject: [PATCH 3/4] linux: support loading compressed firmware Supported since Linux 5.3. --- pkgs/os-specific/linux/kernel/common-config.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/os-specific/linux/kernel/common-config.nix b/pkgs/os-specific/linux/kernel/common-config.nix index fdf54d302bf2..d25e20bac8b2 100644 --- a/pkgs/os-specific/linux/kernel/common-config.nix +++ b/pkgs/os-specific/linux/kernel/common-config.nix @@ -883,6 +883,8 @@ let # Disable the firmware helper fallback, udev doesn't implement it any more FW_LOADER_USER_HELPER_FALLBACK = option no; + FW_LOADER_COMPRESS = option yes; + HOTPLUG_PCI_ACPI = yes; # PCI hotplug using ACPI HOTPLUG_PCI_PCIE = yes; # PCI-Expresscard hotplug support From 8aa8e0ce7f137fe329608efcbb3494a5e6a63f42 Mon Sep 17 00:00:00 2001 From: Alyssa Ross Date: Thu, 12 May 2022 15:23:12 +0000 Subject: [PATCH 4/4] nixos/udev: compress all firmware if supported This should be a significant disk space saving for most NixOS installations. This method is a bit more complicated than doing it in the postInstall for the firmware derivations, but this way it's automatic, so each firmware package doesn't have to separately implement its compression. Currently, only xz compression is supported, but it's likely that future versions of Linux will additionally support zstd, so I've written the code in such a way that it would be very easy to implement zstd compression for those kernels when they arrive, falling back to xz for older (current) kernels. I chose the highest possible level of compression (xz -9) because even at this level, decompression time is negligible. Here's how long it took to decompress every firmware file my laptop uses: i915/kbl_dmc_ver1_04.bin 2ms regulatory.db 4ms regulatory.db.p7s 3ms iwlwifi-7265D-29.ucode 62ms 9d71-GOOGLE-EVEMAX-0-tplg.bin 22ms intel/dsp_fw_kbl.bin 65ms dsp_lib_dsm_core_spt_release.bin 6ms intel/ibt-hw-37.8.10-fw-22.50.19.14.f.bseq 7ms And since booting NixOS is a parallel process, it's unlikely (but difficult to measure) that the time to user interaction was held up at all by most of these. Fixes (partially?) #148197 --- nixos/modules/services/hardware/udev.nix | 7 ++++++- .../kernel/compress-firmware-xz.nix | 16 ++++++++++++++++ pkgs/build-support/kernel/modules-closure.sh | 8 ++++++-- pkgs/top-level/all-packages.nix | 2 ++ 4 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 pkgs/build-support/kernel/compress-firmware-xz.nix diff --git a/nixos/modules/services/hardware/udev.nix b/nixos/modules/services/hardware/udev.nix index 2f3866171873..2e9deebbb74a 100644 --- a/nixos/modules/services/hardware/udev.nix +++ b/nixos/modules/services/hardware/udev.nix @@ -171,6 +171,11 @@ let mv etc/udev/hwdb.bin $out ''; + compressFirmware = if config.boot.kernelPackages.kernelAtLeast "5.3" then + pkgs.compressFirmwareXz + else + id; + # Udev has a 512-character limit for ENV{PATH}, so create a symlink # tree to work around this. udevPath = pkgs.buildEnv { @@ -267,7 +272,7 @@ in ''; apply = list: pkgs.buildEnv { name = "firmware"; - paths = list; + paths = map compressFirmware list; pathsToLink = [ "/lib/firmware" ]; ignoreCollisions = true; }; diff --git a/pkgs/build-support/kernel/compress-firmware-xz.nix b/pkgs/build-support/kernel/compress-firmware-xz.nix new file mode 100644 index 000000000000..56595131c891 --- /dev/null +++ b/pkgs/build-support/kernel/compress-firmware-xz.nix @@ -0,0 +1,16 @@ +{ runCommand }: + +firmware: + +runCommand "${firmware.name}-xz" {} '' + mkdir -p $out/lib + (cd ${firmware} && find lib/firmware -type d -print0) | + (cd $out && xargs -0 mkdir -v --) + (cd ${firmware} && find lib/firmware -type f -print0) | + (cd $out && xargs -0tP "$NIX_BUILD_CORES" -n1 \ + sh -c 'xz -9c -T1 -C crc32 --lzma2=dict=2MiB "${firmware}/$1" > "$1.xz"' --) + (cd ${firmware} && find lib/firmware -type l) | while read link; do + target="$(readlink "${firmware}/$link")" + ln -vs -- "''${target/^${firmware}/$out}.xz" "$out/$link.xz" + done +'' diff --git a/pkgs/build-support/kernel/modules-closure.sh b/pkgs/build-support/kernel/modules-closure.sh index 3b3a38ea1d33..74bc490eb15c 100644 --- a/pkgs/build-support/kernel/modules-closure.sh +++ b/pkgs/build-support/kernel/modules-closure.sh @@ -81,8 +81,12 @@ for module in $(cat closure); do for i in $(modinfo -b $kernel --set-version "$version" -F firmware $module | grep -v '^name:'); do mkdir -p "$out/lib/firmware/$(dirname "$i")" echo "firmware for $module: $i" - cp "$firmware/lib/firmware/$i" "$out/lib/firmware/$i" 2>/dev/null \ - || echo "WARNING: missing firmware $i for module $module" + for name in "$i" "$i.xz" ""; do + [ -z "$name" ] && echo "WARNING: missing firmware $i for module $module" + if cp "$firmware/lib/firmware/$name" "$out/lib/firmware/$name" 2>/dev/null; then + break + fi + done done done diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index e89aa5c04a81..f1170aae20dc 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -808,6 +808,8 @@ with pkgs; sanitizers = [ ]; }; + compressFirmwareXz = callPackage ../build-support/kernel/compress-firmware-xz.nix { }; + makeModulesClosure = { kernel, firmware, rootModules, allowMissing ? false }: callPackage ../build-support/kernel/modules-closure.nix { inherit kernel firmware rootModules allowMissing;