From 32ca66f3edd386cce6c4e619c2b9aa0b8b0cd95c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christina=20S=C3=B8rensen?= Date: Wed, 17 Jul 2024 09:10:17 +0200 Subject: [PATCH] nixos/kubernetes: refactor feature gates to attrsOf bool, making it possible to disable featureGates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a breaking change, requiring users of `featureGates` to change from a `listOf str` to `attrsOf bool`. Before: ```nix featureGates = [ "EphemeralContainers" ]; extraOpts = pkgs.lib.concatStringsSep " " ( [ "--container-runtime=remote" ''--feature-gates="CSIMigration=false"'' }); ``` After: ```nix featureGates = {EphemeralContainers = true; CSIMigration=false;}; ``` This is much nicer, and sets us up for later work of migrating to configuration files for other services, like e.g. has been happening with kubelet (see: #290119). Signed-off-by: Christina Sørensen --- .../manual/release-notes/rl-2411.section.md | 18 ++++++++++++++++++ .../services/cluster/kubernetes/apiserver.nix | 8 ++++---- .../cluster/kubernetes/controller-manager.nix | 8 ++++---- .../services/cluster/kubernetes/default.nix | 4 ++-- .../services/cluster/kubernetes/kubelet.nix | 6 +++--- .../services/cluster/kubernetes/proxy.nix | 8 ++++---- .../services/cluster/kubernetes/scheduler.nix | 8 ++++---- nixos/tests/kubernetes/base.nix | 4 ++++ 8 files changed, 43 insertions(+), 21 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-2411.section.md b/nixos/doc/manual/release-notes/rl-2411.section.md index 79d18f9a8998..78400e868318 100644 --- a/nixos/doc/manual/release-notes/rl-2411.section.md +++ b/nixos/doc/manual/release-notes/rl-2411.section.md @@ -132,6 +132,24 @@ nvimpager settings: user commands in `-c` and `--cmd` now override the respective default settings because they are executed later. +- Kubernetes `featureGates` have changed from a `listOf str` to `attrsOf bool`. + This refactor makes it possible to also disable feature gates, without having + to use `extraOpts` flags. + + A previous configuration may have looked like this: + ```nix + featureGates = [ "EphemeralContainers" ]; + extraOpts = pkgs.lib.concatStringsSep " " ( + [ + ''--feature-gates="CSIMigration=false"'' + }); + ``` + + Using an AttrSet instead, the new configuration would be: + ```nix + featureGates = {EphemeralContainers = true; CSIMigration=false;}; + ``` + - `pkgs.nextcloud27` has been removed since it's EOL. - `services.forgejo.mailerPasswordFile` has been deprecated by the drop-in replacement `services.forgejo.secrets.mailer.PASSWD`, diff --git a/nixos/modules/services/cluster/kubernetes/apiserver.nix b/nixos/modules/services/cluster/kubernetes/apiserver.nix index fe9dacb8b93d..81e359e0e642 100644 --- a/nixos/modules/services/cluster/kubernetes/apiserver.nix +++ b/nixos/modules/services/cluster/kubernetes/apiserver.nix @@ -159,10 +159,10 @@ in }; featureGates = mkOption { - description = "List set of feature gates"; + description = "Attribute set of feature gates."; default = top.featureGates; defaultText = literalExpression "config.${otop.featureGates}"; - type = listOf str; + type = attrsOf bool; }; kubeletClientCaFile = mkOption { @@ -349,8 +349,8 @@ in "--etcd-certfile=${cfg.etcd.certFile}"} \ ${optionalString (cfg.etcd.keyFile != null) "--etcd-keyfile=${cfg.etcd.keyFile}"} \ - ${optionalString (cfg.featureGates != []) - "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ + ${optionalString (cfg.featureGates != {}) + "--feature-gates=${(concatStringsSep "," (builtins.attrValues (mapAttrs (n: v: "${n}=${trivial.boolToString v}") cfg.featureGates)))}"} \ ${optionalString (cfg.basicAuthFile != null) "--basic-auth-file=${cfg.basicAuthFile}"} \ ${optionalString (cfg.kubeletClientCaFile != null) diff --git a/nixos/modules/services/cluster/kubernetes/controller-manager.nix b/nixos/modules/services/cluster/kubernetes/controller-manager.nix index 453043e507d9..b427de22bf89 100644 --- a/nixos/modules/services/cluster/kubernetes/controller-manager.nix +++ b/nixos/modules/services/cluster/kubernetes/controller-manager.nix @@ -44,10 +44,10 @@ in }; featureGates = mkOption { - description = "List set of feature gates"; + description = "Attribute set of feature gates."; default = top.featureGates; defaultText = literalExpression "config.${otop.featureGates}"; - type = listOf str; + type = attrsOf bool; }; kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes controller manager"; @@ -121,8 +121,8 @@ in --bind-address=${cfg.bindAddress} \ ${optionalString (cfg.clusterCidr!=null) "--cluster-cidr=${cfg.clusterCidr}"} \ - ${optionalString (cfg.featureGates != []) - "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ + ${optionalString (cfg.featureGates != {}) + "--feature-gates=${concatStringsSep "," (builtins.attrValues (mapAttrs (n: v: "${n}=${trivial.boolToString v}") cfg.featureGates))}"} \ --kubeconfig=${top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \ --leader-elect=${boolToString cfg.leaderElect} \ ${optionalString (cfg.rootCaFile!=null) diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index 6485df5fffbe..208b2a864f02 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -155,8 +155,8 @@ in { featureGates = mkOption { description = "List set of feature gates."; - default = []; - type = types.listOf types.str; + default = {}; + type = types.attrsOf types.bool; }; masterAddress = mkOption { diff --git a/nixos/modules/services/cluster/kubernetes/kubelet.nix b/nixos/modules/services/cluster/kubernetes/kubelet.nix index f36edeaf64ce..fd9df556e7ec 100644 --- a/nixos/modules/services/cluster/kubernetes/kubelet.nix +++ b/nixos/modules/services/cluster/kubernetes/kubelet.nix @@ -65,7 +65,7 @@ let // lib.optionalAttrs (cfg.tlsKeyFile != null) { tlsPrivateKeyFile = cfg.tlsKeyFile; } // lib.optionalAttrs (cfg.clusterDomain != "") { clusterDomain = cfg.clusterDomain; } // lib.optionalAttrs (cfg.clusterDns != "") { clusterDNS = [ cfg.clusterDns ] ; } - // lib.optionalAttrs (cfg.featureGates != []) { featureGates = cfg.featureGates; } + // lib.optionalAttrs (cfg.featureGates != {}) { featureGates = cfg.featureGates; } )); manifestPath = "kubernetes/manifests"; @@ -185,10 +185,10 @@ in }; featureGates = mkOption { - description = "List set of feature gates"; + description = "Attribute set of feature gate"; default = top.featureGates; defaultText = literalExpression "config.${otop.featureGates}"; - type = listOf str; + type = attrsOf bool; }; healthz = { diff --git a/nixos/modules/services/cluster/kubernetes/proxy.nix b/nixos/modules/services/cluster/kubernetes/proxy.nix index c09e7695f2a4..2e3fdc87b439 100644 --- a/nixos/modules/services/cluster/kubernetes/proxy.nix +++ b/nixos/modules/services/cluster/kubernetes/proxy.nix @@ -30,10 +30,10 @@ in }; featureGates = mkOption { - description = "List set of feature gates"; + description = "Attribute set of feature gates."; default = top.featureGates; defaultText = literalExpression "config.${otop.featureGates}"; - type = listOf str; + type = attrsOf bool; }; hostname = mkOption { @@ -69,8 +69,8 @@ in --bind-address=${cfg.bindAddress} \ ${optionalString (top.clusterCidr!=null) "--cluster-cidr=${top.clusterCidr}"} \ - ${optionalString (cfg.featureGates != []) - "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ + ${optionalString (cfg.featureGates != {}) + "--feature-gates=${concatStringsSep "," (builtins.attrValues (mapAttrs (n: v: "${n}=${trivial.boolToString v}") cfg.featureGates))}"} \ --hostname-override=${cfg.hostname} \ --kubeconfig=${top.lib.mkKubeConfig "kube-proxy" cfg.kubeconfig} \ ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \ diff --git a/nixos/modules/services/cluster/kubernetes/scheduler.nix b/nixos/modules/services/cluster/kubernetes/scheduler.nix index da2f39226a24..6fb90469c706 100644 --- a/nixos/modules/services/cluster/kubernetes/scheduler.nix +++ b/nixos/modules/services/cluster/kubernetes/scheduler.nix @@ -26,10 +26,10 @@ in }; featureGates = mkOption { - description = "List set of feature gates"; + description = "Attribute set of feature gates."; default = top.featureGates; defaultText = literalExpression "config.${otop.featureGates}"; - type = listOf str; + type = attrsOf bool; }; kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes scheduler"; @@ -67,8 +67,8 @@ in Slice = "kubernetes.slice"; ExecStart = ''${top.package}/bin/kube-scheduler \ --bind-address=${cfg.address} \ - ${optionalString (cfg.featureGates != []) - "--feature-gates=${concatMapStringsSep "," (feature: "${feature}=true") cfg.featureGates}"} \ + ${optionalString (cfg.featureGates != {}) + "--feature-gates=${concatStringsSep "," (builtins.attrValues (mapAttrs (n: v: "${n}=${trivial.boolToString v}") cfg.featureGates))}"} \ --kubeconfig=${top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig} \ --leader-elect=${boolToString cfg.leaderElect} \ --secure-port=${toString cfg.port} \ diff --git a/nixos/tests/kubernetes/base.nix b/nixos/tests/kubernetes/base.nix index 13a2bc03831d..c75610723ea4 100644 --- a/nixos/tests/kubernetes/base.nix +++ b/nixos/tests/kubernetes/base.nix @@ -59,6 +59,10 @@ let securePort = 443; advertiseAddress = master.ip; }; + # NOTE: what featureGates are useful for testing might change in + # the future, see link below to find new ones + # https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/ + featureGates = {CPUManager = true; AppArmor= false;}; masterAddress = "${masterName}.${config.networking.domain}"; }; }