From 83c9b7483430d7e361d0c7676a8cc31008f400c3 Mon Sep 17 00:00:00 2001 From: Izorkin Date: Sat, 30 Oct 2021 20:02:13 +0300 Subject: [PATCH 1/4] nifi: init at 1.16.0 --- pkgs/servers/web-apps/nifi/default.nix | 41 ++++++++++++++++++++++++++ pkgs/top-level/all-packages.nix | 2 ++ 2 files changed, 43 insertions(+) create mode 100644 pkgs/servers/web-apps/nifi/default.nix diff --git a/pkgs/servers/web-apps/nifi/default.nix b/pkgs/servers/web-apps/nifi/default.nix new file mode 100644 index 000000000000..43662321f802 --- /dev/null +++ b/pkgs/servers/web-apps/nifi/default.nix @@ -0,0 +1,41 @@ +{ lib, stdenv, fetchurl, makeWrapper, jre8 }: + +stdenv.mkDerivation rec { + pname = "nifi"; + version = "1.16.0"; + + src = fetchurl { + url = "https://dlcdn.apache.org/nifi/${version}/nifi-${version}-bin.tar.gz"; + sha256 = "sha256-BE990mVSocO+UuMa9kULJcbXbqiX8oquZQTQKSRPe4g="; + }; + + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ jre8 ]; + + installPhase = '' + mv ../$sourceRoot $out + rm -f $out/bin/*bat + rm -rf $out/extensions + mkdir -p $out/share/nifi + mv $out/conf $out/share/nifi + mv $out/docs $out/share/nifi + mv $out/{LICENSE,NOTICE,README} $out/share/nifi + + substituteInPlace $out/bin/nifi.sh \ + --replace "/bin/sh" "${stdenv.shell}" + substituteInPlace $out/bin/nifi-env.sh \ + --replace "#export JAVA_HOME=/usr/java/jdk1.8.0/" "export JAVA_HOME=${jre8}" + ''; + + meta = with lib; { + description = "Easy to use, powerful, and reliable system to process and distribute data"; + longDescription = '' + Apache NiFi supports powerful and scalable directed graphs of data routing, + transformation, and system mediation logic. + ''; + license = licenses.asl20; + homepage = "https://nifi.apache.org"; + platforms = [ "x86_64-linux" ]; + maintainers = with maintainers; [ izorkin ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 986ff6589c82..caad752c094c 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -8283,6 +8283,8 @@ with pkgs; ngrok-1 = callPackage ../tools/networking/ngrok-1 { }; + nifi = callPackage ../servers/web-apps/nifi { }; + nitter = callPackage ../servers/nitter { }; noice = callPackage ../applications/misc/noice { }; From d706301b0cb45bb7451a3bfb685b3bf2795baa8f Mon Sep 17 00:00:00 2001 From: Izorkin Date: Sat, 30 Oct 2021 20:04:55 +0300 Subject: [PATCH 2/4] nixos/nifi: init service --- nixos/modules/module-list.nix | 1 + nixos/modules/services/web-apps/nifi.nix | 318 +++++++++++++++++++++++ 2 files changed, 319 insertions(+) create mode 100644 nixos/modules/services/web-apps/nifi.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 329c8685c36d..ae6b1dd0773b 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1054,6 +1054,7 @@ ./services/web-apps/netbox.nix ./services/web-apps/nextcloud.nix ./services/web-apps/nexus.nix + ./services/web-apps/nifi.nix ./services/web-apps/node-red.nix ./services/web-apps/pict-rs.nix ./services/web-apps/peertube.nix diff --git a/nixos/modules/services/web-apps/nifi.nix b/nixos/modules/services/web-apps/nifi.nix new file mode 100644 index 000000000000..21a631272641 --- /dev/null +++ b/nixos/modules/services/web-apps/nifi.nix @@ -0,0 +1,318 @@ +{ lib, pkgs, config, options, ... }: + +let + cfg = config.services.nifi; + opt = options.services.nifi; + + env = { + NIFI_OVERRIDE_NIFIENV = "true"; + NIFI_HOME = "/var/lib/nifi"; + NIFI_PID_DIR = "/run/nifi"; + NIFI_LOG_DIR = "/var/log/nifi"; + }; + + envFile = pkgs.writeText "nifi.env" (lib.concatMapStrings (s: s + "\n") ( + (lib.concatLists (lib.mapAttrsToList (name: value: + if value != null then [ + "${name}=\"${toString value}\"" + ] else [] + ) env)))); + + nifiEnv = pkgs.writeShellScriptBin "nifi-env" '' + set -a + source "${envFile}" + eval -- "\$@" + ''; + +in { + options = { + services.nifi = { + enable = lib.mkEnableOption "Apache NiFi"; + + package = lib.mkOption { + type = lib.types.package; + default = pkgs.nifi; + defaultText = lib.literalExpression "pkgs.nifi"; + description = "Apache NiFi package to use."; + }; + + user = lib.mkOption { + type = lib.types.str; + default = "nifi"; + description = "User account where Apache NiFi runs."; + }; + + group = lib.mkOption { + type = lib.types.str; + default = "nifi"; + description = "Group account where Apache NiFi runs."; + }; + + enableHTTPS = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Enable HTTPS protocol. Don`t use in production."; + }; + + listenHost = lib.mkOption { + type = lib.types.str; + default = if cfg.enableHTTPS then "0.0.0.0" else "127.0.0.1"; + defaultText = lib.literalExpression '' + if config.${opt.enableHTTPS} + then "0.0.0.0" + else "127.0.0.1" + ''; + description = "Bind to an ip for Apache NiFi web-ui."; + }; + + listenPort = lib.mkOption { + type = lib.types.int; + default = if cfg.enableHTTPS then 8443 else 8080; + defaultText = lib.literalExpression '' + if config.${opt.enableHTTPS} + then "8443" + else "8000" + ''; + description = "Bind to a port for Apache NiFi web-ui."; + }; + + proxyHost = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = if cfg.enableHTTPS then "0.0.0.0" else null; + defaultText = lib.literalExpression '' + if config.${opt.enableHTTPS} + then "0.0.0.0" + else null + ''; + description = "Allow requests from a specific host."; + }; + + proxyPort = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = if cfg.enableHTTPS then 8443 else null; + defaultText = lib.literalExpression '' + if config.${opt.enableHTTPS} + then "8443" + else null + ''; + description = "Allow requests from a specific port."; + }; + + initUser = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Initial user account for Apache NiFi. Username must be at least 4 characters."; + }; + + initPasswordFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + example = "/run/keys/nifi/password-nifi"; + description = "nitial password for Apache NiFi. Password must be at least 12 characters."; + }; + + initJavaHeapSize = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + example = 1024; + description = "Set the initial heap size for the JVM in MB."; + }; + + maxJavaHeapSize = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + example = 2048; + description = "Set the initial heap size for the JVM in MB."; + }; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { assertion = cfg.initUser!=null || cfg.initPasswordFile==null; + message = '' + needs to be set if enabled. + ''; + } + { assertion = cfg.initUser==null || cfg.initPasswordFile!=null; + message = '' + needs to be set if enabled. + ''; + } + { assertion = cfg.proxyHost==null || cfg.proxyPort!=null; + message = '' + needs to be set if value specified. + ''; + } + { assertion = cfg.proxyHost!=null || cfg.proxyPort==null; + message = '' + needs to be set if value specified. + ''; + } + { assertion = cfg.initJavaHeapSize==null || cfg.maxJavaHeapSize!=null; + message = '' + needs to be set if value specified. + ''; + } + { assertion = cfg.initJavaHeapSize!=null || cfg.maxJavaHeapSize==null; + message = '' + needs to be set if value specified. + ''; + } + ]; + + warnings = lib.optional (cfg.enableHTTPS==false) '' + Please do not disable HTTPS mode in production. In this mode, access to the nifi is opened without authentication. + ''; + + systemd.tmpfiles.rules = [ + "d '/var/lib/nifi/conf' 0750 ${cfg.user} ${cfg.group}" + "L+ '/var/lib/nifi/lib' - - - - ${cfg.package}/lib" + ]; + + + systemd.services.nifi = { + description = "Apache NiFi"; + after = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + + environment = env; + path = [ pkgs.gawk ]; + + serviceConfig = { + Type = "forking"; + PIDFile = "/run/nifi/nifi.pid"; + ExecStartPre = pkgs.writeScript "nifi-pre-start.sh" '' + #!/bin/sh + umask 077 + test -f '/var/lib/nifi/conf/authorizers.xml' || (cp '${cfg.package}/share/nifi/conf/authorizers.xml' '/var/lib/nifi/conf/' && chmod 0640 '/var/lib/nifi/conf/authorizers.xml') + test -f '/var/lib/nifi/conf/bootstrap.conf' || (cp '${cfg.package}/share/nifi/conf/bootstrap.conf' '/var/lib/nifi/conf/' && chmod 0640 '/var/lib/nifi/conf/bootstrap.conf') + test -f '/var/lib/nifi/conf/bootstrap-hashicorp-vault.conf' || (cp '${cfg.package}/share/nifi/conf/bootstrap-hashicorp-vault.conf' '/var/lib/nifi/conf/' && chmod 0640 '/var/lib/nifi/conf/bootstrap-hashicorp-vault.conf') + test -f '/var/lib/nifi/conf/bootstrap-notification-services.xml' || (cp '${cfg.package}/share/nifi/conf/bootstrap-notification-services.xml' '/var/lib/nifi/conf/' && chmod 0640 '/var/lib/nifi/conf/bootstrap-notification-services.xml') + test -f '/var/lib/nifi/conf/logback.xml' || (cp '${cfg.package}/share/nifi/conf/logback.xml' '/var/lib/nifi/conf/' && chmod 0640 '/var/lib/nifi/conf/logback.xml') + test -f '/var/lib/nifi/conf/login-identity-providers.xml' || (cp '${cfg.package}/share/nifi/conf/login-identity-providers.xml' '/var/lib/nifi/conf/' && chmod 0640 '/var/lib/nifi/conf/login-identity-providers.xml') + test -f '/var/lib/nifi/conf/nifi.properties' || (cp '${cfg.package}/share/nifi/conf/nifi.properties' '/var/lib/nifi/conf/' && chmod 0640 '/var/lib/nifi/conf/nifi.properties') + test -f '/var/lib/nifi/conf/stateless-logback.xml' || (cp '${cfg.package}/share/nifi/conf/stateless-logback.xml' '/var/lib/nifi/conf/' && chmod 0640 '/var/lib/nifi/conf/stateless-logback.xml') + test -f '/var/lib/nifi/conf/stateless.properties' || (cp '${cfg.package}/share/nifi/conf/stateless.properties' '/var/lib/nifi/conf/' && chmod 0640 '/var/lib/nifi/conf/stateless.properties') + test -f '/var/lib/nifi/conf/state-management.xml' || (cp '${cfg.package}/share/nifi/conf/state-management.xml' '/var/lib/nifi/conf/' && chmod 0640 '/var/lib/nifi/conf/state-management.xml') + test -f '/var/lib/nifi/conf/zookeeper.properties' || (cp '${cfg.package}/share/nifi/conf/zookeeper.properties' '/var/lib/nifi/conf/' && chmod 0640 '/var/lib/nifi/conf/zookeeper.properties') + test -d '/var/lib/nifi/docs/html' || (mkdir -p /var/lib/nifi/docs && cp -r '${cfg.package}/share/nifi/docs/html' '/var/lib/nifi/docs/html') + ${lib.optionalString ((cfg.initUser != null) && (cfg.initPasswordFile != null)) '' + awk -F'[<|>]' '/property name="Username"/ {if ($3!="") f=1} END{exit !f}' /var/lib/nifi/conf/login-identity-providers.xml || ${cfg.package}/bin/nifi.sh set-single-user-credentials ${cfg.initUser} $(cat ${cfg.initPasswordFile}) + ''} + ${lib.optionalString (cfg.enableHTTPS == false) '' + sed -i /var/lib/nifi/conf/nifi.properties \ + -e 's|nifi.remote.input.secure=.*|nifi.remote.input.secure=false|g' \ + -e 's|nifi.web.http.host=.*|nifi.web.http.host=${cfg.listenHost}|g' \ + -e 's|nifi.web.http.port=.*|nifi.web.http.port=${(toString cfg.listenPort)}|g' \ + -e 's|nifi.web.https.host=.*|nifi.web.https.host=|g' \ + -e 's|nifi.web.https.port=.*|nifi.web.https.port=|g' \ + -e 's|nifi.security.keystore=.*|nifi.security.keystore=|g' \ + -e 's|nifi.security.keystoreType=.*|nifi.security.keystoreType=|g' \ + -e 's|nifi.security.truststore=.*|nifi.security.truststore=|g' \ + -e 's|nifi.security.truststoreType=.*|nifi.security.truststoreType=|g' \ + -e '/nifi.security.keystorePasswd/s|^|#|' \ + -e '/nifi.security.keyPasswd/s|^|#|' \ + -e '/nifi.security.truststorePasswd/s|^|#|' + ''} + ${lib.optionalString (cfg.enableHTTPS == true) '' + sed -i /var/lib/nifi/conf/nifi.properties \ + -e 's|nifi.remote.input.secure=.*|nifi.remote.input.secure=true|g' \ + -e 's|nifi.web.http.host=.*|nifi.web.http.host=|g' \ + -e 's|nifi.web.http.port=.*|nifi.web.http.port=|g' \ + -e 's|nifi.web.https.host=.*|nifi.web.https.host=${cfg.listenHost}|g' \ + -e 's|nifi.web.https.port=.*|nifi.web.https.port=${(toString cfg.listenPort)}|g' \ + -e 's|nifi.security.keystore=.*|nifi.security.keystore=./conf/keystore.p12|g' \ + -e 's|nifi.security.keystoreType=.*|nifi.security.keystoreType=PKCS12|g' \ + -e 's|nifi.security.truststore=.*|nifi.security.truststore=./conf/truststore.p12|g' \ + -e 's|nifi.security.truststoreType=.*|nifi.security.truststoreType=PKCS12|g' \ + -e '/nifi.security.keystorePasswd/s|^#\+||' \ + -e '/nifi.security.keyPasswd/s|^#\+||' \ + -e '/nifi.security.truststorePasswd/s|^#\+||' + ''} + ${lib.optionalString ((cfg.enableHTTPS == true) && (cfg.proxyHost != null) && (cfg.proxyPort != null)) '' + sed -i /var/lib/nifi/conf/nifi.properties \ + -e 's|nifi.web.proxy.host=.*|nifi.web.proxy.host=${cfg.proxyHost}:${(toString cfg.proxyPort)}|g' + ''} + ${lib.optionalString ((cfg.enableHTTPS == false) || (cfg.proxyHost == null) && (cfg.proxyPort == null)) '' + sed -i /var/lib/nifi/conf/nifi.properties \ + -e 's|nifi.web.proxy.host=.*|nifi.web.proxy.host=|g' + ''} + ${lib.optionalString ((cfg.initJavaHeapSize != null) && (cfg.maxJavaHeapSize != null))'' + sed -i /var/lib/nifi/conf/bootstrap.conf \ + -e 's|java.arg.2=.*|java.arg.2=-Xms${(toString cfg.initJavaHeapSize)}m|g' \ + -e 's|java.arg.3=.*|java.arg.3=-Xmx${(toString cfg.maxJavaHeapSize)}m|g' + ''} + ${lib.optionalString ((cfg.initJavaHeapSize == null) && (cfg.maxJavaHeapSize == null))'' + sed -i /var/lib/nifi/conf/bootstrap.conf \ + -e 's|java.arg.2=.*|java.arg.2=-Xms512m|g' \ + -e 's|java.arg.3=.*|java.arg.3=-Xmx512m|g' + ''} + ''; + ExecStart = "${cfg.package}/bin/nifi.sh start"; + ExecStop = "${cfg.package}/bin/nifi.sh stop"; + # User and group + User = cfg.user; + Group = cfg.group; + # Runtime directory and mode + RuntimeDirectory = "nifi"; + RuntimeDirectoryMode = "0750"; + # State directory and mode + StateDirectory = "nifi"; + StateDirectoryMode = "0750"; + # Logs directory and mode + LogsDirectory = "nifi"; + LogsDirectoryMode = "0750"; + # Proc filesystem + ProcSubset = "pid"; + ProtectProc = "invisible"; + # Access write directories + ReadWritePaths = [ cfg.initPasswordFile ]; + UMask = "0027"; + # Capabilities + CapabilityBoundingSet = ""; + # Security + NoNewPrivileges = true; + # Sandboxing + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + PrivateIPC = true; + PrivateUsers = true; + ProtectHostname = true; + ProtectClock = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + RestrictAddressFamilies = [ "AF_INET AF_INET6" ]; + RestrictNamespaces = true; + LockPersonality = true; + MemoryDenyWriteExecute = false; + RestrictRealtime = true; + RestrictSUIDSGID = true; + RemoveIPC = true; + PrivateMounts = true; + # System Call Filtering + SystemCallArchitectures = "native"; + SystemCallFilter = [ "~@cpu-emulation @debug @keyring @memlock @mount @obsolete @resources @privileged @setuid" "@chown" ]; + }; + }; + + users.users = lib.mkMerge [ + (lib.mkIf (cfg.user == "nifi") { + nifi = { + group = cfg.group; + isSystemUser = true; + home = cfg.package; + }; + }) + (lib.attrsets.setAttrByPath [ cfg.user "packages" ] [ cfg.package nifiEnv ]) + ]; + + users.groups = lib.optionalAttrs (cfg.group == "nifi") { + nifi = { }; + }; + }; +} From 07962ac6f132a12e133c1a057ddb2507d96386d2 Mon Sep 17 00:00:00 2001 From: Izorkin Date: Sun, 10 Apr 2022 15:55:20 +0300 Subject: [PATCH 3/4] nixos/nifi: add release notes --- .../doc/manual/from_md/release-notes/rl-2205.section.xml | 8 ++++++++ nixos/doc/manual/release-notes/rl-2205.section.md | 2 ++ 2 files changed, 10 insertions(+) diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml index 7bcfcebe21b2..ddf4d58ef299 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml @@ -381,6 +381,14 @@ cluster resource manager + + + nifi, an + easy to use, powerful, and reliable system to process and + distribute data. Available as + services.nifi. + +
diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md index a086751a2f37..aca54718c572 100644 --- a/nixos/doc/manual/release-notes/rl-2205.section.md +++ b/nixos/doc/manual/release-notes/rl-2205.section.md @@ -109,6 +109,8 @@ In addition to numerous new and upgraded packages, this release has the followin - [pacemaker](https://clusterlabs.org/pacemaker/) cluster resource manager +- [nifi](https://nifi.apache.org), an easy to use, powerful, and reliable system to process and distribute data. Available as [services.nifi](options.html#opt-services.nifi.enable). + ## Backward Incompatibilities {#sec-release-22.05-incompatibilities} From 63ef6aebef14c23276532a3abdff01fc14b618ee Mon Sep 17 00:00:00 2001 From: Izorkin Date: Sun, 10 Apr 2022 20:41:30 +0300 Subject: [PATCH 4/4] nixos/tests: add nifi test --- nixos/tests/all-tests.nix | 1 + nixos/tests/web-apps/nifi.nix | 30 ++++++++++++++++++++++++++ pkgs/servers/web-apps/nifi/default.nix | 6 +++++- 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 nixos/tests/web-apps/nifi.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index d9429920f13b..488485064206 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -362,6 +362,7 @@ in nginx-sandbox = handleTestOn ["x86_64-linux"] ./nginx-sandbox.nix {}; nginx-sso = handleTest ./nginx-sso.nix {}; nginx-variants = handleTest ./nginx-variants.nix {}; + nifi = handleTestOn ["x86_64-linux"] ./web-apps/nifi.nix {}; nitter = handleTest ./nitter.nix {}; nix-ld = handleTest ./nix-ld {}; nix-serve = handleTest ./nix-serve.nix {}; diff --git a/nixos/tests/web-apps/nifi.nix b/nixos/tests/web-apps/nifi.nix new file mode 100644 index 000000000000..92f7fa231df3 --- /dev/null +++ b/nixos/tests/web-apps/nifi.nix @@ -0,0 +1,30 @@ +import ../make-test-python.nix ({pkgs, ...}: +{ + name = "nifi"; + meta.maintainers = with pkgs.lib.maintainers; [ izorkin ]; + + nodes = { + nifi = { pkgs, ... }: { + virtualisation = { + memorySize = 2048; + diskSize = 4096; + }; + services.nifi = { + enable = true; + enableHTTPS = false; + }; + }; + }; + + testScript = '' + nifi.start() + + nifi.wait_for_unit("nifi.service") + nifi.wait_for_open_port(8080) + + # Check if NiFi is running + nifi.succeed("curl --fail http://127.0.0.1:8080/nifi/login 2> /dev/null | grep 'NiFi Login'") + + nifi.shutdown() + ''; +}) diff --git a/pkgs/servers/web-apps/nifi/default.nix b/pkgs/servers/web-apps/nifi/default.nix index 43662321f802..f25c1931501e 100644 --- a/pkgs/servers/web-apps/nifi/default.nix +++ b/pkgs/servers/web-apps/nifi/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchurl, makeWrapper, jre8 }: +{ lib, stdenv, fetchurl, makeWrapper, jre8, nixosTests }: stdenv.mkDerivation rec { pname = "nifi"; @@ -27,6 +27,10 @@ stdenv.mkDerivation rec { --replace "#export JAVA_HOME=/usr/java/jdk1.8.0/" "export JAVA_HOME=${jre8}" ''; + passthru = { + tests.nifi = nixosTests.nifi; + }; + meta = with lib; { description = "Easy to use, powerful, and reliable system to process and distribute data"; longDescription = ''