diff --git a/.gitignore b/.gitignore index 5e40e6e5..faaae889 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,9 @@ outputs # Some files generated by CI, to ensure the working tree is clean on CI /manpage /public + +# Created by running IOCs +.iocsh_history + +# Created by running NixOS tests in interactive mode +.nixos-test-history diff --git a/doc/nixos/guides/_pre-requisites.md b/doc/nixos/guides/_pre-requisites.md index 67a77503..82b49c70 100644 --- a/doc/nixos/guides/_pre-requisites.md +++ b/doc/nixos/guides/_pre-requisites.md @@ -18,7 +18,7 @@ For example: ``` {.diff filename="flake.nix"} { # ... -+ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; ++ inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11"; + inputs.epnix.url = "github:epics-extensions/EPNix"; # ... diff --git a/doc/nixos/guides/phoebus-alarm.md b/doc/nixos/guides/phoebus-alarm.md index 47c2ef9d..6c941b50 100644 --- a/doc/nixos/guides/phoebus-alarm.md +++ b/doc/nixos/guides/phoebus-alarm.md @@ -29,15 +29,17 @@ The Phoebus Alarm Logging Service can also be called the Phoebus Alarm Logger. # Single server Phoebus Alarm setup To configure Phoebus Alarm, Phoebus Alarm Logger, Apache Kafka, and ElasticSearch on a single server, -add this to your configuration: +add this to your configuration, +while taking care of replacing the IP address +and Kafka's `clusterId`: ``` nix -{config, lib, ...}: let - kafkaPort = toString config.services.apache-kafka.port; - # Replace this with your machine's IP address +{lib, pkgs, ...}: let + # Replace this with your machine's external IP address # or DNS domain name ip = "192.168.1.42"; - kafkaListenSockAddr = "${ip}:${kafkaPort}"; + kafkaListenSockAddr = "${ip}:9092"; + kafkaControllerListenSockAddr = "${ip}:9093"; in { # The Phoebus Alarm server also automatically enables the Phoebus Alarm Logger services.phoebus-alarm-server = { @@ -48,43 +50,48 @@ in { services.phoebus-alarm-logger.settings."bootstrap.servers" = kafkaListenSockAddr; - services.elasticsearch = { - enable = true; - package = pkgs.elasticsearch7; - }; - # Single-server Kafka setup services.apache-kafka = { enable = true; - logDirs = ["/var/lib/apache-kafka"]; - # Tell Apache Kafka to listen on this IP address - # If you don't have a DNS domain name, it's best to set a specific, non-local IP address. - extraProperties = '' - listeners=PLAINTEXT://${kafkaListenSockAddr} - offsets.topic.replication.factor=1 - transaction.state.log.replication.factor=1 - transaction.state.log.min.isr=1 - ''; + # Replace with a randomly generated uuid. You can get one by running: + # nix shell 'nixpkgs#apacheKafka' -c kafka-storage.sh random-uuid + clusterId = "xxxxxxxxxxxxxxxxxxxxxx"; + formatLogDirs = true; + settings = { + listeners = [ + "PLAINTEXT://${kafkaListenSockAddr}" + "CONTROLLER://${kafkaControllerListenSockAddr}" + ]; + # Adapt depending on your security constraints + "listener.security.protocol.map" = [ + "PLAINTEXT:PLAINTEXT" + "CONTROLLER:PLAINTEXT" + ]; + "controller.quorum.voters" = [ + "1@${kafkaControllerListenSockAddr}" + ]; + "controller.listener.names" = ["CONTROLLER"]; + + "node.id" = 1; + "process.roles" = ["broker" "controller"]; + + "log.dirs" = ["/var/lib/apache-kafka"]; + "offsets.topic.replication.factor" = 1; + "transaction.state.log.replication.factor" = 1; + "transaction.state.log.min.isr" = 1; + }; }; - systemd.services.apache-kafka = { - after = ["zookeeper.service"]; - unitConfig.StateDirectory = "apache-kafka"; - }; + systemd.services.apache-kafka.unitConfig.StateDirectory = "apache-kafka"; + + # Open kafka to the outside world + networking.firewall.allowedTCPPorts = [9092]; - services.zookeeper = { + services.elasticsearch = { enable = true; - extraConf = '' - # Port conflicts by default with phoebus-alarm-logger's port - admin.enableServer=false - ''; + package = pkgs.elasticsearch7; }; - # Open kafka to the outside world - networking.firewall.allowedTCPPorts = [ - config.services.apache-kafka.port - ]; - # Elasticsearch, needed by Phoebus Alarm Logger, is not free software (SSPL | Elastic License). # To accept the license, add the code below: nixpkgs.config.allowUnfreePredicate = pkg: @@ -205,4 +212,3 @@ Here is a list of options you might want to set: ::: callout-warning Currently, Phoebus Alarm Server only supports plain SMTP. ::: - diff --git a/doc/nixos/tutorials/archiver-appliance.md b/doc/nixos/tutorials/archiver-appliance.md index a1cdbbee..41681332 100644 --- a/doc/nixos/tutorials/archiver-appliance.md +++ b/doc/nixos/tutorials/archiver-appliance.md @@ -57,7 +57,7 @@ Fill the file with these lines: { description = "Configuration for running Archiver Appliance in a VM"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11"; inputs.epnix.url = "github:epics-extensions/EPNix"; outputs = { self, nixpkgs, epnix }: { @@ -72,11 +72,7 @@ Fill the file with these lines: } ``` -```{=html} - - -``` -Save and quit by typing ``{=html}Ctrl-x``{=html}, ``{=html}y``{=html}, and ``{=html}Enter``{=html}, +Save and quit by typing {{< kbd Ctrl-x >}}, {{< kbd y >}}, and {{< kbd Enter >}}, and run `nixos-rebuild test` to test your changes. Some explanations: diff --git a/flake.lock b/flake.lock index b3c2619a..acc7ae62 100644 --- a/flake.lock +++ b/flake.lock @@ -51,27 +51,141 @@ "type": "github" } }, + "flake-utils_3": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1694529238, + "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "poetry2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1698974481, + "narHash": "sha256-yPncV9Ohdz1zPZxYHQf47S8S0VrnhV7nNhCawY46hDA=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "4bb5e752616262457bc7ca5882192a564c0472d2", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1694499547, - "narHash": "sha256-R7xMz1Iia6JthWRHDn36s/E248WB1/je62ovC/dUVKI=", + "lastModified": 1707650010, + "narHash": "sha256-dOhphIA4MGrH4ElNCy/OlwmN24MsnEqFjRR6+RY7jZw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e5f018cf150e29aac26c61dac0790ea023c46b24", + "rev": "809cca784b9f72a5ad4b991e0e7bcf8890f9c3a6", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05", + "ref": "nixos-23.11", "repo": "nixpkgs", "type": "github" } }, + "poetry2nix": { + "inputs": { + "flake-utils": "flake-utils_3", + "nix-github-actions": "nix-github-actions", + "nixpkgs": [ + "nixpkgs" + ], + "systems": "systems_2", + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1707195113, + "narHash": "sha256-xPFxTMe4rKE/ZWLlOWv22qpGwpozpR+U1zhyf1040Zk=", + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "4eb2ac54029af42a001c9901194e9ce19cbd8a40", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "poetry2nix", + "type": "github" + } + }, "root": { "inputs": { "bash-lib": "bash-lib", "flake-utils": "flake-utils_2", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "poetry2nix": "poetry2nix" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "id": "systems", + "type": "indirect" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "poetry2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1699786194, + "narHash": "sha256-3h3EH1FXQkIeAuzaWB+nK0XK54uSD46pp+dMD3gAcB4=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "e82f32aa7f06bbbd56d7b12186d555223dc399d1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index a8980512..63c68160 100644 --- a/flake.nix +++ b/flake.nix @@ -1,12 +1,18 @@ { description = "A Nix flake containing EPICS-related modules and packages"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05"; - inputs.bash-lib = { - url = "github:minijackson/bash-lib"; - inputs.nixpkgs.follows = "nixpkgs"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.11"; + bash-lib = { + url = "github:minijackson/bash-lib"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + flake-utils.url = "github:numtide/flake-utils"; + poetry2nix = { + url = "github:nix-community/poetry2nix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; - inputs.flake-utils.url = "github:numtide/flake-utils"; outputs = { self, @@ -19,7 +25,11 @@ systemDependentOutputs = system: let pkgs = import nixpkgs { inherit system; - overlays = [overlay inputs.bash-lib.overlay]; + overlays = [ + overlay + inputs.bash-lib.overlay + inputs.poetry2nix.overlays.default + ]; }; in { packages = flake-utils.lib.flattenTree pkgs.epnix; @@ -46,7 +56,7 @@ category = "development tools"; } { - package = pkgs.quarto; + package = pkgs.quartoMinimal; category = "development tools"; } { diff --git a/ioc/modules/common.nix b/ioc/modules/common.nix index d98303da..11838b6b 100644 --- a/ioc/modules/common.nix +++ b/ioc/modules/common.nix @@ -51,6 +51,10 @@ with lib; { }; config = { - nixpkgs.overlays = [epnix.inputs.bash-lib.overlay epnix.overlays.default]; + nixpkgs.overlays = [ + epnix.inputs.poetry2nix.overlays.default + epnix.inputs.bash-lib.overlay + epnix.overlays.default + ]; }; } diff --git a/nixos/modules/phoebus/alarm-logger.nix b/nixos/modules/phoebus/alarm-logger.nix index 4e81d31d..47af9914 100644 --- a/nixos/modules/phoebus/alarm-logger.nix +++ b/nixos/modules/phoebus/alarm-logger.nix @@ -87,8 +87,6 @@ in { "bootstrap.servers" = lib.mkOption { description = "Location of the Kafka server"; type = lib.types.str; - default = "localhost:${toString config.services.apache-kafka.port}"; - defaultText = lib.literalExpression ''"localhost:''${toString config.services.apache-kafka.port}"''; }; date_span_units = lib.mkOption { diff --git a/nixos/modules/phoebus/alarm-server.nix b/nixos/modules/phoebus/alarm-server.nix index 2345f1a0..d7ed08f4 100644 --- a/nixos/modules/phoebus/alarm-server.nix +++ b/nixos/modules/phoebus/alarm-server.nix @@ -69,8 +69,6 @@ in { "org.phoebus.applications.alarm/server" = lib.mkOption { description = "Kafka server host:port"; type = lib.types.str; - default = "localhost:${toString config.services.apache-kafka.port}"; - defaultText = lib.literalExpression ''"localhost:''${toString config.services.apache-kafka.port}"''; }; # Waiting for: https://github.com/ControlSystemStudio/phoebus/issues/2843 diff --git a/nixos/tests/phoebus/alarm.nix b/nixos/tests/phoebus/alarm.nix index 4a5840b1..5db6467b 100644 --- a/nixos/tests/phoebus/alarm.nix +++ b/nixos/tests/phoebus/alarm.nix @@ -1,22 +1,17 @@ # This tests both the phoebus-alarm-server, and phoebus-alarm-logger services -{ - epnixLib, - lib, - pkgs, - ... -}: { +{epnixLib, ...}: { name = "phoebus-alarm-server-simple-check"; meta.maintainers = with epnixLib.maintainers; [minijackson]; nodes = { - client = { + client = {pkgs, ...}: { environment = { sessionVariables.EPICS_CA_ADDR_LIST = ["ioc"]; systemPackages = [pkgs.kcat pkgs.epnix.epics-base]; }; }; - ioc = { + ioc = {pkgs, ...}: { systemd.services.ioc = { description = "Test IOC to be monitored with the Phoebus Alarm server"; serviceConfig.ExecStart = "${pkgs.epnix.epics-base}/bin/softIoc -S -d ${./ioc.db}"; @@ -32,12 +27,13 @@ server = { config, + lib, pkgs, ... }: let - kafkaPort = toString config.services.apache-kafka.port; serverAddr = "192.168.1.3"; - kafkaListenSockAddr = "${serverAddr}:${kafkaPort}"; + kafkaListenSockAddr = "${serverAddr}:9092"; + kafkaControllerListenSockAddr = "${serverAddr}:9093"; in { services.phoebus-alarm-server = { enable = true; @@ -51,43 +47,43 @@ services.phoebus-alarm-logger.settings."bootstrap.servers" = kafkaListenSockAddr; - services.elasticsearch = { - enable = true; - package = pkgs.elasticsearch7; - }; - - # Single-server Kafka setup services.apache-kafka = { enable = true; - logDirs = ["/var/lib/apache-kafka"]; - # Tell Apache Kafka to listen on this IP address - # If you don't have a DNS domain name, it's best to set a specific, non-local IP address. - extraProperties = '' - listeners=PLAINTEXT://${kafkaListenSockAddr} - offsets.topic.replication.factor=1 - transaction.state.log.replication.factor=1 - transaction.state.log.min.isr=1 - ''; - }; + clusterId = "Wwbk0wwKTueL2hJD0IGGdQ"; + formatLogDirs = true; + settings = { + listeners = [ + "PLAINTEXT://${kafkaListenSockAddr}" + "CONTROLLER://${kafkaControllerListenSockAddr}" + ]; + "listener.security.protocol.map" = [ + "PLAINTEXT:PLAINTEXT" + "CONTROLLER:PLAINTEXT" + ]; + "controller.quorum.voters" = [ + "1@${kafkaControllerListenSockAddr}" + ]; + "controller.listener.names" = ["CONTROLLER"]; - systemd.services.apache-kafka = { - after = ["zookeeper.service"]; - unitConfig.StateDirectory = "apache-kafka"; + "node.id" = 1; + "process.roles" = ["broker" "controller"]; + + "log.dirs" = ["/var/lib/apache-kafka"]; + "offsets.topic.replication.factor" = 1; + "transaction.state.log.replication.factor" = 1; + "transaction.state.log.min.isr" = 1; + }; }; - services.zookeeper = { + systemd.services.apache-kafka.unitConfig.StateDirectory = ["apache-kafka"]; + + networking.firewall.allowedTCPPorts = [9092]; + + services.elasticsearch = { enable = true; - extraConf = '' - # Port conflicts by default with phoebus-alarm-logger's port - admin.enableServer=false - ''; + package = pkgs.elasticsearch7; }; - # Open kafka to the outside world - networking.firewall.allowedTCPPorts = [ - config.services.apache-kafka.port - ]; - nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ # Elasticsearch can be used as an SSPL-licensed software, which is diff --git a/pkgs/book/default.nix b/pkgs/book/default.nix index bd519688..23fcc96d 100644 --- a/pkgs/book/default.nix +++ b/pkgs/book/default.nix @@ -3,7 +3,7 @@ lib, epnixLib, epnix, - quarto, + quartoMinimal, writeText, documentedEpnixPkgs ? epnix, iocConfig ? {}, @@ -70,7 +70,7 @@ in name = "epnix-book"; src = ../../doc; - nativeBuildInputs = [quarto]; + nativeBuildInputs = [quartoMinimal]; dontConfigure = true; diff --git a/pkgs/doc-support/psu-simulator/default.nix b/pkgs/doc-support/psu-simulator/default.nix index 2a850e08..ec2776ec 100644 --- a/pkgs/doc-support/psu-simulator/default.nix +++ b/pkgs/doc-support/psu-simulator/default.nix @@ -3,12 +3,15 @@ epnix, epnixLib, }: +# Use recursiveUpdate so that it doesn't override meta.mainProgram +lib.recursiveUpdate (epnix.mkLewisSimulator { - name = "psu_simulator"; - package = "psu_simulator"; + name = "psu-simulator"; source = ./.; + package = "psu_simulator"; + device = "psu_simulator"; }) -// { +{ pname = "psu_simulator"; version = "0.2.0"; diff --git a/pkgs/epnix/epics-base/default.nix b/pkgs/epnix/epics-base/default.nix index 58524070..988ee2af 100644 --- a/pkgs/epnix/epics-base/default.nix +++ b/pkgs/epnix/epics-base/default.nix @@ -136,6 +136,12 @@ in # TODO: Some tests fail doCheck = false; + # _FORTIFY_SOURCE=3 causes a buffer overflow in some cases: + # *** buffer overflow detected ***: terminated + # + # Fall back to _FORTIFY_SOURCE=2 + hardeningDisable = ["fortify3"]; + meta = { description = "The Experimental Physics and Industrial Control System"; homepage = "https://epics-controls.org/"; diff --git a/pkgs/epnix/tools/lewis/lib.nix b/pkgs/epnix/tools/lewis/lib.nix index f6a61586..e90fd59e 100644 --- a/pkgs/epnix/tools/lewis/lib.nix +++ b/pkgs/epnix/tools/lewis/lib.nix @@ -1,5 +1,4 @@ { - lib, epnix, writeShellApplication, }: { diff --git a/pkgs/epnix/tools/phoebus/deps/default.nix b/pkgs/epnix/tools/phoebus/deps/default.nix index cafe6038..ed7a43da 100644 --- a/pkgs/epnix/tools/phoebus/deps/default.nix +++ b/pkgs/epnix/tools/phoebus/deps/default.nix @@ -58,7 +58,7 @@ stdenv.mkDerivation { outputHashAlgo = "sha256"; outputHashMode = "recursive"; - outputHash = "sha256-9MJdmIVAqjPW5ihZYWCh+zsWlxrtoHBH7NFwPh01pRc="; + outputHash = "sha256-6pUHn6tU54e4r5tJwoailVX3YEqO9BouWM3fweAdyt8="; doCheck = false; diff --git a/pkgs/epnix/tools/phoebus/olog/default.nix b/pkgs/epnix/tools/phoebus/olog/default.nix index 9bf50d96..213667ea 100644 --- a/pkgs/epnix/tools/phoebus/olog/default.nix +++ b/pkgs/epnix/tools/phoebus/olog/default.nix @@ -56,7 +56,7 @@ in outputHashAlgo = "sha256"; outputHashMode = "recursive"; - outputHash = "sha256-feT4qSN7B+KSUIoCmBkjDjsib39QfUW/4PxsU3Jx2Gs="; + outputHash = "sha256-40n06R2KBuuzqvVq1bWsd1jjQtcNQfK/4RbgtFmxTf8="; doCheck = false; }; diff --git a/templates/top/.gitignore b/templates/top/.gitignore index 4030d251..abf0cd0f 100644 --- a/templates/top/.gitignore +++ b/templates/top/.gitignore @@ -27,3 +27,9 @@ envPaths # Compilation database generated by bear or other compile_commands.json + +# Created by running IOCs +.iocsh_history + +# Created by running NixOS tests in interactive mode +.nixos-test-history diff --git a/templates/top/flake.nix b/templates/top/flake.nix index 3be88393..b0aaa1d8 100644 --- a/templates/top/flake.nix +++ b/templates/top/flake.nix @@ -33,7 +33,7 @@ # If you have a support module as a separate EPNix repository, # uncomment this line to make the package available: # --- - #overlays = [inputs.mySupportModule.overlays.default]; + #nixpkgs.overlays = [inputs.mySupportModule.overlays.default]; epnix = { inherit inputs;