diff --git a/docs/conf.py b/docs/conf.py index a56cfb77..852aff42 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -63,6 +63,12 @@ }, } +# -- Options for Sphinx Copybutton ------------------------------------------- +# https://sphinx-copybutton.readthedocs.io/en/latest/use.html + +# Exclude copying line numbers, prompts, and prompt outputs +copybutton_exclude = ".linenos, .gp, .go" + # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/docs/ioc/tutorials/imgs/integration-vm-ioc-screenshot.png b/docs/ioc/tutorials/imgs/integration-vm-ioc-screenshot.png new file mode 100644 index 00000000..a1948182 Binary files /dev/null and b/docs/ioc/tutorials/imgs/integration-vm-ioc-screenshot.png differ diff --git a/docs/ioc/tutorials/imgs/integration-vm-simulator-screenshot.png b/docs/ioc/tutorials/imgs/integration-vm-simulator-screenshot.png new file mode 100644 index 00000000..8e0ebcf9 Binary files /dev/null and b/docs/ioc/tutorials/imgs/integration-vm-simulator-screenshot.png differ diff --git a/docs/ioc/tutorials/integration-tests.rst b/docs/ioc/tutorials/integration-tests.rst index 77be658e..7ef2fbf4 100644 --- a/docs/ioc/tutorials/integration-tests.rst +++ b/docs/ioc/tutorials/integration-tests.rst @@ -1,37 +1,97 @@ -Integration tests -================= +Adding integration tests to your IOC +==================================== + +.. note:: + This tutorial is a continuation of the :doc:`streamdevice` tutorial. + + If you haven't already, follow the StreamDevice tutorial first. + +In this tutorial, +you'll learn how to test your created StreamDevice IOC +by running it against the simulator, +inside a declared NixOS VM, +and checking that it behaves as expected. + +This method of testing can then be automated +by running it inside a Continuous Integration (CI) system. + +Pre-requisites +-------------- + +.. warning:: + Nix assumes you can run hardware-accelerated VMs, + through KVM. + +Make sure that you have KVM on your Linux machine +by checking if the file :file:`/dev/kvm` is present. + +If the file is present, +you can proceed to the next section. + +If you don't have KVM, +and you're running Nix on a physical machine, +examine your firmware settings +to see if you can enable hardware-accelerated virtualization. +The setting can show up as: + +- Virtualization +- Intel Virtualization Technology +- Intel VT +- VT-d +- SVM Mode +- AMD-v + +If you don't have KVM, +and you're running Nix on a virtual machine, +check your firmware settings +as said before, +and look up your hypervisor documentation +to enable nested virtualization. + +If this doesn't work, +you can still proceed without hardware acceleration +by adding this line to your :file:`nix.conf`: + +.. code-block:: + :caption: :file:`/etc/nix/nix.conf` + + extra-system-features = kvm + +Note that this means much slower integration tests. Writing the test ---------------- Through the `NixOS testing framework`_, EPNix provides a way of specifying a machine configuration, -and running a Python script that can do various kind of testing. +and running a Python script that can do various kinds of testing. -If you created your IOC using the EPNix template, -like suggested in the :doc:`streamdevice` tutorial, -you will see a ``checks/`` directory. -This directory should contain the integration tests you want to run. +With your IOC created during the :doc:`streamdevice` tutorial, +you'll see a :file:`checks/` directory, +which is the place to add your integration tests. -To add an integration test to EPNix, -record it in your ``flake.nix`` under the ``epnix.checks.files`` option. +These tests are imported using the :ref:`opt-epnix.checks.imports` option. -For example, in the EPNix template, you will see in your ``flake.nix`` file: +For example, +in the EPNix template, +you'll see in your :file:`flake.nix` file: .. code-block:: nix + :caption: :file:`flake.nix`: importing an integration test - checks.files = [ ./checks/simple.nix ]; + checks.imports = [ ./checks/simple.nix ]; -The ``./checks/.nix`` file should contain a NixOS test like so: +The :file:`./checks/simple.nix` file should contain a NixOS test such as this: .. code-block:: nix + :caption: :file:`checks/simple.nix`: structure of a test { build, pkgs, ... }: pkgs.nixosTest { - name = "myTest"; + name = "simple"; - machine = { + nodes.machine = {config, ...}: { # Description of the NixOS machine... }; @@ -40,102 +100,422 @@ The ``./checks/.nix`` file should contain a NixOS test like so: ''; } -This test will create a NixOS virtual machine +Running this test creates a NixOS virtual machine from the given configuration, -and run the test script. -Note that the test script does *not* run on the virtual machine, -but communicates with it. -This is because the test script can start, +and runs the test script. + +The test script can, +among other things, +run commands on the machine, +start, shut down, -or reboot the machine, -and also because NixOS tests can also manage several virtual machines, -not just one. +or reboot the machine. -For an overview of what you can input in the machine configuration, -please refer to the `NixOS documentation`_. -You can also read more about it -in the `Python test script API documentation`_. +.. tip:: + The Python test script does *not* run on the virtual machine, + but communicates with it. + + If you want to run Python code on the VM machine, + you need to package it and run it as a command. + +For a more detailed overview of what you can put in the machine configuration, +examine the `NixOS documentation`_, +or the :doc:`../../nixos-services/tutorials/archiver-appliance` tutorial. .. _NixOS testing framework: https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests .. _NixOS documentation: https://nixos.org/manual/nixos/stable/index.html#sec-configuration-syntax -.. _Python test script API documentation: https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests Starting your IOC through systemd --------------------------------- -We recommend starting your IOC through a systemd service, -which you can describe in Nix like so: +First, you need to ensure that your IOC will start inside the VM. -.. TODO: change that +In the default template, +you'll see this particular configuration: .. code-block:: nix + :caption: :file:`checks/simple.nix`: config for starting an IOC + :emphasize-lines: 3-4,8 + + nodes.machine = {config, ...}: { + imports = [ + epnix.nixosModules.ioc + epnixConfig + ]; + environment.systemPackages = [pkgs.epnix.epics-base]; + + systemd.services.ioc = config.epnix.nixos.services.ioc.config; + }; + +The first two emphasized lines are about importing the ability to define an IOC, +and then importing your IOC configuration +that you defined in your :file:`flake.nix`. + +Then, +on the last emphasized line, +the systemd service configuration generated by EPNix is used +to generate ``ioc.service``. + +EPNix uses the configuration :ref:`opt-epnix.nixos.services` +from your :file:`flake.nix` +to figure out the name of your app +and the name of your :file:`iocBoot` folder. + +Make sure yours is correct in your :file:`flake.nix`: + +.. code-block:: nix + :caption: :file:`flake.nix`: configuring the name of your app + and iocBoot folder for the test systemd service + :emphasize-lines: 5-6 + + # Used when generating NixOS systemd services, for example for + # deployment to production, or for the NixOS tests in checks/ + # --- + nixos.services.ioc = { + app = "example"; # Name of your app + ioc = "iocExample"; # Name of your iocBoot folder + }; + +Also take note of the package :ref:`pkg-epics-base` being installed, +with the ``environment.systemPackages`` option. +This enables you to use the :command:`caget`, :command:`caput` commands +inside the VM. + +.. _run-test: + +Running the test +---------------- + +To run the test, +run this command: + +.. code-block:: bash + :caption: Running the test "simple" + + nix build -L '.#checks.x86_64-linux.simple' - # Inside the `machine` attribute - { - systemd.services.my-ioc = { - wantedBy = [ "multi-user.target" ]; - serviceConfig = { - ExecStart = "${build}/iocBoot/iocexample/st.cmd"; - WorkingDirectory = "${build}/iocBoot/iocexample"; - - # Makes the EPICS command-line not quit for 100 seconds, if it doesn't - # receive anything on the standard input - StandardInputText = "epicsThreadSleep(100)"; +If you left the test script as-is, +you should see that the test fails. +That's because the test script is currently not adapted to our IOC. + +We'll change it afterward, +but for now in the logs you should see your IOC being run. + +If you have several tests, +you can run them all using: + +.. code-block:: bash + :caption: Running all tests + + nix flake check -L + +.. _run-driverInteractive: + +Running the test interactively +------------------------------ + +It's often desirable to run the VM interactively, +to figure out what works and what doesn't, +before writing the test. + +To do so, +run: + +.. code-block:: bash + :caption: Running the test "simple" interactively + + nix run -L '.#checks.x86_64-linux.simple.driverInteractive' + +This runs a Python shell prompt in the same environment as the test script. +Any command run here is the same as running it in the test script, +but interactively. + +You can use the ``start_all()`` functions +to start all VMs that you declared in ``nodes``: + +.. code-block:: pycon + + >>> start_all() + +In our case, +we only defined ``machine``, +so this starts a single VM, +and runs your IOC inside it. + +You can log in to that VM with the user ``root`` and no password. +You can then run any command you want +to inspect the state of the VM. + +.. figure:: ./imgs/integration-vm-ioc-screenshot.png + :alt: Integration VM screenshot showing the IOC running + + Integration VM screenshot showing the IOC running + +.. tip:: + If you have a non-English-language keyboard, + change your keyboard layout inside the VM by using :command:`loadkeys`. + + For example, + to set the keyboard to "french": + + .. code-block:: console + + [root@machine:~]# loadkeys fr + +.. tip:: + To exit the Python shell prompt, + press :kbd:`Ctrl-d`, then :kbd:`y`. + + Exiting the Python shell prompt automatically shuts down the VMs. + +Adding the simulator +-------------------- + +The simulator is a program listening on port 9999. +Inside the test VM, +it should be a program run by a systemd service. + +Same as the IOC, +you should use the ``systemd.services`` options. + +Change your Nix test file like this: + +.. code-block:: nix + :caption: Adding the simulator as systemd service, + important changes emphasized + :emphasize-lines: 1,10-13 + + nodes.machine = {config, lib, ...}: { + imports = [ + epnix.nixosModules.ioc + epnixConfig + ]; + environment.systemPackages = [pkgs.epnix.epics-base]; + + systemd.services = { + ioc = config.epnix.nixos.services.ioc.config; + simulator = { + serviceConfig.ExecStart = lib.getExe pkgs.epnix.psu-simulator; + wantedBy = ["multi-user.target"]; + }; }; }; - # Provides the caget / caput / etc. commands to the test script - environment.systemPackages = [ pkgs.epnix.epics-base ]; - } +The first emphasized line is about adding the ``lib`` argument used below. -You can view the list of options available for a NixOS machine `here `__. +The second set of emphasized lines is about creating the ``simulator.service`` systemd service. These lines will generate the following service file: -Then, you can write your test script. -Note that the test script doesn’t run directly on the machine, -but communicates with the machine through the ``machine`` variable. +.. code-block:: dosini + :caption: generated :file:`/etc/systemd/system/simulator.service` -An example of a testing script: + [Unit] + + [Service] + # ... + ExecStart=/nix/store/...-psu-simulator/bin/psu-simulator + +And this service is automatically started at boot, +by being a dependency of ``multi-user.target``. + +The ``serviceConfig`` option adds configuration keys to the ``[Service]`` section. +Here, +we set ``ExecStart`` to main executable program of the ``psu-simulator`` package, +by using the ``lib.getExe`` function. + +A ``unitConfig`` for the ``[Unit]`` section also exists. + +The ``[Install]`` section isn't present in NixOS, +because managed differently, +by using options such as ``wantedBy``, ``requiredBy``, etc. + +For more information, +see the `systemd.services`_ options in the NixOS manual. + +.. _systemd.services: https://nixos.org/manual/nixos/stable/options#opt-systemd.services + +---- + +With this configuration, +you can run the VM interactively +(see :ref:`run-driverInteractive`), +and you should see the simulator up and running after booting. + +.. tip:: + If you make changes to your configuration, + or your IOC, + you *don't* need to rebuild anything + before running the ``nix run`` command. + + Nix will by itself figure out what it needs to rebuild, + and rebuild it before running the test. + +.. figure:: ./imgs/integration-vm-simulator-screenshot.png + :alt: Integration VM screenshot showing the simulator running + + Integration VM screenshot showing the simulator running + +Writing the test +---------------- + +Now that the VM configuration is appropriate, +you can start writing your test script. + +Here is a sample of useful Python functions: + +.. py:function:: start_all() + + Start all defined VMs + +.. py:function:: Machine.wait_for_unit(self, unit: str, user: str | None = None, timeout: int = 900) + + Wait for a systemd unit to get into “active” state. + Throws exceptions on “failed” and “inactive” states + as well as after timing out. + + .. code-block:: python + :caption: Example + + machine.wait_for_unit("ioc.service") + +.. py:function:: Machine.succeed(self, command: str, timeout: int | None = None) + + Execute a shell command, + raising an exception if the exit status is not zero, + otherwise returning the standard output + + .. code-block:: python + :caption: Example + + machine.succeed("caput VOLT 42") + +.. py:function:: Machine.wait_until_succeeds(self, command: str, timeout: int = 900) + + Repeat a shell command with 1-second intervals until it succeeds. + + Be careful of the ``s`` in ``succeeds``. + + .. code-block:: python + :caption: Example + + machine.wait_until_succeeds("caget -t my:stringout | grep -qxF 'expected value'") + +.. py:function:: Machine.fail(self, command: str, timeout: int | None = None) + + Like :py:func:`succeed`, + but raising an exception if the command returns a zero status. + + .. code-block:: python + :caption: Example + + machine.fail("caget unknown-PV") + +.. py:function:: Machine.wait_for_open_port(self, addr: int | str, timeout: int = 900) + + Wait until a process is listening on the given TCP port and IP address (default ``localhost``). + + .. code-block:: python + :caption: Example + + machine.wait_for_open_port(9999) + +.. py:function:: retry(fn: Callable, timeout: int = 900) + + Call the given function repeatedly, with 1-second intervals, + until it returns ``True`` or a timeout is reached. + + .. code-block:: python + :caption: Example + + def check_value(_last_call: bool) -> bool: + """Check whether the VOLT-RB PV is 42.""" + value = float(machine.succeed("caget -t VOLT-RB")) + return value == 42. + + retry(check_value, timeout=10) + +.. py:function:: subtest(name: str) + + Group logs under a given test name. + + To be used with the ``with`` syntax. + + .. code-block:: python + :caption: Example + + with subtest("check voltage"): + test_setting_voltage() + test_voltage_readback() + ... + +You can also read more about the Python functions available in the test script +in the `NixOS tests documentation`_. + +.. _NixOS tests documentation: https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests + +Example test script +^^^^^^^^^^^^^^^^^^^ + +Here an example test script +that should work with your StreamDevice IOC: .. code-block:: python + :caption: :file:`checks/simple.nix`: Example test script start_all() - machine.wait_for_unit("default.target") - machine.wait_for_unit("my-ioc.service") + with subtest("check services"): + machine.wait_for_unit("ioc.service") + machine.wait_for_unit("simulator.service") + machine.wait_for_unit("default.target") - machine.wait_until_succeeds("caget stringin") - machine.wait_until_succeeds("caget stringout") - machine.fail("caget non-existing") + machine.wait_for_open_port(9999) - with subtest("testing stringout"): - def test_stringout(_) -> bool: - machine.succeed("caput stringout 'hello'") - status, _output = machine.execute("caget -t stringout | grep -qxF 'hello'") + # Prefer using 'wait_until_succeeds', + # since the 'ioc.service' being active doesn't necessarily means + # that the IOC is initialized + machine.wait_until_succeeds("caget VOLT-RB", timeout=10) + machine.fail("caget unknown-PV") - return status == 0 + with subtest("check voltage"): + # Initial value is zero + machine.succeed("caget -t VOLT-RB | grep -qxF '0'") - retry(test_stringout) + machine.succeed("caput VOLT 42") - assert "hello" not in machine.succeed("caget -t stringin") + def check_value(_last_call: bool) -> bool: + """Check whether the VOLT-RB PV is 42.""" + value = float(machine.succeed("caget -t VOLT-RB")) + return value == 42. -Note that the script extensively uses the ``wait_until_succeeds`` method and the ``retry`` function. -This is because EPICS has few guarantees about whether it propagates changes immediately, -and so it’s better to encourage the use of retries, + retry(check_value, timeout=10) + +Note that the script uses the ``wait_until_succeeds`` method and the ``retry`` function. +This is because EPICS has few guarantees about whether it propagates changes immediately. +It’s better to encourage the use of retries, instead of hoping the timing lines up. -If you would like to use a fully fledged python script on the machine, -which can use Python dependencies like pyepics, -please refer to the guide :doc:`../user-guides/testing/packaging-python-scripts`. +After changing your test script, +run your test as explained in :ref:`run-test`. -You can find methods available on the ``machine`` variable and other particularities in the `NixOS tests documentation`_. +Next steps +---------- -You can also look at examples either in the EPNix repository, -under the `ioc/tests folder`_, -or in nixpkgs under the `nixos/tests folder`_. +You can examine other NixOS test examples: -.. TODO: this doesn't explain how to run the test +- In the `EPNix' ioc/tests`_ folder, for IOC tests, +- In the `EPNix' nixos/tests`_ folder, for EPICS-related NixOS services tests, +- Or in the `nixpkgs' nixos/tests`_ folder. -.. _Packaging Python scripts: ../guides/testing/packaging-python-scripts.md -.. _NixOS tests documentation: https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests -.. _ioc/tests folder: https://github.com/epics-extensions/epnix/tree/master/ioc/tests -.. _nixos/tests folder: https://github.com/NixOS/nixpkgs/tree/master/nixos/tests +If you'd like to run a complete python script on the test VM, +which can use Python dependencies such as ``pyepics``, +examine the guide :doc:`../user-guides/testing/packaging-python-scripts`. + +If you're interested in adding unit tests, +examine the :doc:`../user-guides/testing/unit-testing` guide. + +For all testing related guides, +see :doc:`../user-guides/testing/index`. + +.. _EPNix' ioc/tests: https://github.com/epics-extensions/epnix/tree/master/ioc/tests +.. _EPNix' nixos/tests: https://github.com/epics-extensions/epnix/tree/master/nixos/tests +.. _nixpkgs' nixos/tests: https://github.com/NixOS/nixpkgs/tree/master/nixos/tests diff --git a/docs/ioc/tutorials/streamdevice.rst b/docs/ioc/tutorials/streamdevice.rst index 528a7c9e..0898d7f1 100644 --- a/docs/ioc/tutorials/streamdevice.rst +++ b/docs/ioc/tutorials/streamdevice.rst @@ -85,7 +85,7 @@ change yours like so: # Add one of the supported modules here: # --- - - #support.modules = with pkgs.epnix.support; [ StreamDevice ]; + - #support.modules = with pkgs.epnix.support; [ StreamDevice mySupportModule ]; + support.modules = with pkgs.epnix.support; [ StreamDevice ]; Then, @@ -147,11 +147,11 @@ and what to expect in return. Terminator = LF; getVoltage { - out ":VOLT?"; in "%f"; + out ":volt?"; in "%f"; } setVoltage { - out ":VOLT %f"; + out ":volt %f"; @init { getVoltage; } } @@ -160,7 +160,7 @@ That file specifies the name, type, and properties of the Process Variables (PV) that EPICS exposes over the network. It also specifies how they relate to the functions written in the protocol file. -.. code-block:: perl +.. code-block:: bash :caption: :file:`exampleApp/Db/example.db` record(ai, "${PREFIX}VOLT-RB") { @@ -171,6 +171,7 @@ It also specifies how they relate to the functions written in the protocol file. record(ao, "${PREFIX}VOLT") { field(DTYP, "stream") field(OUT, "@example.proto setVoltage ${PORT}") + field(FLNK, "${PREFIX}VOLT-RB") } Change ``exampleApp/Db/Makefile`` @@ -207,7 +208,7 @@ and how to connect to the remote power supply. # Where to find the protocol files epicsEnvSet("STREAM_PROTOCOL_PATH", "${TOP}/db") # The TCP/IP address of the power supply - drvAsynIPPortConfigure("PS1", "localhost:8727") + drvAsynIPPortConfigure("PS1", "localhost:9999") ## Load record instances dbLoadRecords("${TOP}/db/example.db", "PREFIX=, PORT=PS1") @@ -252,7 +253,29 @@ Then, run: ./st.cmd -You should see the IOC starting and connecting to ``localhost:8727``. +You should see the IOC starting and connecting to ``localhost:9999``. + +.. tip:: + :file:`./result` is a symbolic link, + so if you made any changes to your IOC and re-ran ``nix build``, + a terminal window already in :file:`./result/iocBoot/iocExample` will still point to the old version. + + To run the new version, + either re-open a new window + and ``cd`` into the new :file:`./result/`, + or in the old location, + you can run: + + .. code-block:: console + + user@machine .../result/iocBoot/iocExample $ cd . + + For quickly re-running an IOC, + you can use this command: + + .. code-block:: console + + user@machine .../result/iocBoot/iocExample $ cd . ; ./st.cmd Recompiling with make --------------------- @@ -274,9 +297,9 @@ and open a direct connection to the simulator: .. code-block:: bash - nc localhost 8727 + nc localhost 9999 # or - telnet localhost 8727 + telnet localhost 9999 You can install the ``nc`` command through the ``netcat`` package, or you can install the ``telnet`` command through the ``telnet`` package, diff --git a/templates/top/checks/simple.nix b/templates/top/checks/simple.nix index 54a5948d..f511af04 100644 --- a/templates/top/checks/simple.nix +++ b/templates/top/checks/simple.nix @@ -22,8 +22,8 @@ pkgs.nixosTest { machine.wait_for_unit("default.target") machine.wait_for_unit("ioc.service") - machine.wait_until_succeeds("caget stringin") - machine.wait_until_succeeds("caget stringout") + machine.wait_until_succeeds("caget stringin", timeout=10) + machine.wait_until_succeeds("caget stringout", timeout=10) machine.fail("caget non-existing") with subtest("testing stringout"):