diff --git a/partition/roles/dhcp/defaults/main.yaml b/partition/roles/dhcp/defaults/main.yaml index 96794a45..b412604c 100644 --- a/partition/roles/dhcp/defaults/main.yaml +++ b/partition/roles/dhcp/defaults/main.yaml @@ -19,7 +19,7 @@ dhcp_global_options: [] # examples: # - default-url = "http://{{ ansible_host }}/onie-installer" # - ztp_provisioning_script_url code 239 = text -# - ztp_provisioning_script_url "http://{{ ansible_host }}/ztp.sh" +# - ztp_provisioning_script_url "http://{{ ansible_host }}/user.sh" dhcp_global_deny_list: [] # examples: diff --git a/partition/roles/sonic/README.md b/partition/roles/sonic/README.md index 3e6f921c..9352b327 100644 --- a/partition/roles/sonic/README.md +++ b/partition/roles/sonic/README.md @@ -18,6 +18,7 @@ It depends on the `switch_facts` module from `ansible-common`, so make sure modu | sonic_ip_masquerade | | Enable ip masquerading on eth0. | | sonic_breakouts | | The breakout configuration for ports, e.g. `dict('Ethernet0'='4x25G')` | | sonic_config_action | | Either `load` or `reload`. In the latter case all services will be restarted. If not given, defaults to `load` | +| sonic_render_config_db_template | | When `true` the `metal.yaml.j2` template will be rendered into `/etc/sonic/config_db.json` | | sonic_ports | | Configuration for ports (mtu, fec, have highest precedence). These ports will be up by default. | | sonic_ports.name | | The port name. | | sonic_ports.speed | | Speed of the port. | diff --git a/partition/roles/sonic/defaults/main.yaml b/partition/roles/sonic/defaults/main.yaml index 1506557c..f0445fdb 100644 --- a/partition/roles/sonic/defaults/main.yaml +++ b/partition/roles/sonic/defaults/main.yaml @@ -5,6 +5,7 @@ sonic_nameservers: [] sonic_ip_masquerade: false sonic_timezone: Europe/Berlin sonic_config_action: load +sonic_render_config_db_template: true ## Physical settings sonic_ports: [] diff --git a/partition/roles/sonic/tasks/config_db.yaml b/partition/roles/sonic/tasks/config_db.yaml new file mode 100644 index 00000000..28478855 --- /dev/null +++ b/partition/roles/sonic/tasks/config_db.yaml @@ -0,0 +1,79 @@ +--- +- name: Check mandatory variables on non-empty sonic_ports are set + assert: + fail_msg: "default port configuration is necessary on non-empty sonic_ports" + quiet: yes + that: + - sonic_ports_default_speed + - sonic_ports_default_mtu + when: sonic_ports + +- name: Check mandatory variables on non-empty sonic_portchannels are set + assert: + fail_msg: "default configuration is necessary on non-empty sonic_portchannels" + quiet: yes + that: + - sonic_portchannels_default_mtu + when: sonic_portchannels + +- name: Populate sonic_ports_dict + set_fact: + sonic_ports_dict: "{{ sonic_ports_dict|default({}) | combine( {item.name: item} ) }}" + loop: "{{ sonic_ports }}" + +# Dependencies are returned by config. +- name: Configure breakouts + command: "config interface breakout --yes {{ item.key }} '{{ item.value }}'" + register: breakout_result + changed_when: "'Breakout process got successfully completed.' in breakout_result.stdout" + failed_when: "breakout_result.rc != 0 or 'Dependecies Exist. No further action will be taken' in breakout_result.stdout" + with_dict: "{{ sonic_breakouts }}" + when: sonic_breakouts is defined + +- name: Delete deprecated metal.yaml + ansible.builtin.file: + path: "/etc/sonic/metal.yaml" + state: absent + +- name: Get running configuration + ansible.builtin.command: show runningconfiguration all + register: sonic_running_cfg_result + changed_when: false + +- name: Parse running configuration + ansible.builtin.set_fact: + sonic_running_cfg: "{{ sonic_running_cfg_result.stdout | from_json }}" + +- name: Extract running configuration for breakouts and ports + ansible.builtin.set_fact: + sonic_running_cfg_breakouts: "{{ sonic_running_cfg | community.general.json_query('BREAKOUT_CFG') }}" + sonic_running_cfg_hwsku: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.hwsku') }}" + sonic_running_cfg_mac: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.mac') }}" + sonic_running_cfg_platform: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.platform') }}" + sonic_running_cfg_ports: "{{ sonic_running_cfg | community.general.json_query('PORT') }}" + +- name: Fail if running configuration doesn't contain required information + ansible.builtin.assert: + that: + - sonic_running_cfg_hwsku + - sonic_running_cfg_mac + - sonic_running_cfg_platform + - sonic_running_cfg_ports + fail_msg: The running configuration is incomplete because it does not contain 'PORT' or complete 'DEVICE_METADATA'. + +- name: Fail if running configuration doesn't contain breakout configuration + ansible.builtin.assert: + that: + - sonic_running_cfg_breakouts + fail_msg: The running configuration is incomplete because it does not contain 'BREAKOUT_CFG'. + when: sonic_breakouts is defined + +- name: Render config_db + set_fact: + config_db: "{{ lookup('template', 'metal.yaml.j2') }}" + +- name: Save config_db as JSON file + copy: + content: "{{ config_db | from_yaml | to_nice_json }}" + dest: /etc/sonic/config_db.json + notify: "config {{ sonic_config_action }}" diff --git a/partition/roles/sonic/tasks/main.yaml b/partition/roles/sonic/tasks/main.yaml index 1bae92e4..bbf3263c 100644 --- a/partition/roles/sonic/tasks/main.yaml +++ b/partition/roles/sonic/tasks/main.yaml @@ -15,28 +15,6 @@ - sonic_nameservers is defined - metal_stack_switch_os_is_sonic -- name: Check mandatory variables on non-empty sonic_ports are set - assert: - fail_msg: "default port configuration is necessary on non-empty sonic_ports" - quiet: yes - that: - - sonic_ports_default_speed - - sonic_ports_default_mtu - when: sonic_ports - -- name: Check mandatory variables on non-empty sonic_portchannels are set - assert: - fail_msg: "default configuration is necessary on non-empty sonic_portchannels" - quiet: yes - that: - - sonic_portchannels_default_mtu - when: sonic_portchannels - -- name: Populate sonic_ports_dict - set_fact: - sonic_ports_dict: "{{ sonic_ports_dict|default({}) | combine( {item.name: item} ) }}" - loop: "{{ sonic_ports }}" - - name: render resolv.conf template: src: resolv.conf.j2 @@ -58,62 +36,9 @@ value: "1" when: sonic_ip_masquerade -# Dependencies are returned by config. -- name: Configure breakouts - command: "config interface breakout --yes {{ item.key }} '{{ item.value }}'" - register: breakout_result - changed_when: "'Breakout process got successfully completed.' in breakout_result.stdout" - failed_when: "breakout_result.rc != 0 or 'Dependecies Exist. No further action will be taken' in breakout_result.stdout" - with_dict: "{{ sonic_breakouts }}" - when: sonic_breakouts is defined - -- name: Delete deprecated metal.yaml - ansible.builtin.file: - path: "/etc/sonic/metal.yaml" - state: absent - -- name: Get running configuration - ansible.builtin.command: show runningconfiguration all - register: sonic_running_cfg_result - changed_when: false - -- name: Parse running configuration - ansible.builtin.set_fact: - sonic_running_cfg: "{{ sonic_running_cfg_result.stdout | from_json }}" - -- name: Extract running configuration for breakouts and ports - ansible.builtin.set_fact: - sonic_running_cfg_breakouts: "{{ sonic_running_cfg | community.general.json_query('BREAKOUT_CFG') }}" - sonic_running_cfg_hwsku: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.hwsku') }}" - sonic_running_cfg_mac: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.mac') }}" - sonic_running_cfg_platform: "{{ sonic_running_cfg | community.general.json_query('DEVICE_METADATA.localhost.platform') }}" - sonic_running_cfg_ports: "{{ sonic_running_cfg | community.general.json_query('PORT') }}" - -- name: Fail if running configuration doesn't contain required information - ansible.builtin.assert: - that: - - sonic_running_cfg_hwsku - - sonic_running_cfg_mac - - sonic_running_cfg_platform - - sonic_running_cfg_ports - fail_msg: The running configuration is incomplete because it does not contain 'PORT' or complete 'DEVICE_METADATA'. - -- name: Fail if running configuration doesn't contain breakout configuration - ansible.builtin.assert: - that: - - sonic_running_cfg_breakouts - fail_msg: The running configuration is incomplete because it does not contain 'BREAKOUT_CFG'. - when: sonic_breakouts is defined - -- name: Render config_db - set_fact: - config_db: "{{ lookup('template', 'metal.yaml.j2') }}" - -- name: Save config_db as JSON file - copy: - content: "{{ config_db | from_yaml | to_nice_json }}" - dest: /etc/sonic/config_db.json - notify: "config {{ sonic_config_action }}" +- name: Render and save config_db + import_tasks: config_db.yaml + when: sonic_render_config_db_template - name: Set NTP timezone timezone: diff --git a/partition/roles/sonic/templates/frr.conf.j2 b/partition/roles/sonic/templates/frr.conf.j2 index e903786e..2feb614b 100644 --- a/partition/roles/sonic/templates/frr.conf.j2 +++ b/partition/roles/sonic/templates/frr.conf.j2 @@ -4,6 +4,7 @@ hostname {{ inventory_hostname }} ! service integrated-vtysh-config ! +agentx log syslog {{ sonic_frr_syslog_level }} {% if sonic_frr_debug_options is defined %} {% for option in sonic_frr_debug_options %} diff --git a/partition/roles/sonic/test/data/exit/frr.conf b/partition/roles/sonic/test/data/exit/frr.conf index d6e31f66..fd8ab387 100644 --- a/partition/roles/sonic/test/data/exit/frr.conf +++ b/partition/roles/sonic/test/data/exit/frr.conf @@ -3,6 +3,7 @@ hostname exit01 ! service integrated-vtysh-config ! +agentx log syslog informational ! vrf VrfMpls @@ -105,4 +106,4 @@ route-map LOOPBACKS permit 10 ip route 0.0.0.0/0 10.1.2.1 ! line vty -! \ No newline at end of file +! diff --git a/partition/roles/sonic/test/data/l2_leaf/frr.conf b/partition/roles/sonic/test/data/l2_leaf/frr.conf index e4e1b838..8fda0baf 100644 --- a/partition/roles/sonic/test/data/l2_leaf/frr.conf +++ b/partition/roles/sonic/test/data/l2_leaf/frr.conf @@ -3,6 +3,7 @@ hostname l2leaf01 ! service integrated-vtysh-config ! +agentx log syslog informational ! vrf Vrf46 @@ -62,4 +63,4 @@ route-map LOOPBACKS permit 10 match interface Loopback0 ! line vty -! \ No newline at end of file +! diff --git a/partition/roles/sonic/test/data/mgmtleaf/frr.conf b/partition/roles/sonic/test/data/mgmtleaf/frr.conf index ff440493..a33e16b6 100644 --- a/partition/roles/sonic/test/data/mgmtleaf/frr.conf +++ b/partition/roles/sonic/test/data/mgmtleaf/frr.conf @@ -3,6 +3,7 @@ hostname r01mgmtleaf ! service integrated-vtysh-config ! +agentx log syslog informational ! interface Ethernet120 @@ -31,4 +32,4 @@ route-map DENY_MGMT deny 10 route-map DENY_MGMT permit 20 ! line vty -! \ No newline at end of file +! diff --git a/partition/roles/sonic/test/data/sonic-vs/frr.conf b/partition/roles/sonic/test/data/sonic-vs/frr.conf index 120ac33e..dea03d84 100644 --- a/partition/roles/sonic/test/data/sonic-vs/frr.conf +++ b/partition/roles/sonic/test/data/sonic-vs/frr.conf @@ -3,6 +3,7 @@ hostname sonic-vs ! service integrated-vtysh-config ! +agentx log syslog informational ! interface Ethernet0 @@ -26,4 +27,4 @@ route-map DENY_MGMT deny 10 route-map DENY_MGMT permit 20 ! line vty -! \ No newline at end of file +! diff --git a/partition/roles/sonic/test/data/spine/frr.conf b/partition/roles/sonic/test/data/spine/frr.conf index 5826ab01..e35fb497 100644 --- a/partition/roles/sonic/test/data/spine/frr.conf +++ b/partition/roles/sonic/test/data/spine/frr.conf @@ -3,6 +3,7 @@ hostname spine01 ! service integrated-vtysh-config ! +agentx log syslog informational ! interface Ethernet120 @@ -35,4 +36,4 @@ route-map LOOPBACKS permit 10 match interface Loopback0 ! line vty -! \ No newline at end of file +! diff --git a/partition/roles/ztp/README.md b/partition/roles/ztp/README.md index f49ae6f1..b6f7ca57 100644 --- a/partition/roles/ztp/README.md +++ b/partition/roles/ztp/README.md @@ -2,24 +2,11 @@ Configures a server for providing zero-touch-provisioning scripts for switches. -## Variables - -| Name | Mandatory | Description | -| -------------------- | --------- | ----------------------------------------------------------- | -| ztp_nginx_image_name | yes | the docker image to use to serve ztp scripts. | -| ztp_nginx_image_tag | yes | the tag of the docker image to use to serve ztp scripts. | -| ztp_host_dir_path | | the path to serve ztp scripts from. | -| ztp_listen_address | | the address used to serve ztp requests | -| ztp_port | | the port to serve ztp scripts on. | -| ztp_authorized_keys | yes | the authorized keys that should be installed by ztp. | -| ztp_admin_user | | the user for which the authorized keys will be provisioned. | -| ztp_additional_files | | puts additional files into serve directory. | - ## Provisioning SONiC Switches via ztp.json On SONiC switches it is possible to describe the ZTP procedure in a file called `ztp.json`. It contains all steps that should be performed during ZTP along with some additional options. -We use `ztp.json` to trigger a restart of the BGP service after the initial switch provisioning. +For example, host-specific download paths for the `config_db.json` or any additional files or scripts can be provided in the `ztp.json`. To use the `ztp.json` file, add a DHCP option with code 67 to the DHCP server that serves the file. For example, add a section like the following to `/etc/dhcp/dhcpd.conf`: @@ -34,3 +21,51 @@ host leaf01 { ``` For more information on the `ztp.json` format refer to the [documentation](https://github.com/sonic-net/SONiC/blob/master/doc/ztp/ztp.md). + +Note that each switch that uses the `ztp.json` file needs an individual `config_db.json`, that it can download at `http://{{ ztp_listen_address }}:{{ ztp_port }}/_config_db.json`. +For example, if the switch's hostname is `r01leaf02`, there should be a file called `r01leaf02_config_db.json` located in `{{ ztp_host_dir_path }}/config/`. +The configs can be added to the `ztp_additional_files` variable, e.g. + +```yaml +ztp_additional_files: + - name: r01leaf02_config_db.json + data: "{{ lookup('file', 'path/to/r01leaf02_config_db.json)' | string }}" # using `string` to keep the formatting + - name: r02leaf01_config_db.json + data: ... +``` + +When a SONiC switch is deployed via `ztp.json` and configured by the `sonic` role afterwards, make sure to leave the `sonic_ports`, `sonic_portchannels` and `sonic_breakouts` variables empty and set `sonic_render_config_db_template` to false. +Otherwise the `sonic` role will override the `config_db.json` provided by the `ztp.json`. +The result of this may not be intended and, in the worst case, the switch will reach a broken state from which it only can be restored by a factory reset. +Of course it is also possible to load only a minimal `config_db.json` via ZTP and allow the `sonic` role to render its template based on the `sonic_ports`, `sonic_portchannels` and `sonic_breakouts` variables. +Both approaches have their pros and cons. + +### Pros and Cons of Loading a Static config_db.json via ZTP + +The main advantage of loading the `config_db.json` once via ZTP and disabling template rendering by the `sonic` role is a better stability and the ability to configure the switch exactly as needed without relying on the complex templating logic in the `sonic` role. +As mentioned above, the problem with loading a new config each time the `sonic` role is run is that even seemingly small changes might break the system (swss crash). +On the other hand, with a ZTP-only approach, since ZTP only runs during initial setup of the switch, the only way of changing the config is by resetting the switch to activate ZTP. +So the desicion of whether to use the `sonic` role's dynamic config or a static ZTP-only config comes down to questions like: + +- how often will the config need to change? +- do all ports on the switch look more or less the same or are there ports that require some specific configuration? + +In the latter case the templating might run into certain edge cases, where the resulting config breaks the system. +Then you should consider using only a static config. + +> For the time being it is up to the user which provisioning procedure they prefer. +> In the future we hope to come up with a single solution that is both flexible and reliable. + +## Variables + +| Name | Mandatory | Description | +| ----------------------- | --------- | ----------------------------------------------------------- | +| ztp_nginx_image_name | yes | the docker image to use to serve ztp scripts. | +| ztp_nginx_image_tag | yes | the tag of the docker image to use to serve ztp scripts. | +| ztp_host_dir_path | | the path to serve ztp scripts from. | +| ztp_listen_address | | the address used to serve ztp requests | +| ztp_port | | the port to serve ztp scripts on. | +| ztp_authorized_keys | yes | the authorized keys that should be installed by ztp. | +| ztp_admin_user | | the user for which the authorized keys will be provisioned. | +| ztp_additional_files | | puts additional files into serve directory. | +| ztp_provisioning_script | | shell script to be executed as a last step in the ztp.json | diff --git a/partition/roles/ztp/files/config_db.json b/partition/roles/ztp/files/config_db.json deleted file mode 100644 index 0d7ecddd..00000000 --- a/partition/roles/ztp/files/config_db.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "DEVICE_METADATA": { - "localhost": { - "docker_routing_config_mode": "split" - } - } -} \ No newline at end of file diff --git a/partition/roles/ztp/files/frr.conf b/partition/roles/ztp/files/frr.conf new file mode 100644 index 00000000..684de281 --- /dev/null +++ b/partition/roles/ztp/files/frr.conf @@ -0,0 +1,9 @@ +frr defaults datacenter +password zebra +enable password zebra +! +log syslog informational +log facility local4 +! +agentx +! diff --git a/partition/roles/ztp/files/reload.sh b/partition/roles/ztp/files/reload.sh deleted file mode 100644 index 4712145e..00000000 --- a/partition/roles/ztp/files/reload.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -sudo systemctl restart bgp diff --git a/partition/roles/ztp/tasks/main.yaml b/partition/roles/ztp/tasks/main.yaml index 1f17f311..02c9b2e8 100644 --- a/partition/roles/ztp/tasks/main.yaml +++ b/partition/roles/ztp/tasks/main.yaml @@ -10,38 +10,29 @@ - ztp_nginx_image_name is defined - ztp_nginx_image_tag is defined - ztp_authorized_keys is not none - - "'ztp.sh' not in ztp_additional_files | map(attribute='name')" + - "'user.sh' not in ztp_additional_files | map(attribute='name')" - name: create ztp config directory file: path: "{{ ztp_host_dir_path }}/config" state: directory -- name: render ztp script +- name: render templates template: - src: "ztp.sh.j2" - dest: "{{ ztp_host_dir_path }}/config/ztp.sh" + src: "{{ item }}" + dest: "{{ ztp_host_dir_path }}/config/{{ item | splitext | first }}" mode: 0644 + loop: + - ztp.json.j2 + - user.sh.j2 -- name: copy config_db.json +- name: copy frr.conf copy: - src: "config_db.json" - dest: "{{ ztp_host_dir_path }}/config/config_db.json" + dest: "{{ ztp_host_dir_path }}/config/frr.conf" + src: frr.conf mode: 0644 -- name: copy reload script - copy: - src: "reload.sh" - dest: "{{ ztp_host_dir_path }}/config/reload.sh" - mode: 0644 - -- name: render ztp.json - template: - src: "ztp.json.j2" - dest: "{{ ztp_host_dir_path }}/config/ztp.json" - mode: 0644 - -- name: copy additional contents +- name: copy additional files copy: dest: "{{ ztp_host_dir_path }}/config/{{ item.name }}" content: "{{ item.data }}" @@ -50,7 +41,14 @@ loop_control: label: "{{ item.name }}" -- name: deploy server for serving ztp.sh +- name: copy ztp-provisioning-script + copy: + dest: "{{ ztp_host_dir_path }}/config/ztp-provisioning-script.sh" + content: "{{ ztp_provisioning_script }}" + mode: 0644 + when: ztp_provisioning_script is defined + +- name: deploy server for serving ztp files include_role: name: ansible-common/roles/systemd-docker-service vars: diff --git a/partition/roles/ztp/templates/ztp.sh.j2 b/partition/roles/ztp/templates/user.sh.j2 similarity index 100% rename from partition/roles/ztp/templates/ztp.sh.j2 rename to partition/roles/ztp/templates/user.sh.j2 diff --git a/partition/roles/ztp/templates/ztp.json.j2 b/partition/roles/ztp/templates/ztp.json.j2 index 29da1d14..e40b96b5 100644 --- a/partition/roles/ztp/templates/ztp.json.j2 +++ b/partition/roles/ztp/templates/ztp.json.j2 @@ -2,20 +2,37 @@ "ztp": { "02-user": { "plugin": { - "url": "http://{{ ztp_listen_address }}:{{ ztp_port }}/ztp.sh" + "url": "http://{{ ztp_listen_address }}:{{ ztp_port }}/user.sh" } }, - "03-configdb-json": { - "url": { - "source": "http://{{ ztp_listen_address }}:{{ ztp_port }}/config_db.json" + "03-download": { + "files": [ + { + "url": { + "source": "http://{{ ztp_listen_address }}:{{ ztp_port }}/frr.conf", + "destination": "/etc/sonic/frr/frr.conf" + } + } + ] + }, + "04-configdb-json": { + "dynamic-url": { + "source": { + "prefix": "http://{{ ztp_listen_address }}:{{ ztp_port }}/", + "identifier": "hostname", + "suffix": "_config_db.json" + } }, - "clear-config": false + "clear-config": true }, - "04-reload": { +{% if ztp_provisioning_script %} + "05-provisioning-script": { "plugin": { - "url": "http://{{ ztp_listen_address }}:{{ ztp_port }}/reload.sh" - } + "url": "http://{{ ztp_listen_address }}:{{ ztp_port }}/ztp-provisioning-script.sh" + }, + "shell": true }, - "restart-ztp-no-config": false +{% endif %} + "reboot-on-success": true } }