ci: Add tft plan and workflow

This change is for running tests in Testing Farm CI. This is a replacement for BaseOS CI that we are currently using. Running it Testing Farm gives us more control. It adds a workflow for running tests, and a plans directory containing a test plan and a README-plans.md with some info. Note that this workflow runs from the main branch. This means that changes to the workflow must be merged to main, then pull requests will be able to run it. This is because the workflow uses on: issue_comment context, this is a security measure recommended by GitHub. It saves us from leaking organization secrets. The functionality is WIP, so await future fixes and updates. Signed-off-by: Sergei Petrosian <[email protected]>
linux-system-roles · Jul 25, 2024 · c1d9947 · c1d9947
1 parent f038fca
commit c1d9947
Show file tree

Hide file tree

Showing 5 changed files with 277 additions and 1 deletion.
diff --git a/.fmf/version b/.fmf/version
@@ -0,0 +1 @@
+1
diff --git a/.github/workflows/tft.yml b/.github/workflows/tft.yml
@@ -0,0 +1,178 @@
+name: Run integration tests in Testing Farm
+on:
+  issue_comment:
+    types:
+      - created
+permissions:
+  contents: read
+  # This is required for the ability to create/update the Pull request status
+  statuses: write
+# The concurrency key is used to prevent multiple workflows from running at the same time
+concurrency:
+  # group name contains reponame-pr_num to allow simualteneous runs in different PRs
+  group: testing-farm-${{ github.event.repository.name }}-${{ github.event.issue.number }}
+  cancel-in-progress: true
+jobs:
+  prepare_vars:
+    name: Get info from role and PR to determine if and how to test
+    # Let's schedule tests only on user request. NOT automatically.
+    # Only repository owner or member can schedule tests
+    if: |
+      github.event.issue.pull_request
+      && (contains(github.event.comment.body, '[citest]') || contains(github.event.comment.body, '[citest-all]'))
+      && (contains(fromJson('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR"]'), github.event.comment.author_association)
+      || contains('systemroller', github.event.comment.user.login))
+    runs-on: ubuntu-latest
+    outputs:
+      supported_platforms: ${{ steps.supported_platforms.outputs.supported_platforms }}
+      head_sha: ${{ steps.head_sha.outputs.head_sha }}
+      datetime: ${{ steps.datetime.outputs.datetime }}
+      memory: ${{ steps.memory.outputs.memory }}
+    steps:
+
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Get head sha of the PR
+        id: head_sha
+        run: |
+          head_sha=$(gh api "repos/$REPO/pulls/$PR_NO" --jq '.head.sha')
+          echo "head_sha=$head_sha" >> $GITHUB_OUTPUT
+        env:
+          REPO: ${{ github.repository }}
+          PR_NO: ${{ github.event.issue.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get cuurent datetime
+        id: datetime
+        run: |
+          printf -v datetime '%(%Y%m%d-%H%M%S)T' -1
+          echo "datetime=$datetime" >> $GITHUB_OUTPUT
+
+      - name: Get memory
+        id: memory
+        run: |
+          if [ -d tests/provision.fmf ]; then
+            memory=$(grep -rPo '    m: \K(.*)' tests/provision.fmf)
+          fi
+          if [ -n "$memory" ]; then
+            echo "memory=$memory" >> $GITHUB_OUTPUT
+          else
+            echo "memory=2048" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Get supported platforms
+        id: supported_platforms
+        run: |
+          supported_platforms=""
+          meta_main=meta/main.yml
+          # All Fedora are supported, add latest Fedora versions to supported_platforms
+          if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi fedora$; then
+            supported_platforms+=" Fedora-39"
+            supported_platforms+=" Fedora-40"
+          fi
+          # Specific Fedora versions supported
+          if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qiP 'fedora\d+$'; then
+            for fedora_ver in $(yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -iPo 'fedora\K(\d+$)'); do
+              supported_platforms+=" Fedora-$fedora_ver"
+            done
+          fi
+          if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi el7; then
+            supported_platforms+=" CentOS-7-latest"
+          fi
+          for ver in 8 9 10; do
+            if yq '.galaxy_info.galaxy_tags[]' "$meta_main" | grep -qi el"$ver"; then
+              supported_platforms+=" CentOS-Stream-$ver"
+            fi
+          done
+          echo "supported_platforms=$supported_platforms" >> $GITHUB_OUTPUT
+
+  testing-farm:
+    name: ${{ matrix.platform }}/ansible-${{ matrix.ansible_version }}
+    needs: prepare_vars
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: Fedora-39
+            ansible_version: 2.17
+          - platform: Fedora-40
+            ansible_version: 2.17
+          - platform: CentOS-7-latest
+            ansible_version: 2.9
+          - platform: CentOS-Stream-8
+            ansible_version: 2.9
+          # On CentOS-Stream-8, latest supported Ansible is 2.16
+          - platform: CentOS-Stream-8
+            ansible_version: 2.16
+          - platform: CentOS-Stream-9
+            ansible_version: 2.17
+          - platform: CentOS-Stream-10
+            ansible_version: 2.17
+    runs-on: ubuntu-latest
+    env:
+      ARTIFACTS_DIR_NAME: "tf_${{ github.event.repository.name }}-${{ github.event.issue.number }}_\
+        ${{ matrix.platform }}-${{ matrix.ansible_version }}_\
+        ${{ needs.prepare_vars.outputs.datetime }}/artifacts"
+      ARTIFACT_TARGET_DIR: /srv/pub/alt/linuxsystemroles/logs
+    steps:
+      - name: Set commit status as pending
+        if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
+        uses: myrotvorets/set-commit-status-action@master
+        with:
+          sha: ${{ needs.prepare_vars.outputs.head_sha }}
+          status: pending
+          context: ${{ matrix.platform }}|ansible-${{ matrix.ansible_version }}
+          description: Test started
+          targetUrl: ""
+
+      - name: Set commit status as success with a description that platform is skipped
+        if: "!contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)"
+        uses: myrotvorets/set-commit-status-action@master
+        with:
+          sha: ${{ needs.prepare_vars.outputs.head_sha }}
+          status: success
+          context: ${{ matrix.platform }}|ansible-${{ matrix.ansible_version }}
+          description: The role does not support this platform. Skipping.
+          targetUrl: ""
+
+      - name: Run test in testing farm
+        uses: sclorg/testing-farm-as-github-action@v2
+        if: contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
+        env:
+          ARTIFACTS_DIR: ${{ env.ARTIFACT_TARGET_DIR }}/${{ env.ARTIFACTS_DIR_NAME }}
+          ARTIFACTS_URL: https://dl.fedoraproject.org/pub/alt/linuxsystemroles/logs/${{ env.ARTIFACTS_DIR_NAME }}
+        with:
+          git_url: ${{ github.server_url }}/${{ github.repository }}
+          git_ref: ${{ needs.prepare_vars.outputs.head_sha }}
+          pipeline_settings: '{ "type": "tmt-multihost" }'
+          variables: "ANSIBLE_VER=${{ matrix.ansible_version }};\
+            REPO_NAME=${{ github.event.repository.name }};\
+            GITHUB_ORG=linux-system-roles;\
+            PR_NUM=${{ github.event.issue.number }};\
+            ARTIFACTS_DIR=${{ env.ARTIFACTS_DIR }};\
+            ARTIFACTS_URL=${{ env.ARTIFACTS_URL }}"
+          # Note that LINUXSYSTEMROLES_SSH_KEY must be single-line, TF doesn't read multi-line variables fine.
+          secrets: "LINUXSYSTEMROLES_USER=${{ secrets.LINUXSYSTEMROLES_USER }};\
+            LINUXSYSTEMROLES_DOMAIN=${{ secrets.LINUXSYSTEMROLES_DOMAIN }};\
+            LINUXSYSTEMROLES_SSH_KEY=${{ secrets.LINUXSYSTEMROLES_SSH_KEY }}"
+          compose: ${{ matrix.platform }}
+          # There are two blockers for using public ranch:
+          # 1. multihost is not supported in public https://github.com/teemtee/tmt/issues/2620
+          # 2. Security issue that leaks long secrets - Jira TFT-2698
+          tf_scope: private
+          api_key: ${{ secrets.TF_API_KEY_RH }}
+          update_pull_request_status: false
+          tmt_hardware: '{ "memory": ">= ${{ needs.prepare_vars.outputs.memory }} MB" }'
+
+      - name: Set final commit status
+        uses: myrotvorets/set-commit-status-action@master
+        if: always() && contains(needs.prepare_vars.outputs.supported_platforms, matrix.platform)
+        env:
+          ARTIFACTS_URL: https://dl.fedoraproject.org/pub/alt/linuxsystemroles/logs/${{ env.ARTIFACTS_DIR_NAME }}
+        with:
+          sha: ${{ needs.prepare_vars.outputs.head_sha }}
+          status: ${{ job.status }}
+          context: ${{ matrix.platform }}|ansible-${{ matrix.ansible_version }}
+          description: Test finished
+          targetUrl: ${{ env.ARTIFACTS_URL }}
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # timesync
 
-[![ansible-lint.yml](https://github.com/linux-system-roles/timesync/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/timesync/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/timesync/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/timesync/actions/workflows/ansible-test.yml) [![markdownlint.yml](https://github.com/linux-system-roles/timesync/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/timesync/actions/workflows/markdownlint.yml) [![shellcheck.yml](https://github.com/linux-system-roles/timesync/actions/workflows/shellcheck.yml/badge.svg)](https://github.com/linux-system-roles/timesync/actions/workflows/shellcheck.yml) [![woke.yml](https://github.com/linux-system-roles/timesync/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/timesync/actions/workflows/woke.yml)
+[![ansible-lint.yml](https://github.com/linux-system-roles/timesync/actions/workflows/ansible-lint.yml/badge.svg)](https://github.com/linux-system-roles/timesync/actions/workflows/ansible-lint.yml) [![ansible-test.yml](https://github.com/linux-system-roles/timesync/actions/workflows/ansible-test.yml/badge.svg)](https://github.com/linux-system-roles/timesync/actions/workflows/ansible-test.yml) [![markdownlint.yml](https://github.com/linux-system-roles/timesync/actions/workflows/markdownlint.yml/badge.svg)](https://github.com/linux-system-roles/timesync/actions/workflows/markdownlint.yml) [![shellcheck.yml](https://github.com/linux-system-roles/timesync/actions/workflows/shellcheck.yml/badge.svg)](https://github.com/linux-system-roles/timesync/actions/workflows/shellcheck.yml) [![tft.yml](https://github.com/linux-system-roles/timesync/actions/workflows/tft.yml/badge.svg)](https://github.com/linux-system-roles/timesync/actions/workflows/tft.yml) [![woke.yml](https://github.com/linux-system-roles/timesync/actions/workflows/woke.yml/badge.svg)](https://github.com/linux-system-roles/timesync/actions/workflows/woke.yml)
 
 This role installs and configures an NTP and/or PTP implementation to operate
 <!--- wokeignore:rule=slave -->

diff --git a/plans/README-plans.md b/plans/README-plans.md
@@ -0,0 +1,35 @@
+# Introduction CI Testing Plans
+
+Linux System Roles CI runs [tmt](https://tmt.readthedocs.io/en/stable/index.html) test plans in [Testing farm](https://docs.testing-farm.io/Testing%20Farm/0.1/index.html) with the [tmt.yml](https://github.com/linux-system-roles/timesync/blob/main/.github/workflows/tmt.yml) GitHub workflow.
+
+The plans/general.fmf plan is a test plan that is general for all roles. It does the following steps:
+
+1. Provisions two machines, one used as an Ansible control node, and second used as a managed node.
+2. Does the required preparation on machines.
+3. For the given role and the given PR, runs the general test from [test.sh](https://github.com/linux-system-roles/tft-tests/blob/main/tests/general/test.sh).
+
+The [tmt.yml](https://github.com/linux-system-roles/timesync/blob/main/.github/workflows/tmt.yml) workflow runs the above plan and uploads the results to our Fedora storage for public access.
+This workflow uses Testing Farm's Github Action [Schedule tests on Testing Farm](https://github.com/marketplace/actions/schedule-tests-on-testing-farm).
+
+## Running Tests
+
+You can run tests locally with the `tmt try` cli.
+
+### Prerequisites
+
+* Install `tmt` as described in [Installation](https://tmt.readthedocs.io/en/stable/stories/install.html).
+
+### Running Tests Locally
+
+For now, this functionality requires you to push changes to a PR because the plan only runs from the main branch, or from the PR branch.
+So this is WIP.
+
+To run tests locally, in the role repository, enter `tmt run plans --name plans/general <platform>`.
+Where `<platform>` is the name of the platform you want to run tests against.
+
+For example, `tmt run plans --name plans/general Fedora-40`.
+
+This command identifies the plans/general plan and provisions two machines, one used as an Ansible control node, and second used as a managed node.
+
+You can also use `tmt try` to get to an interreactive prompt and be able to ssh into test machines.
+You must run `tmt try -p plans/general Fedora-40`, and the in the promt type `p` to prepare the machines, then `t` to run the tests.
diff --git a/plans/general.fmf b/plans/general.fmf
@@ -0,0 +1,62 @@
+summary: A general test for a system role
+provision:
+  - name: control_node
+    role: control_node
+    # TF uses `how: artemis`, tmt try uses `how: virtual`. No need to define `how`
+    # `connection: system` is for `how: virtual` to make VMs get a real IP to configure ssh easily
+    # This setting is ignored on artemis so we can keep it
+    connection: system
+  - name: managed_node1
+    role: managed_node
+    connection: system
+environment:
+    ANSIBLE_VER: 2.17
+    REPO_NAME: timesync
+    PYTHON_VERSION: 3.12
+    SYSTEM_ROLES_ONLY_TESTS: ""
+    PR_NUM: ""
+prepare:
+  - name: Use vault.centos.org repos (CS 7, 8 EOL workaround)
+    script: |
+      if grep -q -e 'CentOS Stream release 8' -e 'CentOS Linux release 7.9' /etc/redhat-release; then
+        sed -i '/^mirror/d;s/#\(baseurl=http:\/\/\)mirror/\1vault/' /etc/yum.repos.d/*.repo
+      fi
+
+  - name: Enable epel to install beakerlib on all platforms except CS10 and Fedora, there it's not available and not needed
+    script: |
+      if ! grep -q -e 'CentOS Stream release 10' -e 'Fedora release' /etc/redhat-release; then
+        yum install epel-release -y
+      fi
+    where: control_node
+
+  - name: Additional steps to enable EPEL on EL 7
+    script: |
+      if grep -q 'CentOS Linux release 7.9' /etc/redhat-release; then
+        yum install yum-utils -y
+        yum-config-manager --enable epel epel-debuginfo epel-source
+      fi
+    where: control_node
+
+  - name: Install python on managed node when running CS8 with ansible!=2.9
+    script: |
+      if [ "$ANSIBLE_VER" != "2.9" ] && grep -q 'CentOS Stream release 8' /etc/redhat-release; then
+        dnf install -y python"$PYTHON_VERSION"
+      fi
+    where: managed_node
+
+  - name: Distribute SSH keys when provisioned with how=virtual
+    script: |
+      if [ -f ${TMT_TREE%/*}/provision/control_node/id_ecdsa.pub ]; then
+        cat ${TMT_TREE%/*}/provision/control_node/id_ecdsa.pub >> ~/.ssh/authorized_keys
+      fi
+    where: managed_node
+
+discover:
+  - name: Run test playbooks from control_node
+    how: fmf
+    url: https://github.com/linux-system-roles/tft-tests
+    ref: main
+    where: control_node
+    filter: tag:test_playbooks
+execute:
+    how: tmt