Create minimega windows instance and install elastic agent (#534)

* Create windows qcow installer * Updates the README and the build_azure_linux_network.py file * Updates the README for the installers * Set up windows minimega instance in pipeline * Hardcode the path to the script in pipeline_install.sh * Passes the azure credentials to the windows qcow installer * Export the environment variables to the remote machine * Fix the check_agent_reporting.sh script * Don't remove the .env file for debug purposed * Pass the secrets to the minimega installer * Fix the adding of the environment variables to the .env file * Don't run the pipeline install script as sudo * Install linux before windows in minimega * Change the way we get the minimega ip address for linux * Adds getting the windows ip address from minimega * Set up to install elastic agent on windows minimega * Prep scripts for installing elastic agent on windows minimega * Prep scripts for installing elastic agent on windows minimega * Install the Elastic Agent in Windows Minimega * Use sshpass to install the Elastic Agent in Windows Minimega * Try escaping the password in the Windows Minimega workflow * Put minimega password in secrets * Install the Elastic Agent in Windows Minimega * Increase the size of the Azure instance * Remove comments causing errors * Rework the Windows Minimega install command * Don't remove the containers so we can run builds again * Comments out the password correction for the Windows Minimega tests * Adds the unique id to the Windows Minimega installer command * Keeps the azure instance running after the tests are done * Add Windows Elastic Agent reporting check * Remove the azure resources when the tests are complete * Adds some more timeouts for the installers * Adds a delay to wait for kibana to be fully ready * Fix the typo in the post install script * Adds in the linux only workflow to run on pull requests * Make the hosts and fingerprint setters more robust
cisagov · Dec 18, 2024 · 2da8cb0 · 2da8cb0
1 parent ead34e8
commit 2da8cb0
Show file tree

Hide file tree

Showing 14 changed files with 325 additions and 85 deletions.
diff --git a/.github/workflows/cluster.yml b/.github/workflows/cluster.yml
@@ -1,9 +1,9 @@
 name: Cluster Run - Minimega
 
 on:
-  #pull_request:
-    # branches:
-    #   - '*'
+  # pull_request:
+  #   branches:
+  #     - '*'
   workflow_dispatch:
     inputs:
       azure_region:
@@ -37,10 +37,13 @@ jobs:
       LS1_IP: ""
       elastic: ""
       AZURE_IP: ""
-      MINIMEGA_IP: ""
+      LINUX_IP: ""
+      WINDOWS_IP: ""
       ENROLLMENT_TOKEN: ""
       ES_PASSWORD: ""
       KIBANA_PASSWORD: ""
+      ELASTIC_AGENT_VERSION: "8.15.3"
+      MINIMEGA_PASSWORD: ${{ secrets.MINIMEGA_PASSWORD }}
 
     steps:
       - name: Checkout repository
@@ -104,7 +107,7 @@ jobs:
               python3 ./azure/build_azure_linux_network.py \
                 -g pipe-${{ env.UNIQUE_ID }} \
                 -s ${{ env.IP_ADDRESS }}/32 \
-                -vs Standard_D8_v4 \
+                -vs Standard_E8_v4 \
                 -l ${{ inputs.azure_region || 'centralus' }} \
                 -ast 23:00 \
                 -y
@@ -119,7 +122,7 @@ jobs:
             IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
             ./minimega/install.sh lme-user \$IP_ADDRESS  "pipe-${{ env.UNIQUE_ID }}.password.txt" 
           "
-          
+
       - name: Install Linux in minimega 
         run: |
           cd testing/v2/development
@@ -128,6 +131,27 @@ jobs:
             IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
             ./ubuntu_qcow_maker/install.sh lme-user \$IP_ADDRESS  "pipe-${{ env.UNIQUE_ID }}.password.txt" 
           "
+
+      - name: Install windows minimega on Azure instance
+        env:
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_SECRET }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+        run: |
+          cd testing/v2/development
+          sleep 30
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T \
+            -e AZURE_CLIENT_ID \
+            -e AZURE_CLIENT_SECRET \
+            -e AZURE_TENANT_ID \
+            -e AZURE_SUBSCRIPTION_ID \
+            pipeline bash -c "
+            cd /home/lme-user/LME/testing/v2/installers && \
+            IP_ADDRESS=\$(cat pipe-${{ env.UNIQUE_ID }}.ip.txt) && \
+            ./windows_qcow/pipeline_install.sh lme-user \$IP_ADDRESS  "pipe-${{ env.UNIQUE_ID }}.password.txt" 
+          "
+          
           
       - name: Check if linux is running in minimega 
         run: |
@@ -139,33 +163,50 @@ jobs:
             ssh lme-user@\$IP_ADDRESS 'sudo /opt/minimega/bin/minimega -e vm info'
           "
 
-      - name: Get Azure and Minimega IP addresses
+      - name: Get Azure, Windows, and Linux IP addresses
         run: |
           cd testing/v2/development
           AZURE_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "cat /home/lme-user/LME/testing/v2/installers/pipe-${{ env.UNIQUE_ID }}.ip.txt")
           echo "AZURE_IP=$AZURE_IP" >> $GITHUB_ENV
           echo "Azure IP:$AZURE_IP"
-          MINIMEGA_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-              ssh lme-user@$AZURE_IP 'sudo /opt/minimega/bin/minimega -e .json true .filter name=\"linux-runner\" vm info | jq -r \".[].Data[].Networks[].IP4\"'
-          " )
+          LINUX_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
+            ssh lme-user@$AZURE_IP 'sudo /opt/minimega/bin/minimega -e .json true vm info | jq -r --arg name \"ubuntu-runner\" \".[].Data[] | select(.Name == \\\$name) | .Networks[0].IP4\"'
+          ")
           EXIT_CODE=$?
           if [ $EXIT_CODE -ne 0 ]; then
-            echo "Failed to get Minimega IP. Exit code: $EXIT_CODE" >&2
+            echo "Failed to get Linux IP. Exit code: $EXIT_CODE" >&2
             exit 1
           fi
-          if [ -z "$MINIMEGA_IP" ]; then
-            echo "Minimega IP is empty" >&2
+          if [ -z "$LINUX_IP" ]; then
+            echo "Linux IP is empty" >&2
             exit 1
           fi
-          echo "MINIMEGA_IP=$MINIMEGA_IP" >> $GITHUB_ENV    
-          echo "Azure IP:$AZURE_IP Minimega IP:$MINIMEGA_IP"
+          echo "LINUX_IP=$LINUX_IP" >> $GITHUB_ENV    
+          echo "Linux IP:$LINUX_IP"
 
+          WINDOWS_IP=$(docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
+            ssh lme-user@$AZURE_IP 'sudo /opt/minimega/bin/minimega -e .json true vm info | jq -r --arg name \"windows-runner\" \".[].Data[] | select(.Name == \\\$name) | .Networks[0].IP4\"'
+          ")
+          EXIT_CODE=$?
+          if [ $EXIT_CODE -ne 0 ]; then
+            echo "Failed to get Windows IP. Exit code: $EXIT_CODE" >&2
+            exit 1
+          fi
+          if [ -z "$WINDOWS_IP" ]; then
+            echo "Windows IP is empty" >&2
+            exit 1
+          fi
+          echo "WINDOWS_IP=$WINDOWS_IP" >> $GITHUB_ENV    
+          echo "Windows IP:$WINDOWS_IP"
 
-      - name: Run a command in Minimega
+          echo "Azure_IP:$AZURE_IP LINUX_IP:$LINUX_IP WINDOWS_IP:$WINDOWS_IP"
+
+
+      - name: Run a command in Linux Minimega
         run: |
           cd testing/v2/development
           docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-              ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} ls -la'
+              ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.LINUX_IP }} ls -la'
           "
 
       - name: Install LME on Azure instance
@@ -239,36 +280,44 @@ jobs:
             '
           ")
           echo "Retrieved enrollment token: $ENROLLMENT_TOKEN"
-          
+
           # Mask the enrollment token in logs and set it as an environment variable
           echo "::add-mask::$ENROLLMENT_TOKEN"
           echo "ENROLLMENT_TOKEN=$ENROLLMENT_TOKEN" >> $GITHUB_ENV
           echo "Policy ID and Enrollment Token retrieved successfully"
 
-      - name: Copy the Elastic Agent installer to Minimega
+      - name: Copy the Elastic Agent installer to Linux Minimega
         run: |
           cd testing/v2/development
           docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
               ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
-              'sudo scp -p -o StrictHostKeyChecking=no /home/lme-user/LME/testing/v2/installers/lib/install_agent_linux.sh vmuser@${{ env.MINIMEGA_IP }}:~'
+              'sudo scp -p -o StrictHostKeyChecking=no /home/lme-user/LME/testing/v2/installers/lib/install_agent_linux.sh vmuser@${{ env.LINUX_IP }}:~'
           "
 
-      - name: Run a command in Minimega
+      - name: Run a command in Linux Minimega
         run: |
           cd testing/v2/development
           docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-              ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} ls -la'
+              ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.LINUX_IP }} ls -la'
           "
 
-      - name: Install the Elastic Agent in Minimega
+      - name: Install the Elastic Agent in Linux Minimega
         run: |
           cd testing/v2/development
           docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
-              ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} chmod +x ./install_agent_linux.sh ' && \
-              ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.MINIMEGA_IP }} ./install_agent_linux.sh --token ${{ env.ENROLLMENT_TOKEN }}'
+              ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.LINUX_IP }} chmod +x ./install_agent_linux.sh ' && \
+              ssh lme-user@${{ env.AZURE_IP }} 'sudo ssh -o StrictHostKeyChecking=no vmuser@${{ env.LINUX_IP }} ./install_agent_linux.sh --token ${{ env.ENROLLMENT_TOKEN }} --version ${{ env.ELASTIC_AGENT_VERSION }}'
           "
 
-      - name: Check if the Elastic agent is reporting 
+      - name: Install the Elastic Agent in Windows Minimega
+        run: |
+          set +H
+          cd testing/v2/development
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c '
+            ssh lme-user@${{ env.AZURE_IP }} "export SSHPASS='\''${{ env.MINIMEGA_PASSWORD }}'\'' && export WINDOWS_HOST=${{ env.WINDOWS_IP }} && cd /home/lme-user/LME/testing/v2/installers/lib/ && sudo -E ./install_agent_windows.sh  --token ${{ env.ENROLLMENT_TOKEN }} --version ${{ env.ELASTIC_AGENT_VERSION }} --clientip ${{ env.WINDOWS_IP }}"
+          '
+
+      - name: Check if the Linux Elastic agent is reporting 
         env:
           ES_PASSWORD: ${{ env.ES_PASSWORD }}
         run: |
@@ -279,6 +328,17 @@ jobs:
               'export ES_PASSWORD=\"$ES_PASSWORD\" && /home/lme-user/LME/testing/v2/installers/lib/check_agent_reporting.sh'
           "
 
+      - name: Check if the Windows Elastic agent is reporting 
+        env:
+          ES_PASSWORD: ${{ env.ES_PASSWORD }}
+        run: |
+          sleep 360
+          cd testing/v2/development
+          docker compose -p ${{ env.UNIQUE_ID }} exec -T pipeline bash -c "
+              ssh -o StrictHostKeyChecking=no lme-user@${{ env.AZURE_IP }} \
+              'export ES_PASSWORD=\"$ES_PASSWORD\" && /home/lme-user/LME/testing/v2/installers/lib/check_agent_reporting.sh windows'
+          "
+
       - name: Run api tests on Azure instance
         env:
           ES_PASSWORD: ${{ env.ES_PASSWORD }}

diff --git a/ansible/post_install_local.yml b/ansible/post_install_local.yml
@@ -212,6 +212,9 @@
         body:
           hosts: ["https://{{ ipvar }}:9200"]
       register: fleet_output_hosts_result
+      until: fleet_output_hosts_result.status == 200
+      retries: 12
+      delay: 30
       no_log: "{{ not debug_mode }}"
       ignore_errors: yes
 
@@ -233,6 +236,9 @@
         body:
           ca_trusted_fingerprint: "{{ ca_fingerprint.stdout }}"
       register: fleet_output_fingerprint_result
+      until: fleet_output_fingerprint_result.status == 200
+      retries: 12
+      delay: 30
       no_log: "{{ not debug_mode }}"
 
     - name: Set Fleet default output SSL verification mode
@@ -250,7 +256,12 @@
       register: fleet_output_ssl_result
       no_log: "{{ not debug_mode }}"
 
+    - name: Wait for Kibana to be fully ready
+      pause:
+        seconds: 180
+
     - name: Create Endpoint Policy
+      # TODO: This might need to check if the system is running before creating the policy
       uri:
         url: "{{ local_kbn_url }}/api/fleet/agent_policies?sys_monitoring=true"
         method: POST
@@ -333,6 +344,10 @@
         state: touch
       when: not installed_file_check.stat.exists
 
+    - name: Wait for Fleet to be fully initialized
+      pause:
+        seconds: 120
+
 - name: Install LME Dashboards 
   hosts: localhost
   become: yes
@@ -723,6 +738,7 @@
         clone_directory: "{{clone_directory | expanduser }}"
 
     - name: fix wazuh password 
+      # TODO: This might need to check if the container is running. It has failed before.
       ansible.builtin.expect:
         #source + podman exec
         command: "{{ clone_directory }}/scripts/wazuh_rbac.sh"
@@ -731,7 +747,7 @@
             - "{{ wazuh_password }}"
           ".*'wazuh-wui'.*":
             - "{{ wazuh_api_password }}"
-        timeout: 30
+        timeout: 60
       become: yes
 
 - name: Create Read Only User

diff --git a/testing/v2/installers/lib/check_agent_reporting.sh b/testing/v2/installers/lib/check_agent_reporting.sh
@@ -11,22 +11,29 @@ if [ -z "$ES_PASSWORD" ]; then
     handle_error "ES_PASSWORD environment variable is not set"
 fi
 
+# Set default type and allow override via argument
+TYPE="linux"
+if [ "$1" = "windows" ]; then
+    TYPE="windows"
+fi
+
 # Initialize retry variables
 MAX_ATTEMPTS=100
 ATTEMPT=1
 WAIT_TIME=15
 
 while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
-    echo "Attempt $ATTEMPT of $MAX_ATTEMPTS to check agent reporting..."
+    echo "Attempt $ATTEMPT of $MAX_ATTEMPTS to check agent reporting for $TYPE type..."
 
     if [ $ATTEMPT -gt 1 ]; then
         echo "Waiting before next attempt..."
         sleep $WAIT_TIME
-        ATTEMPT=$((ATTEMPT + 1))
     fi
 
+    ATTEMPT=$((ATTEMPT + 1))
+
     # Run the curl command and capture the output
-    output=$(curl -kL -s -X GET "https://localhost:9200/.ds-metrics-system.cpu-default-*/_search" \
+    output=$(curl -kL -s -X GET "https://localhost:9200/metrics-endpoint.metadata-*/_search" \
          -H 'Content-Type: application/json' \
          -H "kbn-xsrf: true" \
          -u "elastic:$ES_PASSWORD" \
@@ -36,17 +43,7 @@ while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
           "must": [
             {
               "term": {
-                "host.name": "ubuntu-vm"
-              }
-            },
-            {
-              "term": {
-                "event.module": "system"
-              }
-            },
-            {
-              "term": {
-                "event.dataset": "system.cpu"
+                "host.os.type": "'$TYPE'"
               }
             }
           ]
@@ -77,17 +74,16 @@ while [ $ATTEMPT -le $MAX_ATTEMPTS ]; do
         continue
     fi
 
-    echo "Hit count: $output"
     echo "Hit count: $hit_count"
 
     # Check the hit count and exit if successful
     if [ "$hit_count" -gt 0 ]; then
-        echo "ubuntu-vm is reporting"
+        echo "$TYPE type agent is reporting"
         exit 0
     fi
 
-    echo "No recent data from ubuntu-vm, retrying..."
+    echo "No recent data from $TYPE type, retrying..."
 done
 
-echo "No recent data from ubuntu-vm after $MAX_ATTEMPTS attempts"
+echo "No recent data from $TYPE type after $MAX_ATTEMPTS attempts"
 exit 1
diff --git a/testing/v2/installers/lib/get_ip_of_machine.sh b/testing/v2/installers/lib/get_ip_of_machine.sh
@@ -5,7 +5,7 @@ MAX_ATTEMPTS=30
 SLEEP_INTERVAL=10
 
 get_ip() {
-    /opt/minimega/bin/minimega -e .json true .filter name="$VM_NAME" vm info | jq -r '.[].Data[].Networks[].IP4'
+    /opt/minimega/bin/minimega -e ".json true vm info" | jq -r --arg name "$VM_NAME" '.[].Data[] | select(.Name == $name) .Networks[0].IP4'
 }
 
 echo "Waiting for IP assignment for VM: $VM_NAME" >&2