datadog-agent cleanups and do not install .yaml.default files (#34268)

wolfi-dev · Nov 16, 2024 · 9e12dd2 · 9e12dd2
1 parent c3df607
commit 9e12dd2
Showing 1 changed file with 75 additions and 52 deletions.
diff --git a/datadog-agent.yaml b/datadog-agent.yaml
@@ -3,7 +3,7 @@ package:
   # This package has two git checkouts. For each new release, the commit SHA for
   # DataDog/integrations-core must also be updated.
   version: 7.59.0
-  epoch: 0
+  epoch: 1
   description: "Collect events and metrics from your hosts that send data to Datadog."
   copyright:
     - license: Apache-2.0
@@ -16,6 +16,10 @@ package:
       - libseccomp
       - shadow
 
+vars:
+  py-version: "3.11"
+  destd: /opt/datadog-agent
+
 var-transforms:
   - from: ${{package.version}}
     match: ^(\d+\.\d+)\.\d+$
@@ -62,9 +66,9 @@ environment:
       - ninja
       - openssf-compiler-options
       - procps-dev
-      - py3.11-pip
-      - py3.11-semver
-      - python3-dev~3.11 # strictly requires python3.11
+      - py${{vars.py-version}}-pip
+      - py${{vars.py-version}}-semver
+      - python-${{vars.py-version}}-dev # strictly requires python3.11
       - systemd-dev
       - util-linux-misc # unshare
       - wget # Required for downloading clang-12 and kernel headers from debian
@@ -94,10 +98,10 @@ pipeline:
   # these won't leak into the package.
   - runs: |
       # install `invoke` and its dependencies
-      python3.11 -m pip install invoke requests toml pyyaml packaging
+      python${{vars.py-version}} -m pip install invoke requests toml pyyaml packaging
 
       # install `gitlab`
-      python3.11 -m pip install python-gitlab
+      python${{vars.py-version}} -m pip install python-gitlab
 
   - uses: go/bump
     with:
@@ -183,15 +187,15 @@ pipeline:
 
   - runs: |
       mkdir -p \
-        ${{targets.contextdir}}/opt/datadog-agent/bin/agent \
-        ${{targets.contextdir}}/opt/datadog-agent/embedded/bin
+        ${{targets.contextdir}}${{vars.destd}}/bin/agent \
+        ${{targets.contextdir}}${{vars.destd}}/embedded/bin
 
-      install -Dm755 bin/agent/agent ${{targets.contextdir}}/opt/datadog-agent/bin/agent
+      install -Dm755 bin/agent/agent ${{targets.contextdir}}${{vars.destd}}/bin/agent
       # *-agent is just a symlink to the "agent" multicall
-      ln -s /opt/datadog-agent/bin/agent/agent ${{targets.contextdir}}/opt/datadog-agent/embedded/bin/process-agent
-      ln -s /opt/datadog-agent/bin/agent/agent ${{targets.contextdir}}/opt/datadog-agent/embedded/bin/security-agent
-      ln -s /opt/datadog-agent/bin/agent/agent ${{targets.contextdir}}/opt/datadog-agent/embedded/bin/trace-agent
-      ln -s /opt/datadog-agent/bin/agent/agent ${{targets.contextdir}}/opt/datadog-agent/embedded/bin/system-probe
+      ln -s ${{vars.destd}}/bin/agent/agent ${{targets.contextdir}}${{vars.destd}}/embedded/bin/process-agent
+      ln -s ${{vars.destd}}/bin/agent/agent ${{targets.contextdir}}${{vars.destd}}/embedded/bin/security-agent
+      ln -s ${{vars.destd}}/bin/agent/agent ${{targets.contextdir}}${{vars.destd}}/embedded/bin/trace-agent
+      ln -s ${{vars.destd}}/bin/agent/agent ${{targets.contextdir}}${{vars.destd}}/embedded/bin/system-probe
 
       mkdir -p ${{targets.contextdir}}/etc/datadog-agent/
 
@@ -211,7 +215,14 @@ pipeline:
       install -Dm755 Dockerfiles/agent/secrets-helper/readsecret.sh ${{targets.contextdir}}/readsecret.sh
       install -Dm755 Dockerfiles/agent/secrets-helper/readsecret_multiple_providers.sh ${{targets.contextdir}}/readsecret_multiple_providers.sh
 
-      cp -r bin/agent/dist/conf.d ${{targets.contextdir}}/etc/datadog-agent/conf.d
+      confd=${{targets.contextdir}}/etc/datadog-agent/conf.d
+      cp -r bin/agent/dist/conf.d "$confd"
+      # https://github.com/NixOS/nixpkgs/pull/189795
+      # the agent apparently loads .yaml.default files, which then cause noise in logs.
+      rm -rf \
+          "$confd/apm.yaml.default" \
+          "$confd/process_agent.yaml.default" \
+          "$confd/winproc.d"
 
       # Setup s3-overlay overrides
       mkdir -p ${{targets.contextdir}}/etc/s6/init
@@ -231,7 +242,7 @@ subpackages:
     pipeline:
       - runs: |
           mkdir -p ${{targets.contextdir}}/usr/bin/dist/jmx
-          ln -sf /opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar ${{targets.contextdir}}/usr/bin/dist/jmx/jmxfetch.jar
+          ln -sf ${{vars.destd}}/bin/agent/dist/jmx/jmxfetch.jar ${{targets.contextdir}}/usr/bin/dist/jmx/jmxfetch.jar
 
   - name: datadog-agent-oci-compat
     dependencies:
@@ -245,15 +256,15 @@ subpackages:
           mkdir -p \
             ${{targets.contextdir}}/conf.d \
             ${{targets.contextdir}}/checks.d \
-            ${{targets.contextdir}}/opt/datadog-agent/embedded \
-            ${{targets.contextdir}}/opt/datadog-agent/bin/agent/dist
+            ${{targets.contextdir}}${{vars.destd}}/embedded \
+            ${{targets.contextdir}}${{vars.destd}}/bin/agent/dist
 
-          cp -r /opt/datadog-agent/embedded/share ${{targets.contextdir}}/opt/datadog-agent/embedded/share
+          cp -r ${{vars.destd}}/embedded/share ${{targets.contextdir}}${{vars.destd}}/embedded/share
 
-          cp -r bin/agent/dist/checks ${{targets.contextdir}}/opt/datadog-agent/bin/agent/dist/
-          install -Dm644 bin/agent/dist/config.py ${{targets.contextdir}}/opt/datadog-agent/bin/agent/dist/config.py
-          cp -r bin/agent/dist/utils ${{targets.contextdir}}/opt/datadog-agent/bin/agent/dist/
-          cp -r bin/agent/dist/views ${{targets.contextdir}}/opt/datadog-agent/bin/agent/dist/
+          cp -r bin/agent/dist/checks ${{targets.contextdir}}${{vars.destd}}/bin/agent/dist/
+          install -Dm644 bin/agent/dist/config.py ${{targets.contextdir}}${{vars.destd}}/bin/agent/dist/config.py
+          cp -r bin/agent/dist/utils ${{targets.contextdir}}${{vars.destd}}/bin/agent/dist/
+          cp -r bin/agent/dist/views ${{targets.contextdir}}${{vars.destd}}/bin/agent/dist/
 
           cp -r Dockerfiles/agent/entrypoint.d ${{targets.contextdir}}/opt/entrypoints
 
@@ -262,7 +273,7 @@ subpackages:
       runtime:
         - bash
         - coreutils
-        - python-3.11
+        - python-${{vars.py-version}}-base
     pipeline:
       - working-directory: /home/integrations
         pipeline:
@@ -279,7 +290,7 @@ subpackages:
               SOURCE_DATE_EPOCH=315532800
 
               # Create virtual environment
-              python3.11 -m venv .venv
+              python${{vars.py-version}} -m venv .venv
 
               # Install locked dependencies
               .venv/bin/pip install --require-hashes --only-binary=:all: --no-deps -r .deps/resolved/linux-${{build.arch}}_py3.txt
@@ -295,19 +306,19 @@ subpackages:
               find .venv -name "*.pyc" -delete
               find .venv -name "__pycache__" -exec rm -rf {} +
 
-              mkdir -p ${{targets.contextdir}}/opt/datadog-agent
-              .venv/bin/pip freeze > ${{targets.contextdir}}/opt/datadog-agent/final_constraints-py3.txt
+              mkdir -p ${{targets.contextdir}}${{vars.destd}}
+              .venv/bin/pip freeze > ${{targets.contextdir}}${{vars.destd}}/final_constraints-py3.txt
 
               # Include the agent's requirements for the core integrations.
-              cp requirements-agent-release.txt ${{targets.contextdir}}/opt/datadog-agent/
+              cp requirements-agent-release.txt ${{targets.contextdir}}${{vars.destd}}/
 
               # Use Python in virtual environment
-              sed -i "s|$(pwd)/.venv|/opt/datadog-agent/embedded|g" .venv/pyvenv.cfg
-              sed -i "s|$(pwd)/.venv|/opt/datadog-agent/embedded|g" .venv/bin/*
+              sed -i "s|$(pwd)/.venv|${{vars.destd}}/embedded|g" .venv/pyvenv.cfg
+              sed -i "s|$(pwd)/.venv|${{vars.destd}}/embedded|g" .venv/bin/*
 
               # Install virtual environment
-              mkdir -p ${{targets.contextdir}}/opt/datadog-agent/embedded
-              cp -r .venv/* ${{targets.contextdir}}/opt/datadog-agent/embedded/
+              mkdir -p ${{targets.contextdir}}${{vars.destd}}/embedded
+              cp -r .venv/* ${{targets.contextdir}}${{vars.destd}}/embedded/
 
               # this is intentionally referencing the main package, since we only "install" if a config doesn't already exist
               conf_dir="${{targets.destdir}}/etc/datadog-agent/conf.d"
@@ -368,9 +379,9 @@ subpackages:
         - coreutils
     pipeline:
       - runs: |
-          mkdir -p ${{targets.subpkgdir}}/opt/datadog-agent/bin/
-          ln -s /usr/bin/datadog-cluster-agent ${{targets.subpkgdir}}/opt/datadog-agent/bin/datadog-cluster-agent
-          ln -s /usr/bin/datadog-cluster-agent ${{targets.subpkgdir}}/opt/datadog-agent/bin/agent
+          mkdir -p ${{targets.subpkgdir}}${{vars.destd}}/bin/
+          ln -s /usr/bin/datadog-cluster-agent ${{targets.subpkgdir}}${{vars.destd}}/bin/datadog-cluster-agent
+          ln -s /usr/bin/datadog-cluster-agent ${{targets.subpkgdir}}${{vars.destd}}/bin/agent
       - working-directory: Dockerfiles/cluster-agent
         runs: |
           install -Dm755 entrypoint.sh ${{targets.subpkgdir}}/entrypoint.sh
@@ -537,27 +548,38 @@ test:
         - datadog-agent-fakeintake=${{package.full-version}}
         - datadog-agent-core-integrations=${{package.full-version}}
     environment:
-      PYTHONPATH: "/opt/datadog-agent/embedded/lib/python3.11/site-packages"
+      # cannot use vars.destd here. https://github.com/chainguard-dev/melange/issues/1402
+      # setting PATH here has no effect.
+      mypath: /opt/datadog-agent/bin/agent:/opt/datadog-agent/embedded/bin
   pipeline:
     - name: Ensure the agent's requirements.txt for integrations is included
       runs: |
-        ls /opt/datadog-agent/requirements-agent-release.txt
+        PATH=$mypath:$PATH
+        ls ${{vars.destd}}/requirements-agent-release.txt
     - name: Ensure the agent integration subcommand works
       runs: |
-        /opt/datadog-agent/bin/agent/agent integration freeze >/dev/null
+        PATH=$mypath:$PATH
+        agent integration freeze >/dev/null
     - name: Ensure checks can be loaded
       runs: |
-        python -c "import datadog_checks.base"
-        /opt/datadog-agent/bin/agent/agent version
-        /opt/datadog-agent/bin/agent/agent --help
-        /opt/datadog-agent/embedded/bin/process-agent --version
-        /opt/datadog-agent/embedded/bin/process-agent --help
-        /opt/datadog-agent/embedded/bin/security-agent --help
-        /opt/datadog-agent/embedded/bin/system-probe --help
-        /opt/datadog-agent/embedded/bin/trace-agent --version
-        /opt/datadog-agent/embedded/bin/trace-agent --help
+        PATH=$mypath:$PATH
+        # this is left without path to be explicit which python is used.
+        ${{vars.destd}}/embedded/bin/python3 -c "import datadog_checks.base"
+    - name: Execute bins
+      runs: |
+        PATH=$mypath:$PATH
+        process-agent --version
+        process-agent --help
+        security-agent --help
+        system-probe --help
+        trace-agent --version
+        trace-agent --help
+
+        agent version
+        agent --help
     - name: Ensure agent can be started
       runs: |
+        PATH=$mypath:$PATH
         cat > /etc/datadog-agent/datadog.yaml <<EOF
         api_key: "test"
         dd_url: "http://localhost:80"
@@ -566,11 +588,12 @@ test:
 
         export DD_LOG_LEVEL=debug
         export DD_HOSTNAME_FILE=/etc/hostname
-        /opt/datadog-agent/bin/agent/agent check uptime
-        /opt/datadog-agent/bin/agent/agent check disk
+        agent check uptime
+        agent check disk
     - name: Validate multicall components are correctly linked
       runs: |
-        /opt/datadog-agent/embedded/bin/trace-agent version | grep trace-agent
-        /opt/datadog-agent/embedded/bin/security-agent version | grep "Security agent"
-        /opt/datadog-agent/embedded/bin/process-agent version | grep Agent
-        /opt/datadog-agent/embedded/bin/system-probe version | grep "System Probe"
+        PATH=$mypath:$PATH
+        trace-agent version | grep trace-agent
+        security-agent version | grep "Security agent"
+        process-agent version | grep Agent
+        system-probe version | grep "System Probe"