Move S3 keys secrets out of tasks secrets

Subdirectories in secrets are awkward with kubernetes secrets. It's also
desirable to hand out access to them in a finer-grained manner.

Our ci-secrets.git repo already moved the S3 keys out of tasks/ into the
top level. Follow suit.

ansible/roles/local-s3-alias/tasks/main.yml

@@ -1,7 +1,7 @@
 - name: Create s3-keys alias
   file:
     src: self-hosted
-    dest: "/var/lib/cockpit-secrets/tasks/s3-keys/{{ hostvars[groups['psi_s3'][0]].ansible_host }}"
+    dest: "/var/lib/cockpit-secrets/s3-keys/{{ hostvars[groups['psi_s3'][0]].ansible_host }}"
     state: link
     owner: cockpituous
     group: cockpituous

ansible/roles/local-secrets-archive/tasks/main.yml

@@ -17,4 +17,4 @@
    become: false
    run_once: yes
    shell: |
-     tar -C $XDG_RUNTIME_DIR/ci-secrets  -hz --hard-dereference -c webhook tasks > $XDG_RUNTIME_DIR/ci-secrets.tar.gz
+     tar -C $XDG_RUNTIME_DIR/ci-secrets  -hz --hard-dereference -c webhook s3-keys tasks > $XDG_RUNTIME_DIR/ci-secrets.tar.gz

ansible/roles/tasks-systemd/tasks/main.yml

@@ -85,7 +85,7 @@
       [logs.s3]
       # bots lib/stores.py LOG_STORE
       url = 'https://cockpit-logs.us-east-1.linodeobjects.com/'
-      key = [{file="/run/secrets/tasks/s3-keys/cockpit-logs.us-east-1.linodeobjects.com"}]
+      key = [{file="/run/secrets/s3-keys/cockpit-logs.us-east-1.linodeobjects.com"}]
 
       [container]
       command = ['podman-remote', '--url=unix:///podman.sock']
@@ -128,7 +128,7 @@
       # these are *host* paths, this is podman-remote
       # secret from issue-scan for image refreshes
       image-upload=[
-          '--volume=/var/lib/cockpit-secrets/tasks/s3-keys/:/run/secrets/s3-keys:ro',
+          '--volume=/var/lib/cockpit-secrets/s3-keys/:/run/secrets/s3-keys:ro',
           '--env=COCKPIT_S3_KEY_DIR=/run/secrets/s3-keys',
           # password for console.redhat.com when image-create'ing rhel4edge
           '--volume=/var/lib/cockpit-secrets/tasks/crc_passwd:/run/secrets/crc_passwd:ro',
@@ -137,7 +137,7 @@
       # secret from tests-scan for downloading RHEL images
       image-download=[
           # FIXME: create a new "download only" S3 token
-          '--volume=/var/lib/cockpit-secrets/tasks/s3-keys/:/run/secrets/s3-keys:ro',
+          '--volume=/var/lib/cockpit-secrets/s3-keys/:/run/secrets/s3-keys:ro',
           '--env=COCKPIT_S3_KEY_DIR=/run/secrets/s3-keys',
       ]
       github-token=[

local-s3/install-s3-service

@@ -41,7 +41,7 @@ EOF
 cat <<EOF > /usr/local/lib/setup-s3.sh
 #!/bin/sh
 set -eu
-read s3user s3key < "$SECRETS/tasks/s3-keys/self-hosted"
+read s3user s3key < "$SECRETS/s3-keys/self-hosted"
 $RUNC run --interactive --rm --network=host \
     -v "$SECRETS"/webhook/ca.pem:/etc/pki/ca-trust/source/anchors/ca.pem:ro \
     --entrypoint /bin/sh quay.io/minio/mc <<EOC

tasks/README.md

@@ -10,7 +10,7 @@ The container has optional mounts:
    mounted from `/var/cache/cockpit-tasks/images`.
  * S3 access tokens for image and log buckets. Defined by `$COCKPIT_S3_KEY_DIR`
    env variable, conventionally `/run/secrets/s3-keys`.
-   On production hosts, this is mounted from `/var/lib/cockpit-secrets/tasks/s3-keys`.
+   On production hosts, this is mounted from `/var/lib/cockpit-secrets/s3-keys`.
  * A directory for GitHub and AMQP secrets. Used by both the tasks and the the webhook container.
    Must be in `/run/secrets/webhook` (bots currently assumes that).
    * `.config--github-token`: GitHub token to create and update issues and PRs.

tasks/build-secrets

@@ -16,7 +16,7 @@ metadata:
 data:
 EOF
 cd "$BASE/tasks"
-# This intentionally does not capture subdirs like tasks/s3-keys/. As OpenShift secret volumes don't have subdirectories,
+# This intentionally does not capture subdirs. As OpenShift secret volumes don't have subdirectories,
 # these need to be created as a separate volume if and when we ever need that.
 for f in $(find -maxdepth 1 -type f -o -type l); do
     printf '  %s: %s\n' "${f#./}" "$(base64 --wrap=0 $f)"
@@ -31,7 +31,7 @@ metadata:
   name: cockpit-s3-secrets
 data:
 EOF
-cd "$BASE/tasks/s3-keys"
+cd "$BASE/s3-keys"
 for f in $(find -maxdepth 1 -type f -o -type l); do
     printf '  %s: %s\n' "${f#./}" "$(base64 --wrap=0 $f)"
 done

tasks/install-service

@@ -49,12 +49,13 @@ ExecStartPre=/usr/bin/flock /tmp/cockpit-image-pull podman pull ghcr.io/cockpit-
 ExecStart=/usr/bin/podman run --name=cockpit-tasks-%i --hostname=${CONTAINER_HOSTNAME} \
     --volume=${CACHE}/images:/cache/images:rw \
     --volume=${SECRETS}/tasks:/run/secrets/tasks:ro \
+    --volume=${SECRETS}/s3-keys:/run/secrets/s3-keys:ro \
     --volume=${SECRETS}/webhook:/run/secrets/webhook:ro \
     --volume=/etc/job-runner.toml:/config/job-runner.toml:ro \
     --volume=%t/podman/podman.sock:/podman.sock:rw \
     --env=JOB_RUNNER_CONFIG=/config/job-runner.toml \
     --env=COCKPIT_GITHUB_TOKEN_FILE=/run/secrets/webhook/.config--github-token \
-    --env=COCKPIT_S3_KEY_DIR=/run/secrets/tasks/s3-keys \
+    --env=COCKPIT_S3_KEY_DIR=/run/secrets/s3-keys \
     --env=COCKPIT_IMAGES_DATA_DIR=/cache/images \
     --env=GIT_COMMITTER_NAME=Cockpituous \
     [email protected] \

test/test_deployment.py

@@ -35,6 +35,7 @@ class Config:
     secrets: Path
     webhook: Path
     tasks: Path
+    s3_keys: Path
 
 
 @pytest.fixture(scope='session')
@@ -71,9 +72,11 @@ def config(tmp_path_factory) -> Config:
     config.tasks = config.secrets / 'tasks'
     config.tasks.mkdir()
     subprocess.run(ROOT_DIR / 'local-s3/generate-s3-cert.sh', cwd=config.tasks, check=True)
+
     # minio S3 key
-    (config.tasks / 's3-keys').mkdir()
-    (config.tasks / 's3-keys/localhost.localdomain').write_text('cockpituous foobarfoo')
+    config.s3_keys = config.secrets / 's3-keys'
+    config.s3_keys.mkdir()
+    (config.s3_keys / 'localhost.localdomain').write_text('cockpituous foobarfoo')
 
     # need to make secrets world-readable, as containers run as non-root
     subprocess.run(['chmod', '-R', 'go+rX', configdir], check=True)
@@ -156,7 +159,7 @@ def pod(config: Config, pytestconfig) -> Iterator[PodData]:
                    check=True)
 
     # wait until S3 started, create bucket
-    (s3user, s3key) = (config.tasks / 's3-keys/localhost.localdomain').read_text().strip().split()
+    (s3user, s3key) = (config.s3_keys / 'localhost.localdomain').read_text().strip().split()
     exec_c(data.mc, f'''
 set -e
 cat /etc/pki/ca-trust/source/anchors/ca.pem >> /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem
@@ -175,6 +178,7 @@ def pod(config: Config, pytestconfig) -> Iterator[PodData]:
                     '-v', f'{PODMAN_SOCKET}:/podman.sock',
                     '-v', f'{config.webhook}:/run/secrets/webhook:ro',
                     '-v', f'{config.tasks}:/run/secrets/tasks:ro',
+                    '-v', f'{config.s3_keys}:/run/secrets/s3-keys:ro',
                     '-e', 'COCKPIT_GITHUB_TOKEN_FILE=/run/secrets/webhook/.config--github-token',
                     '-e', 'COCKPIT_CA_PEM=/run/secrets/webhook/ca.pem',
                     '-e', f'COCKPIT_BOTS_REPO={os.getenv("COCKPIT_BOTS_REPO", "")}',
@@ -183,7 +187,7 @@ def pod(config: Config, pytestconfig) -> Iterator[PodData]:
                     '-e', 'JOB_RUNNER_CONFIG=/run/secrets/tasks/job-runner.toml',
                     '-e', f'AMQP_SERVER={AMQP_POD}',
                     '-e', f'S3_LOGS_URL={S3_URL_POD}/logs/',
-                    '-e', 'COCKPIT_S3_KEY_DIR=/run/secrets/tasks/s3-keys',
+                    '-e', 'COCKPIT_S3_KEY_DIR=/run/secrets/s3-keys',
                     '-e', f'COCKPIT_IMAGE_UPLOAD_STORE={S3_URL_POD}/images/',
                     '-e', 'COCKPIT_IMAGES_DATA_DIR=/cache/images',
                     '-e', 'GIT_COMMITTER_NAME=Cockpituous',
@@ -312,7 +316,7 @@ def generate_config(config: Config, forge_opts: str, run_args: str) -> Path:
         [logs.s3]
         url = '{S3_URL_POD}/logs'
         ca = [{{file='/run/secrets/webhook/ca.pem'}}]
-        key = [{{file="/run/secrets/tasks/s3-keys/localhost.localdomain"}}]
+        key = [{{file="/run/secrets/s3-keys/localhost.localdomain"}}]
 
         [container]
         command = ['podman-remote', '--url=unix:///podman.sock']
@@ -328,7 +332,7 @@ def generate_config(config: Config, forge_opts: str, run_args: str) -> Path:
         [container.secrets]
         # these are *host* paths, this is podman-remote
         image-upload=[
-            '--volume={config.tasks}/s3-keys:/run/secrets/s3-keys:ro',
+            '--volume={config.s3_keys}:/run/secrets/s3-keys:ro',
             '--env=COCKPIT_S3_KEY_DIR=/run/secrets/s3-keys',
             '--volume={config.webhook}/ca.pem:/run/secrets/ca.pem:ro',
             '--env=COCKPIT_CA_PEM=/run/secrets/ca.pem',