Skip to content

Commit

Permalink
Merge branch 'lsst-sqre:main' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
stvoutsin authored Jan 25, 2024
2 parents 4bda777 + 0ff0033 commit 4a0ee93
Show file tree
Hide file tree
Showing 702 changed files with 19,961 additions and 8,214 deletions.
48 changes: 21 additions & 27 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,17 @@ jobs:
- uses: actions/checkout@v4

- name: Set up go
uses: actions/setup-go@v4
uses: actions/setup-go@v5

- name: Install helm-docs
run: go install github.com/norwoodj/helm-docs/cmd/[email protected].0
run: go install github.com/norwoodj/helm-docs/cmd/[email protected].3
env:
GOBIN: /usr/local/bin/

- name: Set up Python
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: "3.11"
python-version: "3.12"

- name: Run pre-commit
uses: pre-commit/[email protected]
Expand All @@ -35,6 +35,7 @@ jobs:
matrix:
python:
- "3.11"
- "3.12"

steps:
- uses: actions/checkout@v4
Expand All @@ -54,22 +55,16 @@ jobs:
with:
fetch-depth: 0

- name: Set up Python
uses: actions/setup-python@v4
- uses: azure/setup-helm@v3
with:
python-version: "3.11"

- name: Install test dependencies
run: make init

- name: Expand modified charts
run: expand-charts

- name: Set up chart-testing
uses: helm/[email protected]
# Used to query GitHub for the latest Helm release.
token: ${{ secrets.GITHUB_TOKEN }}

- name: Run chart-testing (lint)
run: ct lint --all --config ct.yaml
- uses: lsst-sqre/run-tox@v1
with:
python-version: "3.12"
tox-envs: phalanx-lint-change
cache-key-prefix: test

# The minikube job always runs, but it quickly does nothing if no files that
# would affect minikube were changed. This unfortunately requires a lot of
Expand All @@ -93,12 +88,12 @@ jobs:
filters: |
minikube:
- ".github/workflows/ci.yaml"
- "applications/*/Chart.yaml"
- "applications/*/templates/**"
- "applications/*/values.yaml"
- "applications/*/values-minikube.yaml"
- "applications/{argocd,gafaelfawr,ingress-nginx,mobu,postgres,squareone,vault-secrets-operator}/Chart.yaml"
- "applications/{argocd,gafaelfawr,ingress-nginx,mobu,postgres,squareone,vault-secrets-operator}/templates/**"
- "applications/{argocd,gafaelfawr,ingress-nginx,mobu,postgres,squareone,vault-secrets-operator}/values.yaml"
- "applications/{argocd,gafaelfawr,ingress-nginx,mobu,postgres,squareone,vault-secrets-operator}/values-minikube.yaml"
- "environments/Chart.yaml"
- "environments/templates/**"
- "environments/templates/{argocd,gafaelfawr,ingress-nginx,mobu,postgres,squareone,vault-secrets-operator}*"
- "environments/values-minikube.yaml"
- "installer/**"
Expand All @@ -117,21 +112,20 @@ jobs:
- name: Download installer dependencies
if: steps.filter.outputs.minikube == 'true'
run: |
curl -sSL -o /tmp/vault.zip https://releases.hashicorp.com/vault/1.14.0/vault_1.14.0_linux_amd64.zip
curl -sSL -o /tmp/vault.zip https://releases.hashicorp.com/vault/1.15.4/vault_1.15.4_linux_amd64.zip
unzip /tmp/vault.zip
sudo mv vault /usr/local/bin/vault
sudo chmod +x /usr/local/bin/vault
sudo curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/download/v2.7.10/argocd-linux-amd64
sudo curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/download/v2.8.6/argocd-linux-amd64
sudo chmod +x /usr/local/bin/argocd
sudo apt-get install socat
sudo pip install -r installer/requirements.txt
- name: Run installer
timeout-minutes: 15
if: steps.filter.outputs.minikube == 'true'
run: |
cd installer
./install.sh minikube ${{ secrets.MINIKUBE_VAULT_KEY }}
./install.sh minikube "${{ secrets.MINIKUBE_VAULT_ROLE_ID }}" "${{ secrets.MINIKUBE_VAULT_SECRET_ID }}"
- name: Get final list of resources
if: steps.filter.outputs.minikube == 'true'
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/dependencies.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
- name: Run neophile
uses: lsst-sqre/run-neophile@v1
with:
python-version: "3.11"
python-version: "3.12"
mode: pr
types: python
app-id: ${{ secrets.NEOPHILE_APP_ID }}
Expand Down
26 changes: 13 additions & 13 deletions .github/workflows/docs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ name: Docs
- "renovate/**"
- "tickets/**"
- "u/**"
tags:
- "*"
workflow_dispatch: {}

jobs:
Expand All @@ -39,25 +37,28 @@ jobs:
- "docs/**"
- "applications/*/Chart.yaml"
- "applications/*/values.yaml"
- "applications/argocd/values-*.yaml"
- "applications/gafaelfawr/values-*.yaml"
- "environments/values-*.yaml"
- "src/phalanx/**"
docsSpecific:
- "docs/**"
- name: Install graphviz
if: steps.filter.outputs.docs == 'true'
if: steps.filter.outputs.docs == 'true' || github.event_name == 'workflow_dispatch'
run: sudo apt-get install graphviz

- name: Build docs
if: steps.filter.outputs.docs == 'true'
if: steps.filter.outputs.docs == 'true' || github.event_name == 'workflow_dispatch'
uses: lsst-sqre/run-tox@v1
with:
python-version: "3.11"
python-version: "3.12"
tox-envs: docs

# Only attempt documentation uploads for tagged releases and pull
# requests from ticket branches in the same repository. This avoids
# version clutter in the docs and failures when a PR doesn't have access
# to secrets.
# Upload docs:
# - on pushes to main if *any* documentation content might have changed
# - on workflow dispatches if any documentation content might have changed
# - on pushes to tickets/ branches if docs/ directory content changed
- name: Upload to LSST the Docs
uses: lsst-sqre/ltd-upload@v1
with:
Expand All @@ -66,7 +67,6 @@ jobs:
username: ${{ secrets.LTD_USERNAME }}
password: ${{ secrets.LTD_PASSWORD }}
if: >-
steps.filter.outputs.docs == 'true'
&& github.event_name != 'merge_group'
&& (github.event_name != 'pull_request'
|| startsWith(github.head_ref, 'tickets/'))
(github.event_name == 'push' && github.ref_name == 'main' && steps.filter.outputs.docs == 'true')
|| (github.event_name == 'workflow_dispatch')
|| (github.event_name == 'pull_request' && startsWith(github.head_ref, 'tickets/') && steps.filter.outputs.docsSpecific == 'true')
2 changes: 1 addition & 1 deletion .github/workflows/linkcheck.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,5 +48,5 @@ jobs:
- name: Check links
uses: lsst-sqre/run-tox@v1
with:
python-version: "3.11"
python-version: "3.12"
tox-envs: docs-linkcheck
16 changes: 6 additions & 10 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
rev: v4.5.0
hooks:
- id: check-merge-conflict
- id: check-toml
- id: trailing-whitespace

- repo: https://github.com/adrienverge/yamllint
rev: v1.32.0
rev: v1.33.0
hooks:
- id: yamllint
args:
- -c=.yamllint.yml

- repo: https://github.com/python-jsonschema/check-jsonschema
rev: 0.26.3
rev: 0.27.3
hooks:
- id: check-jsonschema
files: ^applications/.*/secrets(-[^./-]+)?\.yaml
Expand All @@ -26,7 +26,7 @@ repos:
files: ^docs/extras/schemas/.*\.json

- repo: https://github.com/norwoodj/helm-docs
rev: v1.11.0
rev: v1.11.3
hooks:
- id: helm-docs
args:
Expand All @@ -46,15 +46,11 @@ repos:
- --template-files=../helm-docs.md.gotmpl

- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.0.289
rev: v0.1.8
hooks:
- id: ruff
args: [--fix, --exit-non-zero-on-fix]

- repo: https://github.com/psf/black
rev: 23.7.0
hooks:
- id: black
- id: ruff-format

- repo: https://github.com/adamchainz/blacken-docs
rev: 1.16.0
Expand Down
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ clean:

.PHONY: init
init:
pip install --upgrade pip pre-commit tox
pre-commit install
pip install --editable .
pip install --upgrade -r requirements/main.txt -r requirements/dev.txt
rm -rf .tox
pip install --upgrade pre-commit tox
pre-commit install

# This is defined as a Makefile target instead of only a tox command because
# if the command fails we want to cat output.txt, which contains the
Expand Down
4 changes: 2 additions & 2 deletions applications/alert-stream-broker/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
apiVersion: v2
name: alert-stream-broker
version: "3"
version: 1.0.0
description: Alert transmission to community brokers
sources:
- https://github.com/lsst-dm/alert_database_ingester
- https://github.com/lsst-dm/alert-stream-simulator
dependencies:
- name: alert-stream-broker
version: 2.5.1
version: 2.5.2

# The schema registry is bundled together in the same application as the
# Kafka broker because Strimzi Registry Operator expects everything (the
Expand Down
12 changes: 10 additions & 2 deletions applications/alert-stream-broker/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ Alert transmission to community brokers
| alert-database.storage.gcp.project | string | `""` | Name of a GCP project that has a bucket for database storage |
| alert-database.storage.gcp.schemaBucket | string | `""` | Name of a Google Cloud Storage bucket in GCP with schema data |
| alert-stream-broker.cluster.name | string | `"alert-broker"` | Name used for the Kafka broker, and used by Strimzi for many annotations. |
| alert-stream-broker.clusterName | string | `"alert-broker"` | Name of a Strimzi Kafka cluster to connect to. |
| alert-stream-broker.clusterPort | int | `9092` | Port to connect to on the Strimzi Kafka cluster. It should be an internal TLS listener. |
| alert-stream-broker.fullnameOverride | string | `""` | Override for the full name used for Kubernetes resources; by default one will be created based on the chart name and helm release name. |
| alert-stream-broker.kafka.config | object | `{"log.retention.bytes":"42949672960","log.retention.hours":168,"offsets.retention.minutes":1440}` | Configuration overrides for the Kafka server. |
| alert-stream-broker.kafka.config."log.retention.bytes" | string | `"42949672960"` | Maximum retained number of bytes for a broker's data. This is a string to avoid YAML type conversion issues for large numbers. |
Expand All @@ -76,14 +78,20 @@ Alert transmission to community brokers
| alert-stream-broker.kafkaExporter.groupRegex | string | `".*"` | Consumer groups to monitor |
| alert-stream-broker.kafkaExporter.logLevel | string | `"warning"` | Log level for Sarama logging |
| alert-stream-broker.kafkaExporter.topicRegex | string | `".*"` | Kafka topics to monitor |
| alert-stream-broker.maxBytesRetained | string | `"24000000000"` | Maximum number of bytes for the replay topic, per partition, per replica. Default is 100GB, but should be lower to not fill storage. |
| alert-stream-broker.maxMillisecondsRetained | string | `"604800000"` | Maximum amount of time to save simulated alerts in the replay topic, in milliseconds. Default is 7 days. |
| alert-stream-broker.nameOverride | string | `""` | |
| alert-stream-broker.schemaID | int | `1` | Integer ID to use in the prefix of alert data packets. This should be a valid Confluent Schema Registry ID associated with the schema used. |
| alert-stream-broker.strimziAPIVersion | string | `"v1beta2"` | Version of the Strimzi Custom Resource API. The correct value depends on the deployed version of Strimzi. See [this blog post](https://strimzi.io/blog/2021/04/29/api-conversion/) for more. |
| alert-stream-broker.superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. |
| alert-stream-broker.testTopicName | string | `"alert-stream-test"` | Name of the topic which will be used to send test alerts. |
| alert-stream-broker.testTopicPartitions | int | `8` | |
| alert-stream-broker.testTopicReplicas | int | `2` | |
| alert-stream-broker.tls.certIssuerName | string | `"letsencrypt-dns"` | Name of a ClusterIssuer capable of provisioning a TLS certificate for the broker. |
| alert-stream-broker.tls.subject.organization | string | `"Vera C. Rubin Observatory"` | Organization to use in the 'Subject' field of the broker's TLS certificate. |
| alert-stream-broker.users | list | `[{"groups":["rubin-testing"],"readonlyTopics":["alert-stream","alerts-simulated"],"username":"rubin-testing"}]` | A list of users that should be created and granted access. Passwords for these users are not generated automatically; they are expected to be stored as 1Password secrets which are replicated into Vault. Each username should have a "{{ $username }}-password" secret associated with it. |
| alert-stream-broker.users | list | `[{"groups":["rubin-testing"],"readonlyTopics":["alert-stream","alerts-simulated","alert-stream-test"],"username":"rubin-testing"}]` | A list of users that should be created and granted access. Passwords for these users are not generated automatically; they are expected to be stored as 1Password secrets which are replicated into Vault. Each username should have a "{{ $username }}-password" secret associated with it. |
| alert-stream-broker.users[0].groups | list | `["rubin-testing"]` | A list of string prefixes for groups that the user should get admin access to, allowing them to create, delete, describe, etc consumer groups. Note that these are prefix-matched, not just literal exact matches. |
| alert-stream-broker.users[0].readonlyTopics | list | `["alert-stream","alerts-simulated"]` | A list of topics that the user should get read-only access to. |
| alert-stream-broker.users[0].readonlyTopics | list | `["alert-stream","alerts-simulated","alert-stream-test"]` | A list of topics that the user should get read-only access to. |
| alert-stream-broker.users[0].username | string | `"rubin-testing"` | The username for the user that should be created. |
| alert-stream-broker.vaultSecretsPath | string | `""` | Path to the secret resource in Vault |
| alert-stream-broker.zookeeper.replicas | int | `3` | Number of Zookeeper replicas to run. |
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
apiVersion: v2
name: alert-stream-broker
version: 2.5.1
version: 2.5.2
description: Kafka broker cluster for distributing alerts
maintainers:
- name: bsmart
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ Kafka broker cluster for distributing alerts
| Key | Type | Default | Description |
|-----|------|---------|-------------|
| cluster.name | string | `"alert-broker"` | Name used for the Kafka broker, and used by Strimzi for many annotations. |
| clusterName | string | `"alert-broker"` | Name of a Strimzi Kafka cluster to connect to. |
| clusterPort | int | `9092` | Port to connect to on the Strimzi Kafka cluster. It should be an internal TLS listener. |
| fullnameOverride | string | `""` | Override for the full name used for Kubernetes resources; by default one will be created based on the chart name and helm release name. |
| kafka.config | object | `{"log.retention.bytes":"42949672960","log.retention.hours":168,"offsets.retention.minutes":1440}` | Configuration overrides for the Kafka server. |
| kafka.config."log.retention.bytes" | string | `"42949672960"` | Maximum retained number of bytes for a broker's data. This is a string to avoid YAML type conversion issues for large numbers. |
Expand All @@ -33,14 +35,20 @@ Kafka broker cluster for distributing alerts
| kafkaExporter.groupRegex | string | `".*"` | Consumer groups to monitor |
| kafkaExporter.logLevel | string | `"warning"` | Log level for Sarama logging |
| kafkaExporter.topicRegex | string | `".*"` | Kafka topics to monitor |
| maxBytesRetained | string | `"24000000000"` | Maximum number of bytes for the replay topic, per partition, per replica. Default is 100GB, but should be lower to not fill storage. |
| maxMillisecondsRetained | string | `"604800000"` | Maximum amount of time to save simulated alerts in the replay topic, in milliseconds. Default is 7 days. |
| nameOverride | string | `""` | |
| schemaID | int | `1` | Integer ID to use in the prefix of alert data packets. This should be a valid Confluent Schema Registry ID associated with the schema used. |
| strimziAPIVersion | string | `"v1beta2"` | Version of the Strimzi Custom Resource API. The correct value depends on the deployed version of Strimzi. See [this blog post](https://strimzi.io/blog/2021/04/29/api-conversion/) for more. |
| superusers | list | `["kafka-admin"]` | A list of usernames for users who should have global admin permissions. These users will be created, along with their credentials. |
| testTopicName | string | `"alert-stream-test"` | Name of the topic which will be used to send test alerts. |
| testTopicPartitions | int | `8` | |
| testTopicReplicas | int | `2` | |
| tls.certIssuerName | string | `"letsencrypt-dns"` | Name of a ClusterIssuer capable of provisioning a TLS certificate for the broker. |
| tls.subject.organization | string | `"Vera C. Rubin Observatory"` | Organization to use in the 'Subject' field of the broker's TLS certificate. |
| users | list | `[{"groups":["rubin-testing"],"readonlyTopics":["alert-stream","alerts-simulated"],"username":"rubin-testing"}]` | A list of users that should be created and granted access. Passwords for these users are not generated automatically; they are expected to be stored as 1Password secrets which are replicated into Vault. Each username should have a "{{ $username }}-password" secret associated with it. |
| users | list | `[{"groups":["rubin-testing"],"readonlyTopics":["alert-stream","alerts-simulated","alert-stream-test"],"username":"rubin-testing"}]` | A list of users that should be created and granted access. Passwords for these users are not generated automatically; they are expected to be stored as 1Password secrets which are replicated into Vault. Each username should have a "{{ $username }}-password" secret associated with it. |
| users[0].groups | list | `["rubin-testing"]` | A list of string prefixes for groups that the user should get admin access to, allowing them to create, delete, describe, etc consumer groups. Note that these are prefix-matched, not just literal exact matches. |
| users[0].readonlyTopics | list | `["alert-stream","alerts-simulated"]` | A list of topics that the user should get read-only access to. |
| users[0].readonlyTopics | list | `["alert-stream","alerts-simulated","alert-stream-test"]` | A list of topics that the user should get read-only access to. |
| users[0].username | string | `"rubin-testing"` | The username for the user that should be created. |
| vaultSecretsPath | string | `""` | Path to the secret resource in Vault |
| zookeeper.replicas | int | `3` | Number of Zookeeper replicas to run. |
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
apiVersion: "kafka.strimzi.io/{{ .Values.strimziAPIVersion }}"
kind: KafkaTopic
metadata:
name: "{{ .Values.testTopicName }}"
labels:
strimzi.io/cluster: "{{ .Values.clusterName }}"
spec:
partitions: {{ .Values.testTopicPartitions }}
replicas: {{ .Values.testTopicReplicas }}
config:
cleanup.policy: "delete"
retention.ms: {{ .Values.maxMillisecondsRetained }} # 7 days
retention.bytes: {{ .Values.maxBytesRetained }}
Loading

0 comments on commit 4a0ee93

Please sign in to comment.