✨ Add matrix to compare envvar usage across components

issue: open-zaak/open-zaak#1629

bin/create_envvar_matrix.py

@@ -0,0 +1,74 @@
+import csv
+import re
+from collections import defaultdict
+from pathlib import Path
+
+import requests
+
+SETTINGS_PATHS = [
+    "https://raw.githubusercontent.com/open-zaak/open-zaak/main/src/openzaak/conf",
+    "https://raw.githubusercontent.com/open-zaak/open-notificaties/main/src/nrc/conf",
+    "https://raw.githubusercontent.com/maykinmedia/objects-api/master/src/objects/conf",
+    "https://raw.githubusercontent.com/maykinmedia/objecttypes-api/master/src/objecttypes/conf",  # noqa
+    "https://raw.githubusercontent.com/maykinmedia/open-klant/master/src/openklant/conf",  # noqa
+]
+SETTINGS_FILES = [
+    "api.py",
+    "base.py",
+    "ci.py",
+    "dev.py",
+    "docker.py",
+    "production.py",
+    "staging.py",
+    "test.py",
+]
+
+ENVVAR_REGEX = re.compile(
+    r'config\((?:(?:\"|\')|[\s#A-z0-9:\/-]*")([A-z0-9-_]+)(?:\"|\')'
+)
+
+
+def parse_settings_for_repo(repository_link, settings_file):
+    url = f"{repository_link}/{settings_file}"
+    if (
+        "openzaak" in repository_link or "nrc" in repository_link
+    ) and settings_file in ["base.py", "api.py"]:
+        url = f"{repository_link}/includes/{settings_file}"
+
+    file_data = requests.get(url).text
+
+    return ENVVAR_REGEX.findall(file_data)
+
+
+def parse_all():
+    results = defaultdict(list)
+    for base_dir in SETTINGS_PATHS:
+        project_name = base_dir.split("/")[-5]
+        for settings_file in SETTINGS_FILES:
+            envvar_names = parse_settings_for_repo(base_dir, settings_file)
+            for name in envvar_names:
+                results[name].append(project_name)
+    return results
+
+
+def export_csv(data):
+    Path("reports").mkdir(parents=True, exist_ok=True)
+    with open("reports/envvar_matrix.csv", "w") as f:
+        writer = csv.writer(f)
+        all_components = sorted(
+            list(set(component for values in data.values() for component in values))
+        )
+        export = [["var name"] + all_components]
+        for var_name, components in data.items():
+            export.append(
+                [var_name]
+                + [
+                    "x" if component in components else ""
+                    for component in all_components
+                ]
+            )
+        writer.writerows(export)
+
+
+data = parse_all()
+export_csv(data)

reports/envvar_matrix.csv

@@ -0,0 +1,116 @@
+var name,objects-api,objecttypes-api,open-klant,open-notificaties,open-zaak
+SITE_ID,x,x,x,x,x
+SECRET_KEY,x,x,x,x,x
+DEBUG,x,x,x,x,x
+ALLOWED_HOSTS,x,x,x,x,x
+USE_X_FORWARDED_HOST,x,x,,,x
+IS_HTTPS,x,x,x,x,x
+DB_NAME,x,x,x,x,x
+DB_USER,x,x,x,x,x
+DB_PASSWORD,x,x,x,x,x
+DB_HOST,x,x,x,x,x
+DB_PORT,x,x,x,x,x
+GEOS_LIBRARY_PATH,,,,,x
+GDAL_LIBRARY_PATH,,,,,x
+CACHE_DEFAULT,x,,x,x,x
+CACHE_AXES,x,,x,x,x
+EMAIL_HOST,x,x,x,x,x
+EMAIL_PORT,x,x,x,x,x
+EMAIL_HOST_USER,x,x,x,x,x
+EMAIL_HOST_PASSWORD,x,x,x,x,x
+EMAIL_USE_TLS,x,x,x,x,x
+LOG_STDOUT,,,x,,x
+LOG_LEVEL,,,,,x
+LOG_QUERIES,,,,,x
+LOG_REQUESTS,x,,x,x,x
+OPENZAAK_DOMAIN,,,,,x
+OPENZAAK_REWRITE_HOST,,,,,x
+ENVIRONMENT,x,x,x,,x
+MIN_UPLOAD_SIZE,,,,,x
+DOCUMENTEN_UPLOAD_CHUNK_SIZE,,,,,x
+DOCUMENTEN_UPLOAD_READ_CHUNK,,,,,x
+SUBPATH,x,x,x,x,x
+GIT_SHA,,,x,x,x
+RELEASE,,,x,x,x
+NUM_PROXIES,,,x,x,x
+CORS_ALLOW_ALL_ORIGINS,,,,x,x
+CORS_ALLOWED_ORIGINS,,,,x,x
+CORS_ALLOWED_ORIGIN_REGEXES,,,,x,x
+CORS_EXTRA_ALLOW_HEADERS,,,,x,x
+CSRF_TRUSTED_ORIGINS,,,,,x
+SENDFILE_BACKEND,,,,,x
+NOTIFICATIONS_DISABLED,x,,x,,x
+LOOSE_FK_LOCAL_BASE_URLS,,,,,x
+SENTRY_DSN,x,x,x,x,x
+CELERY_RESULT_BACKEND,x,,,x,x
+ELASTIC_APM_SERVER_URL,x,x,,x,x
+ELASTIC_APM_SECRET_TOKEN,x,x,x,x,x
+LOG_OUTGOING_REQUESTS_DB_SAVE,x,,x,x,x
+JWT_EXPIRY,,,,,x
+JWT_LEEWAY,,,,,x
+CMIS_ENABLED,,,,,x
+CMIS_MAPPER_FILE,,,,,x
+CMIS_URL_MAPPING_ENABLED,,,,,x
+IMPORT_REQUESTS_CACHE_NAME,,,,,x
+ZAAK_EIGENSCHAP_WAARDE_VALIDATION,,,,,x
+SITES_CONFIG_ENABLE,x,x,,x,x
+OPENZAAK_ORGANIZATION,,,,,x
+NOTIF_OPENZAAK_CONFIG_ENABLE,,,,,x
+NOTIF_OPENZAAK_CLIENT_ID,,,,x,x
+NOTIF_OPENZAAK_SECRET,,,,x,x
+OPENZAAK_NOTIF_CONFIG_ENABLE,,,,x,x
+NOTIF_API_ROOT,,,,,x
+NOTIF_API_OAS,,,,,x
+OPENZAAK_NOTIF_CLIENT_ID,,,,x,x
+OPENZAAK_NOTIF_SECRET,,,,x,x
+OPENZAAK_SELECTIELIJST_CONFIG_ENABLE,,,,,x
+SELECTIELIJST_API_ROOT,,,,,x
+SELECTIELIJST_API_OAS,,,,,x
+SELECTIELIJST_ALLOWED_YEARS,,,,,x
+SELECTIELIJST_DEFAULT_YEAR,,,,,x
+DEMO_CONFIG_ENABLE,x,x,,,x
+DEMO_CLIENT_ID,,,,,x
+DEMO_SECRET,,,,,x
+PROFILE,,,,,x
+DISABLE_APM_IN_DEV,,,,x,x
+DB_CONN_MAX_AGE,,,,,x
+LOG_OUTGOING_REQUESTS_DB_SAVE_BODY,,,,x,
+LOG_OUTGOING_REQUESTS_EMIT_BODY,,,,x,
+LOG_NOTIFICATIONS_IN_DB,,,,x,
+PUBLISH_BROKER_URL,,,,x,
+CELERY_BROKER_URL,x,,,x,
+NOTIFICATION_NUMBER_OF_DAYS_RETAINED,,,,x,
+ELASTIC_APM_TRANSACTION_SAMPLE_RATE,,,,x,
+OPENNOTIFICATIES_DOMAIN,,,,x,
+OPENNOTIFICATIES_ORGANIZATION,,,,x,
+AUTHORIZATION_CONFIG_ENABLE,,,,x,
+AUTORISATIES_API_ROOT,,,,x,
+NOTIFICATION_RETRY_CONFIG_ENABLE,,,,x,
+NOTIFICATION_DELIVERY_MAX_RETRIES,,,,x,
+NOTIFICATION_DELIVERY_RETRY_BACKOFF,,,,x,
+NOTIFICATION_DELIVERY_RETRY_BACKOFF_MAX,,,,x,
+CACHE_OIDC,x,,,,
+DEFAULT_FROM_EMAIL,x,x,,,
+VERSION_TAG,x,x,,,
+ELASTIC_APM_SERVICE_NAME,x,x,,,
+DISABLE_2FA,x,x,,,
+CELERY_TASK_HARD_TIME_LIMIT,x,,,,
+CELERY_TASK_SOFT_TIME_LIMIT,x,,,,
+OBJECTS_DOMAIN,x,,,,
+OBJECTS_ORGANIZATION,x,,,,
+OBJECTS_OBJECTTYPES_CONFIG_ENABLE,x,x,,,
+OBJECTTYPES_API_ROOT,x,,,,
+OBJECTTYPES_API_OAS,x,,,,
+OBJECTS_OBJECTTYPES_TOKEN,x,x,,,
+DEMO_TOKEN,x,x,,,
+DEMO_PERSON,x,x,,,
+DEMO_EMAIL,x,x,,,
+OBJECTTYPES_DOMAIN,,x,,,
+OBJECTTYPES_ORGANIZATION,,x,,,
+OBJECTS_OBJECTTYPES_PERSON,,x,,,
+OBJECTS_OBJECTTYPES_EMAIL,,x,,,
+SESSION_COOKIE_SECURE,,x,,,
+SESSION_COOKIE_HTTPONLY,,x,,,
+CSRF_COOKIE_SECURE,,x,,,
+DB_ENGINE,,,x,,
+LOG_OUTGOING_REQUESTS_MAX_AGE,,,x,,