From 962dfa3e5cc8189fa0c2462671fed1f1d742f2c0 Mon Sep 17 00:00:00 2001 From: Ori Hoch Date: Thu, 25 Jan 2024 17:28:26 +0200 Subject: [PATCH] add db backup cronjob --- apps/anyway/.env | 1 + apps/anyway/README.md | 2 -- apps/anyway/compose.yaml | 26 ++++++++++++++++++++++++++ apps/redash/compose.yaml | 7 +++++++ apps/selenium/compose.yaml | 3 +++ bin/apps.py | 20 +++++++++++++++++++- 6 files changed, 56 insertions(+), 3 deletions(-) diff --git a/apps/anyway/.env b/apps/anyway/.env index 8e26f0e..fd1148d 100644 --- a/apps/anyway/.env +++ b/apps/anyway/.env @@ -1,4 +1,5 @@ PIN_DB_IMAGE=ghcr.io/hasadna/anyway/db:sha-6dfd43b +PIN_DB_BACKUP_IMAGE=ghcr.io/hasadna/anyway/db_backup:sha-6dfd43b ANYWAY_IMAGE=ghcr.io/data-for-change/anyway/anyway:sha-020da4e AIRFLOW_IMAGE=ghcr.io/data-for-change/anyway-etl/anyway-etl-airflow:v0.0.55 ETL_NGINX_IMAGE=ghcr.io/data-for-change/anyway-etl/anyway-etl-nginx:v0.0.55 diff --git a/apps/anyway/README.md b/apps/anyway/README.md index 0bda695..09a5c4c 100644 --- a/apps/anyway/README.md +++ b/apps/anyway/README.md @@ -6,8 +6,6 @@ https://docs.google.com/presentation/d/1bXkcCgsXUr1FQA7hCZdb5_m7IXIiP1UixuOHuV88 ![](image.png) -### TODO: db-backup-cronjob - ## Enable DB Redash read-only user Start a shell on DB container and run the following to start an sql session: diff --git a/apps/anyway/compose.yaml b/apps/anyway/compose.yaml index 303be76..c64bb0d 100644 --- a/apps/anyway/compose.yaml +++ b/apps/anyway/compose.yaml @@ -68,6 +68,8 @@ services: airflow-scheduler: image: ${AIRFLOW_IMAGE:-ghcr.io/data-for-change/anyway-etl/anyway-etl-airflow:latest} restart: unless-stopped + depends_on: + - airflow-db environment: ANYWAY_ETL_AIRFLOW_ROLE: "scheduler" ANYWAY_ETL_AIRFLOW_PIP_INSTALL_DEPS: "yes" @@ -85,6 +87,8 @@ services: networks: [dfc] airflow-nginx: + depends_on: + - airflow-webserver image: ${ETL_NGINX_IMAGE:-ghcr.io/data-for-change/anyway-etl/anyway-etl-nginx:latest} restart: unless-stopped volumes: @@ -98,6 +102,8 @@ services: # - "traefik.http.routers.airflow-nginx.tls.certresolver=dfc" airflow-webserver: + depends_on: + - airflow-scheduler image: ${AIRFLOW_IMAGE:-ghcr.io/data-for-change/anyway-etl/anyway-etl-airflow:latest} restart: unless-stopped environment: @@ -130,6 +136,8 @@ services: # - "traefik.http.routers.anyway-reports.tls.certresolver=dfc" nginx: + depends_on: + - anyway-secondary hostname: anyway-nginx image: ${NGINX_IMAGE:-ghcr.io/data-for-change/anyway/nginx:latest} restart: unless-stopped @@ -143,6 +151,20 @@ services: # - "traefik.http.routers.anyway-nginx.tls=true" # - "traefik.http.routers.anyway-nginx.tls.certresolver=dfc" + db-backup-cronjob: + x-cronjob: + schedule: "10 0 * * *" + networks: [dfc] + image: ${PIN_DB_BACKUP_IMAGE:-ghcr.io/hasadna/anyway/db_backup:latest} + environment: + DBDUMP_S3_FILE_PREFIX: "test-docker-migration" + DBDUMP_HOST: "anyway-db" + DBDUMP_USER: "postgres" + DBDUMP_FULL_BUCKET: "dfc-anyway-full-db-dumps" + DBDUMP_PARTIAL_BUCKET: "dfc-anyway-partial-db-dumps" + env_file: + - ./secrets/db-backup.env + networks: dfc: external: true @@ -214,3 +236,7 @@ x-secrets: DBRESTORE_AWS_SECRET_ACCESS_KEY: "vault:projects/anyway/prod/aws_db_dumps_reader_user:secret_access_key" DBRESTORE_SET_ANYWAY_PASSWORD: "vault:projects/anyway/prod/k8s-secret-db:DBRESTORE_SET_ANYWAY_PASSWORD" POSTGRES_PASSWORD: "vault:projects/anyway/prod/k8s-secret-db:POSTGRES_PASSWORD" + db-backup: + DBDUMP_AWS_ACCESS_KEY_ID: "vault:projects/anyway/prod/aws_db_dumps_writer_user:access_key_id" + DBDUMP_AWS_SECRET_ACCESS_KEY: "vault:projects/anyway/prod/aws_db_dumps_writer_user:secret_access_key" + DBDUMP_PASSWORD: "vault:projects/anyway/prod/k8s-secret-db-backup:DBDUMP_PASSWORD" diff --git a/apps/redash/compose.yaml b/apps/redash/compose.yaml index e1f68af..78efe91 100644 --- a/apps/redash/compose.yaml +++ b/apps/redash/compose.yaml @@ -10,6 +10,7 @@ services: hostname: redash-adhoc-worker networks: [dfc] image: *x-redash-image + restart: unless-stopped command: ["worker"] environment: <<: *x-redash-common-env @@ -22,6 +23,7 @@ services: hostname: redash-scheduled-worker networks: [dfc] image: *x-redash-image + restart: unless-stopped command: ["worker"] environment: <<: *x-redash-common-env @@ -34,6 +36,7 @@ services: hostname: redash-scheduler networks: [dfc] image: *x-redash-image + restart: unless-stopped command: ["scheduler"] environment: <<: *x-redash-common-env @@ -46,6 +49,7 @@ services: hostname: redash networks: [dfc] image: *x-redash-image + restart: unless-stopped command: ["server"] environment: <<: *x-redash-common-env @@ -64,6 +68,7 @@ services: # - "traefik.http.routers.anyway-reports.tls.certresolver=dfc" # pulled Nov 1, 2021 image: redash/nginx:latest@sha256:4eaaa7af6476b0422058b0022661ad6129dfbf9065c506fb0904bbf0a16f2007 + restart: unless-stopped volumes: - ./nginx-default.conf:/etc/nginx/conf.d/default.conf:ro @@ -71,6 +76,7 @@ services: hostname: redash-postgres networks: [dfc] image: postgres:9.6-alpine@sha256:8c3455a1c953c7c4fbcedca28a0c9c665cb6175934148335b3b86df5d106b9b5 + restart: unless-stopped env_file: - ./secrets/redash.env volumes: @@ -80,6 +86,7 @@ services: hostname: redash-redis networks: [dfc] image: redis:5.0-alpine@sha256:cb5840073dc475b45bf856eb666f5e4d19e3b1ae882e0f3a1396d399cfa23702 + restart: unless-stopped networks: dfc: diff --git a/apps/selenium/compose.yaml b/apps/selenium/compose.yaml index 240f2e2..16e71e8 100644 --- a/apps/selenium/compose.yaml +++ b/apps/selenium/compose.yaml @@ -3,6 +3,7 @@ services: selenium-chrome-node: networks: [dfc] image: selenium/node-chrome:4.8.3-20230403 + restart: unless-stopped environment: SE_EVENT_BUS_HOST: "selenium-hub" SE_EVENT_BUS_PUBLISH_PORT: "4442" @@ -22,6 +23,7 @@ services: - "traefik.http.routers.selenium-chrome-node-nginx.tls=true" - "traefik.http.routers.selenium-chrome-node-nginx.tls.certresolver=dfc" image: nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94 + restart: unless-stopped volumes: - /tmp/selenium-tempdata:/var/selenium/tempdata - ./nginx-default.conf:/etc/nginx/conf.d/default.conf:ro @@ -30,6 +32,7 @@ services: selenium-hub: networks: [dfc] image: selenium/hub:4.8.3-20230403 + restart: unless-stopped labels: - "traefik.enable=true" - "traefik.http.services.selenium-hub.loadbalancer.server.port=4444" diff --git a/bin/apps.py b/bin/apps.py index a353c19..3784120 100755 --- a/bin/apps.py +++ b/bin/apps.py @@ -3,6 +3,8 @@ import sys import json import subprocess +import tempfile + from ruamel import yaml from functools import lru_cache from tempfile import TemporaryDirectory @@ -120,6 +122,16 @@ def copy_files(app_conf, app_name, stats): scp(f'apps/{app_name}/{src}', f'~/apps/.new.{app_name}/{dst}', stats) +def deploy_cronjob(service_name, service, app_name): + print(f'Deploying cronjob {service_name}') + schedule = service['x-cronjob']['schedule'] + cronline = f'{schedule} ubuntu cd ~/apps/{app_name} && docker compose run --rm {service_name}' + ssh(f''' + sudo rm -f /etc/cron.d/{app_name}-{service_name} + echo "{cronline}" | sudo tee /etc/cron.d/{app_name}-{service_name} + ''') + + def deploy_app(app_name, *args): skip_deploy = '--skip-deploy' in args assert os.path.exists(f'./apps/{app_name}/compose.yaml') @@ -142,9 +154,15 @@ def deploy_app(app_name, *args): for line in app_conf.get('x-pre-deploy', []): print('Running x-pre-deploy:', line) ssh(f'cd ~/apps/{app_name} && {line}') + start_services = set() + for service_name, service in app_conf['services'].items(): + if service.get('x-cronjob'): + deploy_cronjob(service_name, service, app_name) + else: + start_services.add(service_name) ssh(f''' cd ~/apps/{app_name} - docker compose up -d + docker compose up -d {' '.join(start_services)} ''') for on_file_change in app_conf.get('x-on-file-change', []): if not isinstance(on_file_change, dict):