Merge pull request #243 from Tecnativa/imp-multi_dest

[ADD] postgres-multi

.github/workflows/ci.yml

@@ -55,6 +55,7 @@ jobs:
       - run: pip install poetry
       - name: Patch $PATH
         run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+      - run: poetry run python -m pip install --upgrade pip
       - run: poetry install
       # Run tests
       - run: poetry run pytest --prebuild
@@ -76,6 +77,7 @@ jobs:
           - docker-s3
           - postgres
           - postgres-s3
+          - postgres-multi
           - s3
     steps:
       # Set up Docker Environment

Dockerfile

@@ -117,3 +117,9 @@ FROM postgres AS postgres-s3
 ENV JOB_500_WHAT='dup full $SRC $DST' \
     JOB_500_WHEN='weekly' \
     OPTIONS_EXTRA='--metadata-sync-mode partial --full-if-older-than 1W --file-prefix-archive archive-$(hostname -f)- --file-prefix-manifest manifest-$(hostname -f)- --file-prefix-signature signature-$(hostname -f)- --s3-european-buckets --s3-multipart-chunk-size 10 --s3-use-new-style'
+
+
+FROM postgres-s3 AS postgres-multi
+ENV DST='multi' \
+    OPTIONS_EXTRA='--metadata-sync-mode partial --full-if-older-than 1W --file-prefix-archive archive-$(hostname -f)- --file-prefix-manifest manifest-$(hostname -f)- --file-prefix-signature signature-$(hostname -f)-' \
+    OPTIONS_EXTRA_S3='--s3-european-buckets --s3-multipart-chunk-size 10 --s3-use-new-style'

README.md

@@ -42,6 +42,7 @@
   - [PostgreSQL (`docker-duplicity-postgres`)](#postgresql-docker-duplicity-postgres)
   - [Docker (`docker-duplicity-docker`)](#docker-docker-duplicity-docker)
   - [Amazon S3 (`*-s3`)](#amazon-s3--s3)
+  - [Multi (`*-multi`)](#multi--multi)
 - [Development](#development)
   - [Testing](#testing)
   - [Managing packages](#managing-packages)
@@ -419,6 +420,34 @@ JOB_500_WHEN=weekly
 
 Note, that for `DST` variable you should use `boto3+s3://bucket_name[/prefix]` style.
 
+### Multi (`*-multi`)
+
+At the moment only "postgres" has this flavor. It extends from 'postgres-s3' and
+provides some defaults to make good use of "DST\_{N}" env. variables. and uses the extra
+options according to destination.
+
+In this mode the `$DST` is set to `multi`. This enables the use of `$DST_{N}` and
+`$DST_{N}_{ENV_VAR_NAME}`.
+
+`$DST_{N}_{ENV_VAR_NAME}` will be process as `${ENV_VAR_NAME}`.
+
+For example:
+
+```yaml
+  backup:
+    ...
+    environment:
+      ...
+      DST_1: scp://[email protected]//usr/backup
+      DST_2: boto3+s3://mybucket/myfolder
+      DST_2_AWS_ACCESS_KEY_ID: example amazon s3 access key
+      DST_2_AWS_SECRET_ACCESS_KEY: example amazon s3 secret key
+      DST_3: rsync://[email protected]:8022//volume1/folder/
+      DST_3_RSYNC_PASSWORD: the password to use with rsync
+```
+
+The `restore` process uses the first destination defined.
+
 [alpine]: https://alpinelinux.org/
 [dockerfile]: https://github.com/Tecnativa/docker-duplicity/blob/master/Dockerfile
 [duplicity]: http://duplicity.nongnu.org/

bin/dup

@@ -2,21 +2,114 @@
 from __future__ import print_function
 
 import os
+import re
 import subprocess
 import sys
 
-# Expand shell commands from environment options
-options = subprocess.check_output(
-    "echo -n " + os.environ["OPTIONS"], shell=True, stderr=subprocess.STDOUT, text=True
-)
-options_extra = subprocess.check_output(
-    "echo -n " + os.environ["OPTIONS_EXTRA"],
-    shell=True,
-    stderr=subprocess.STDOUT,
-    text=True,
-)
-
-# Execute via duplicity
-command = "duplicity {} {} {}".format(options, options_extra, " ".join(sys.argv[1:]))
-print("Executing: {}".format(command))
-os.execlp("sh", "sh", "-c", command)
+BCK_BIN = "duplicity"
+DEST_MAX_LIMIT = 99
+
+
+def _set_env(index):
+    uri = os.environ.get(f"DST_{index}")
+    if not uri:
+        return (None, None)
+    # Set the valid env. vars
+    target_environ_raw_names = tuple(
+        filter(lambda x: x.startswith(f"DST_{index}_"), os.environ.keys())
+    )
+    for target_env_raw_name in target_environ_raw_names:
+        env_value = os.getenv(target_env_raw_name)
+        target_env_name = re.findall(r"^DEST_\d+_(\w+)$", target_env_raw_name)[0]
+        os.putenv(target_env_name, env_value)
+
+    # Expand shell commands from environment options
+    # this resolves environment variables used in the string
+    options = subprocess.check_output(
+        "echo -n " + os.environ.get("OPTIONS", ""),
+        shell=True,
+        stderr=subprocess.STDOUT,
+        text=True,
+    )
+    options_extra = subprocess.check_output(
+        "echo -n " + os.environ.get("OPTIONS_EXTRA", ""),
+        shell=True,
+        stderr=subprocess.STDOUT,
+        text=True,
+    )
+    options_extra_s3 = subprocess.check_output(
+        "echo -n " + os.environ.get("OPTIONS_EXTRA_S3", ""),
+        shell=True,
+        stderr=subprocess.STDOUT,
+        text=True,
+    )
+
+    # Set valid duplicity params
+    cparams = ""
+    if uri.startswith("boto"):
+        cparams = f"{options} {options_extra} {options_extra_s3}"
+    else:
+        cparams = f"{options} {options_extra}"
+    return (uri, cparams)
+
+
+def _unset_env(index):
+    # Set the valid env. vars
+    target_environ_raw_names = tuple(
+        filter(lambda x: x.startswith(f"DST_{index}_"), os.environ.keys())
+    )
+    for target_env_raw_name in target_environ_raw_names:
+        target_env_name = re.findall(r"^DEST_\d+_(\w+)$", target_env_raw_name)[0]
+        os.unsetenv(target_env_name)
+
+
+is_multi_mode = len(sys.argv) > 2 and sys.argv[2] == "multi" or False
+if is_multi_mode:
+    print("Multi-destination mode enabled")
+    is_restore = sys.argv[1] == "restore"
+    # In this mode we ignore args and use environment variables directly
+    # Only use the $SRC arg
+    if is_restore:
+        # Its a restore operation
+        SRC = sys.argv[3]
+        EXTRA = " ".join(sys.argv[4:])
+        # By default we use the first DST as the main storage from where
+        # restore data
+        dest, command_params = _set_env(1)
+        if dest:
+            command = f"{BCK_BIN} {command_params} {dest} {SRC} {EXTRA}"
+            print(f"Executing: {command}")
+            subprocess.check_call(command, shell=True)
+        _unset_env(1)
+    else:
+        # Its a backup operation
+        SRC = sys.argv[1]
+        EXTRA = " ".join(sys.argv[3:])
+        for i in range(1, DEST_MAX_LIMIT):
+            dest, command_params = _set_env(i)
+            if not dest:
+                # DEST must be consecutive
+                break
+            command = f"{BCK_BIN} {command_params} {SRC} {dest} {EXTRA}"
+            print(f"Executing #{i}: {command}")
+            subprocess.check_call(command, shell=True)
+            _unset_env(i)
+else:
+    print("Single-destination mode enabled")
+    # Expand shell commands from environment options
+    # this resolves environment variables used in the string
+    options = subprocess.check_output(
+        "echo -n " + os.environ.get("OPTIONS", ""),
+        shell=True,
+        stderr=subprocess.STDOUT,
+        text=True,
+    )
+    options_extra = subprocess.check_output(
+        "echo -n " + os.environ.get("OPTIONS_EXTRA", ""),
+        shell=True,
+        stderr=subprocess.STDOUT,
+        text=True,
+    )
+    command = f"{BCK_BIN} {options} {options_extra} {' '.join(sys.argv[1:])}"
+    print(f"Executing: {command}")
+    subprocess.check_call(command, shell=True)

bin/jobrunner

@@ -8,6 +8,7 @@ import re
 import smtplib
 import sys
 from datetime import datetime
+from email.mime.text import MIMEText
 from email.utils import formatdate
 from os import environ, path
 from socket import getfqdn
@@ -17,6 +18,7 @@ from subprocess import STDOUT, CalledProcessError, check_output
 logging.basicConfig(level=logging.INFO)
 logging.root.name = "jobrunner"
 
+
 # Get expected periodicity from this script's placement
 periodicity = path.basename(path.dirname(path.abspath(__file__)))
 logging.info("%s UTC - Running %s jobs", datetime.utcnow(), periodicity)
@@ -39,17 +41,13 @@ for key, when in environ.items():
         njob = int(match.group(1))
         to_run[njob] = environ["JOB_{}_WHAT".format(njob)]
 
+
 if not to_run:
     logging.info("Nothing to do")
     sys.exit()
 
 # Run commands in order
-message = [
-    "From: {}".format(from_),
-    "To: {}".format(to),
-    "Date: {}".format(formatdate()),
-    "",
-]
+message = []
 failed = False
 for njob, command in sorted(to_run.items()):
     expanded_command = Template(command).safe_substitute(environ)
@@ -81,16 +79,6 @@ for njob, command in sorted(to_run.items()):
 # Report results
 if all((smtp_host, smtp_port, from_, to, subject)):
     logging.info("Sending email report")
-    message.insert(
-        0,
-        "Subject: {}".format(
-            subject.format(
-                hostname=getfqdn(),
-                periodicity=periodicity,
-                result="ERROR" if failed else "OK",
-            )
-        ),
-    )
     smtp = None
     try:
         if smtp_tls:
@@ -104,12 +92,16 @@ if all((smtp_host, smtp_port, from_, to, subject)):
                 smtp.starttls()
                 smtp.ehlo()  # re-identify ourselves over TLS connection
             smtp.login(smtp_user, smtp_pass)
-        # if we have commas at "to" then multiple recipients are defined
-        # "sendmail" accepts a list as "to" parameter, so split the variable
-        # and send it. Just to be careful, delete any whitespace present at
-        # destination email addresses
-        to_addrs = to.replace(" ", "").split(",")
-        smtp.sendmail(from_, to_addrs, "\r\n".join(message))
+        message_text_obj = MIMEText("\r\n".join(message), "plain", "utf-8")
+        message_text_obj["Subject"] = subject.format(
+            hostname=getfqdn(),
+            periodicity=periodicity,
+            result="ERROR" if failed else "OK",
+        )
+        message_text_obj["From"] = from_
+        message_text_obj["To"] = to
+        message_text_obj["Date"] = formatdate()
+        smtp.send_message(message_text_obj)
     except Exception:
         logging.exception("Failed sending email")
     finally: