feat: Add s3crypto remote as a S3 with E2E encryption

.gitignore

@@ -4,3 +4,4 @@ __pycache__
 .venv
 .pytest_cache
 tests-env/data
+bin/rclone

.pylintrc

@@ -1,2 +1,2 @@
 [MASTER]
-disable=missing-function-docstring,missing-class-docstring
+disable=missing-function-docstring,missing-class-docstring,too-many-arguments

Makefile

@@ -7,3 +7,10 @@ install:
 
 test:
 	PYTHONPATH=. poetry run pytest -s .
+
+bin/rclone:
+	rm -rf /tmp/rclone
+	mkdir -p /tmp/rclone
+	wget https://downloads.rclone.org/v1.67.0/rclone-v1.67.0-linux-amd64.zip -O /tmp/rclone/rclone.zip
+	cd /tmp/rclone && unzip rclone.zip
+	mv /tmp/rclone/*/rclone ./bin/rclone

README.md

@@ -34,3 +34,9 @@ python3 -m rbackup.upload --remote-type s3 -r '{"endpoint": "http://127.0.0.1:90
 # then rotate files
 python3 -m rbackup.rotate -m 2 --remote-type s3 -r '{"endpoint": "http://127.0.0.1:9000", "access_key_id": "anarchism", "secret_key_id": "anarchism", "bucket_name": "test", "base_dir": "backups"}' -p 'backup-(.*)
 ```
+
+### Example with End-To-End (E2E) encrypted S3 remote (using rclone)
+
+```bash
+python3 -m rbackup --remote-type s3crypto -r '{"endpoint": "http://127.0.0.1:9000", "enc_password": "test123test123", "enc_salt_password": "testtesttesttesttest", "base_dir": "", "access_key_id": "anarchism", "secret_key": "anarchism", "bucket_name": "test"}' -m 4 ./README.md --pattern '(.*)\.md'
+```

rbackup/app.py

@@ -1,20 +1,21 @@
 from argparse import ArgumentParser
-from logging import basicConfig, INFO
+from logging import basicConfig, INFO, DEBUG
 from .rotate.app import add_args as add_rotate_args, run_from_args as run_rotate_from_args
 from .upload.app import add_args as add_upload_args, run_from_args as run_upload_from_args
 from .args import add_common_args
 
 
 def main():
     """ __main__ """
-    basicConfig(level=INFO, format='%(asctime)s - %(levelname)s :: %(message)s')
-
     parser = ArgumentParser("rbackup-rotate")
     add_common_args(parser)
     add_rotate_args(parser)
     add_upload_args(parser)
 
     args = vars(parser.parse_args())
+    basicConfig(level=DEBUG if args['debug'] else INFO,
+                format='%(asctime)s - %(levelname)s :: %(message)s')
+
     run_rotate_from_args(args)
     run_upload_from_args(args)
     run_rotate_from_args(args)

rbackup/args.py

@@ -12,6 +12,8 @@ def add_common_args(parser: ArgumentParser):
     Adds common arguments into an ArgumentParser instance
     """
 
+    parser.add_argument("--debug", help="Increase verbosity level", action="store_true")
+
     parser.add_argument(
         "--remote-type", 
         help="Remote type: s3, local", 

rbackup/filesystem.py

@@ -3,7 +3,8 @@
 import re
 import time
 import subprocess
-from logging import info
+from tempfile import TemporaryDirectory
+from logging import info, debug
 from abc import ABC
 from dataclasses import dataclass
 from datetime import datetime
@@ -129,28 +130,135 @@ def delete(self, remote_name: str):
         self.client.delete_object(Bucket=self.bucket_name, Key=self.base_dir + "/" + remote_name)
 
 
+class S3Crypto(Filesystem):
+    """
+    S3 using rclone with encryption
+    """
+
+    password: str
+    salt_password: str
+    endpoint_url: str
+    access_key_id: str
+    secret_key: str
+    bucket_name: str
+    base_dir: str
+
+    def __init__(self, endpoint_url: str, access_key_id: str, secret_key: str,
+                 password: str, salt_password: str, bucket_name: str, base_dir: str):
+
+        self.endpoint_url = endpoint_url
+        self.access_key_id = access_key_id
+        self.secret_key = secret_key
+        self.password = password
+        self.salt_password = salt_password
+        self.bucket_name = bucket_name
+        self.base_dir = base_dir
+
+    def list_files(self) -> List[File]:
+        """
+        Lists files from the remote using `rclone lsjson`
+        """
+
+        listing_output = self._rclone(["lsjson", self._format_url()])  # todo: crypto
+        as_json = json.loads(listing_output)
+        collected: List[File] = []
+
+        for node in as_json:
+            if node["IsDir"]:
+                continue
+            collected.append(File(node["Name"], datetime.fromisoformat(node["ModTime"])))
+
+        return collected
+
+    def upload(self, local_path: str, remote_name: str = ''):
+        """
+        Uploads a file using `rclone copyto`
+        """
+        self._rclone(["copyto", local_path, f"{self._format_url()}/{remote_name}"])
+
+    def delete(self, remote_name: str):
+        """
+        Deletes a file using `rclone delete`
+        """
+
+        self._rclone(["delete", f"{self._format_url()}/{remote_name}"])
+
+    def _format_url(self) -> str:
+        return f"crypto:{self.bucket_name}/{self.base_dir}".strip("/ ")
+
+    def _rclone(self, command: List[str]) -> str:
+        debug(f"rclone {command}")
+
+        with TemporaryDirectory() as temp_dir:
+            assert temp_dir.startswith("/tmp")
+            self.generate_config(temp_dir + "/rclone.conf")
+            try:
+                out = subprocess.check_output(["rclone", "--config", temp_dir + "/rclone.conf"] + command)\
+                      .decode('utf-8')
+            finally:
+                subprocess.check_call(["rm", "-rf", temp_dir])
+        return out
+
+    def _obscure(self, password: str) -> str:
+        return subprocess.check_output(f"echo '{password}' | rclone obscure -", shell=True)\
+               .decode('utf-8')
+
+    def generate_config(self, path: str):
+        with open(path, "w", encoding="utf-8") as config_handle:
+            config = f"""
+[s3]
+type = s3
+provider = Other
+access_key_id = {self.access_key_id}
+secret_access_key = {self.secret_key}
+endpoint = {self.endpoint_url}
+
+[crypto]
+type = crypt
+remote = s3:
+directory_name_encryption = false
+password = {self._obscure(self.password)}
+password2 = {self._obscure(self.salt_password)}
+        """
+            config_handle.write(config)
+            debug(config)
+
+
 def create_fs(fs_type: str, remote_string: str) -> Filesystem:
     """
     Factory method
     """
 
     if fs_type == "local":
-        info("Creating local filesystem")
+        info("Usin local filesystem")
         data = json.loads(remote_string)
         return Local(from_dict_or_env(data, "path", "RBACKUP_PATH", None))
 
     if fs_type == "s3":
-        info("Creating S3 type filesystem")
+        info("Using S3 type filesystem")
         data = json.loads(remote_string)
         return S3(
             endpoint_url=from_dict_or_env(data, "endpoint", "RBACKUP_ENDPOINT", None),
-            access_key_id=from_dict_or_env(data, "access_key", "RBACKUP_ACCESS_KEY", None),
-            secret_access_key=from_dict_or_env(data, "secret_key_id", "RBACKUP_SECRET_KEY_ID", 
+            access_key_id=from_dict_or_env(data, "access_key_id", "RBACKUP_ACCESS_KEY", None),
+            secret_access_key=from_dict_or_env(data, "secret_key", "RBACKUP_SECRET_KEY_ID", 
             None),
             bucket_name=from_dict_or_env(data, "bucket_name", "RBACKUP_BUCKET_NAME", None),
             base_dir=from_dict_or_env(data, "base_dir", "RBACKUP_BASE_DIR", None),
             retries=int(from_dict_or_env(data, "retries", "RBACKUP_RETRIES", 20))
         )
+
+    if fs_type == "s3crypto":
+        info("Using S3 with Crypto type filesystem (with rclone)")
+        data = json.loads(remote_string)
+        return S3Crypto(
+            endpoint_url=from_dict_or_env(data, "endpoint", "RBACKUP_ENDPOINT", None),
+            access_key_id=from_dict_or_env(data, "access_key_id", "RBACKUP_ACCESS_KEY", None),
+            secret_key=from_dict_or_env(data, "secret_key", "RBACKUP_SECRET_KEY_ID", None),
+            bucket_name=from_dict_or_env(data, "bucket_name", "RBACKUP_BUCKET_NAME", None),
+            base_dir=from_dict_or_env(data, "base_dir", "RBACKUP_BASE_DIR", None),
+            password=from_dict_or_env(data, "enc_password", "RBACKUP_ENC_PASSWORD", None),
+            salt_password=from_dict_or_env(data, "enc_salt_password", "RBACKUP_ENC_SALT_PASSWORD", None),
+        )
 
     raise Exception(f'Unknown filesystem type "{fs_type}"')
 

rbackup/rotate/app.py

@@ -58,10 +58,11 @@ def run_from_args(args: dict):
 
 def main():
     """ __main """
-    basicConfig(level=INFO, format='%(asctime)s - %(levelname)s :: %(message)s')
-
     parser = ArgumentParser("rbackup-rotate")
     add_args(parser)
     add_common_args(parser)
     args = vars(parser.parse_args())
+    basicConfig(level=DEBUG if args['debug'] else INFO,
+                format='%(asctime)s - %(levelname)s :: %(message)s')
+
     run_from_args(args)

rbackup/upload/app.py

@@ -66,12 +66,12 @@ def main():
     """
     __main__
     """
-
-    basicConfig(level=INFO, format='%(asctime)s - %(levelname)s :: %(message)s')
-
     parser = ArgumentParser("rbackup.upload")
     add_common_args(parser)
     add_args(parser)
 
     args = vars(parser.parse_args())
+    basicConfig(level=DEBUG if args['debug'] else INFO,
+                format='%(asctime)s - %(levelname)s :: %(message)s')
+
     run_from_args(args)

tests-env/Makefile

@@ -0,0 +1,2 @@
+test-minio-enc:
+	cd .. && export PATH=$$PATH:./bin && ./bin/rbackup --remote-type s3crypto -r '{"endpoint": "http://127.0.0.1:9000", "enc_password": "test123test123", "enc_salt_password": "testtesttesttesttest", "base_dir": "", "access_key_id": "anarchism", "secret_key": "anarchism", "bucket_name": "test"}' -m 4 ./README.md --pattern '(.*)\.md'