From d071f9e776467bb5d4a6fdf38fe280f73ac0ef11 Mon Sep 17 00:00:00 2001
From: James Maple <jmaple@P-WS-01.jmhomelab.local>
Date: Wed, 27 Nov 2024 11:53:52 +0000
Subject: [PATCH] Push version 1.2

---
 README.md                  |  12 +-
 changelog.md               |   5 +
 conf/config.ini            |   4 +
 src/.DS_Store              | Bin 6148 -> 6148 bytes
 src/crate.py               |  55 ++++++--
 src/functions/file.py      |  20 ++-
 src/functions/ntfy.py      | 278 +++++++++++++++++++++++++++++++++++++
 src/functions/retention.py |   9 +-
 src/functions/sftp.py      |   9 +-
 9 files changed, 376 insertions(+), 16 deletions(-)
 create mode 100644 src/functions/ntfy.py

diff --git a/README.md b/README.md
index 32486be..3f32a1e 100644
--- a/README.md
+++ b/README.md
@@ -95,12 +95,20 @@ This section provides an explanation for each configurable within the config fil
 ### Retention
 | Configurable                  | Description                                                               |
 |-------------------------------|---------------------------------------------------------------------------|
-| LOCAL_RETENTION_ENABLED       | "True" or "False" Enables/Disables retentionc cleanup on the local copy   |
-| OFFSITE_RETENTION_ENABLED     | "True" or "False" Enables/Disables retentionc cleanup on the offsite copy |
+| LOCAL_RETENTION_ENABLED       | "True" or "False" Enables/Disables retention cleanup on the local copy    |
+| OFFSITE_RETENTION_ENABLED     | "True" or "False" Enables/Disables retention cleanup on the offsite copy  |
 | LOCAL_RETENTION_PERIOD_DAYS   | Store backups for X days. Older backups will be deleted.                  |
 | OFFSITE_RETENTION_PERIOD_DAYS | Store backups for X days. Older backups will be deleted.                  |
 
 
+### NTFY
+| Configurable                  | Description                                                               |
+|-------------------------------|---------------------------------------------------------------------------|
+| NTFY_ENABLED                  | "True" or "False" Enables/Disables Ntfy notifications                     |
+| NTFY_HOST                     | Ntfy server URL                                                           |
+| NTFY_TOPIC                    | Ntfy topic to subscribe to                                                |
+
+
 
 ______________________
 &nbsp;
diff --git a/changelog.md b/changelog.md
index a6e87fa..a5fb181 100644
--- a/changelog.md
+++ b/changelog.md
@@ -1,5 +1,10 @@
 ## Change log
 
+### V 1.2
+- Implement ntfy notifications
+- Added some missing error messages before exit conditions
+
+
 ### V 1.1
 - Broke changelog out into its own file
 - Moved issues from readme to git issues 
diff --git a/conf/config.ini b/conf/config.ini
index 61f7f1b..e97ea24 100644
--- a/conf/config.ini
+++ b/conf/config.ini
@@ -38,3 +38,7 @@ OFFSITE_RETENTION_PERIOD_DAYS =
 
 
 
+[NTFY]
+NTFY_ENABLED = False
+NTFY_HOST = https://ntfy.yourhost.tld
+NTFY_TOPIC = CRATE
\ No newline at end of file
diff --git a/src/.DS_Store b/src/.DS_Store
index aeb53b2e6601bff1cdf016bc61e7fb5d4f1cf7f1..5f6ccfe1e997f3818a60af865e168a60ca91abd2 100644
GIT binary patch
delta 307
zcmZoMXfc=|#>B)qu~2NHo}wr_0|Nsi1A_nqLkUABLq0<uLvd31=8KHW85!9p2QrG)
zCzTf$B<18M0oCnDD#*z!E-^5;#>m9X!pg?Z&dJWr9vhsIUmjeNSW;T-lvorE;)Uer
z=On?{iAiCZspatkBF_1FC5f4NsYPH7nJKA2B{AWdc`5njPWh#IDaByD!4L^fP7cm^
z0f}nYY6~+<9R*_(vsxX6YD*&n9R(9hquN?d4pC)&>!A4ToZP(pE}-LpfDz~sFyMvK
zFsd8mlgUbqYbJ*<3iFf&7v<&T=cNN>8G$}^VA;&h!OsB<5+LV0^JIPzOAeqxV6!#{
Ih-_g70OrF_F#rGn

delta 91
zcmZoMXfc=|#>B`mu~2NHo}wr#0|Nsi1A_nqLkUABLq0<uLvc#+#KPr_tdr}R#U}q@
sTC@2ovk}YY2h8kDo7p+|Ie=O>Uu6EyJegm_k^`t2q?>7TfXEhR0P=ek<p2Nx

diff --git a/src/crate.py b/src/crate.py
index db3d54b..dd89cbb 100644
--- a/src/crate.py
+++ b/src/crate.py
@@ -1,9 +1,9 @@
 #!/usr/bin/python3
 
 __author__ = "james_s60"
-__date__ = "02 Nov 2024"
+__date__ = "27 Nov 2024"
 __credits__ = ["james_s60"]
-__version__ = "1.1"
+__version__ = "1.2"
 
 
 
@@ -23,6 +23,7 @@
 from functions import rwcheck
 from functions import script
 from functions import sftp
+from functions import ntfy
 
 
 config = ConfigParser()
@@ -85,6 +86,10 @@
 OFFSITE_RETENTION_ENABLED = config['RETENTION'].getboolean('OFFSITE_RETENTION_ENABLED')
 LOCAL_RETENTION_PERIOD_DAYS = int(config['RETENTION']['LOCAL_RETENTION_PERIOD_DAYS'])
 OFFSITE_RETENTION_PERIOD_DAYS = int(config['RETENTION']['OFFSITE_RETENTION_PERIOD_DAYS'])
+# Ntfy
+NTFY_ENABLED = config['NTFY'].getboolean('NTFY_ENABLED')
+# NTFY_HOST = str(config['NTFY']['NTFY_HOST'])     - This is now parsed directly in src/functions/ntfy.py!
+# NTFY_TOPIC = str(config['NTFY']['NTFY_TOPIC'])   - This is now parsed directly in src/functions/ntfy.py!
 ############################
 
 
@@ -94,18 +99,28 @@
 # Sanity check arguments
 if FIX_PERMS == True and RUN_MODE not in {"recover-local", "rl"}:
     script.err_msg("Fix Permissions argument specificed but CRATE is running in recovery mode. Program will exit.\n\n")
+    if NTFY_ENABLED == True:
+        ntfy.ntfy_err_FP_wrong_mode_fail()
     exit()
 if RUN_MODE == "backup-local" or RUN_MODE == "bl" and RECOVERY_ZIP is not None:
     script.err_msg("Zip file argument specificed but CRATE is running in backup mode. Program will exit.\n\n")
+    if NTFY_ENABLED == True:
+        ntfy.ntfy_err_ZIP_wrong_mode_fail()
     exit()
 if RUN_MODE == "backup-local" or RUN_MODE == "bl" and SKIP_RM is True:
     script.err_msg("SRM argument specificed but CRATE is running in backup mode. Program will exit.\n\n")
+    if NTFY_ENABLED == True:
+        ntfy.ntfy_err_SRM_wrong_mode_fail()
     exit()
 if RUN_MODE == "backup-local" or RUN_MODE == "bl" and FIX_PERMS is True:
     script.err_msg("Fix Permissions argument specificed but CRATE is running in backup mode. Program will exit.\n\n")
+    if NTFY_ENABLED == True:
+        ntfy.ntfy_err_FP_wrong_mode_fail()
     exit()
 if RUN_MODE == "recover-local" or RUN_MODE == "rl" and RECOVERY_ZIP is None:
     script.err_msg("ZIP file not specified. Program will exit.\n\n")
+    if NTFY_ENABLED == True:
+        ntfy.ntfy_err_ZIP_missing_fail()
     exit()
 
 
@@ -117,27 +132,37 @@
 
     source_dir_readable = rwcheck.check_source_readable(SOURCE_DIR) # Will call exit() if unreadables are found
     if source_dir_readable == False:
+        script.err_msg("Source dir is not readable. Program will exit.\n\n")
+        if NTFY_ENABLED == True:
+            ntfy.ntfy_err_source_dir_unreadable_fail()
         exit()
 
     dest_dir_writable = rwcheck.check_destination_writable(DEST_DIR) # Will call exit() if unwritable
     if dest_dir_writable == False:
+        script.err_msg("Source dir is not readable. Program will exit.\n\n")
+        if NTFY_ENABLED == True:
+            ntfy.ntfy_err_dest_dir_unwriteable_fail()
         exit()
 
     
     if source_dir_readable == True and dest_dir_writable == True:
         file.create_hostname_subdirs(HOSTS, DEST_DIR, RUN_MODE)
-        zip_to_offsite = file.perform_backup_local(SOURCE_DIR, DEST_DIR) # Will call exit() if zip fails validity check
-
+        zip_to_offsite = file.perform_backup_local(SOURCE_DIR, DEST_DIR, NTFY_ENABLED)
+        # Errors & Ntfy are handled within perform_backup_local
 
 
     # Run retention on local
     if LOCAL_RETENTION_ENABLED == True:
         script.info_msg("Running retention cleanup on local copy. Retention period: " + str(LOCAL_RETENTION_PERIOD_DAYS) + " days")
-        retention.localhost_basic_retention(DEST_DIR, LOCAL_RETENTION_PERIOD_DAYS)
+        retention.localhost_basic_retention(DEST_DIR, LOCAL_RETENTION_PERIOD_DAYS, NTFY_ENABLED)
     else:
         script.info_msg("Local retention cleanup disabled, skipping retention cleanup.\n\n")
 
 
+    # Ntfy for local backup completion
+    if NTFY_ENABLED == True:
+        ntfy.ntfy_ok_local_backup_complete()
+
 
     # Run Offsite backup
     if OFFSITE_BACKUP_ENABLED == True:
@@ -145,11 +170,13 @@
 
 
         if OFFSITE_BACKUP_MODE == "local":
-            file.copy_to_offsite_dir(zip_to_offsite, LOCAL_OFFSITE_BACKUP_DIR, HOSTS)
+            file.copy_to_offsite_dir(zip_to_offsite, LOCAL_OFFSITE_BACKUP_DIR, HOSTS, NTFY_ENABLED)
+            #Ntfy handled within function
 
 
         if OFFSITE_BACKUP_MODE == "sftp":
-            sftp.copy_to_sftp_dir(zip_to_offsite, HOSTS, SFTP_USER, SFTP_PASS, SFTP_HOST, SFTP_PATH)
+            sftp.copy_to_sftp_dir(zip_to_offsite, HOSTS, SFTP_USER, SFTP_PASS, SFTP_HOST, SFTP_PATH, NTFY_ENABLED)
+            #Ntfy handled within function
 
     else:
         script.info_msg("Offsite backup disabled, skipping...")
@@ -161,10 +188,10 @@
         script.info_msg("Running retention cleanup on offsite copy. Retention period: " + str(LOCAL_RETENTION_PERIOD_DAYS) + " days")
 
         if OFFSITE_BACKUP_MODE == "local":
-            retention.localhost_basic_retention(LOCAL_OFFSITE_BACKUP_DIR, LOCAL_RETENTION_PERIOD_DAYS)
+            retention.localhost_basic_retention(LOCAL_OFFSITE_BACKUP_DIR, LOCAL_RETENTION_PERIOD_DAYS, NTFY_ENABLED)
         
         elif OFFSITE_BACKUP_MODE == "sftp":
-            retention.sftp_basic_retention(HOSTS, SFTP_USER, SFTP_PASS, SFTP_HOST, SFTP_PATH, OFFSITE_RETENTION_PERIOD_DAYS)
+            retention.sftp_basic_retention(HOSTS, SFTP_USER, SFTP_PASS, SFTP_HOST, SFTP_PATH, OFFSITE_RETENTION_PERIOD_DAYS, NTFY_ENABLED)
 
     else:
         script.info_msg("Offsite retention cleanup disabled, skipping retention cleanup.")
@@ -202,10 +229,16 @@
             recovery_status = file.perform_recovery_local(RECOVERY_ZIP, SOURCE_DIR, SKIP_RM) # Will call exit if no perms to delete SOURCE_DIR and -srm is not used
             if recovery_status == "ok":
                 script.ok_msg("Recovery successful!\n\n")
+                if NTFY_ENABLED == True:
+                    ntfy.ntfy_ok_recovery_sucess()
             if recovery_status == "badzip":
                 script.err_msg("Recovery failed due to an invalid zip archive. Try again as root or with a different archive.\n\n")
+                if NTFY_ENABLED == True:
+                    ntfy.ntfy_err_recovery_zip_error()
             if recovery_status == "error":
                 script.err_msg("Recovery failed due to an unknown error. Try again as root or with a different archive.\n\n\\s")
+                if NTFY_ENABLED == True:
+                    ntfy.ntfy_err_recovery_unknown_error()
 
             if FIX_PERMS == True:
                 perms_valid = file.sanity_check_permission_config(CONTAINER_USER, PERMISSION)
@@ -213,6 +246,8 @@
                     file.fix_recovery_permissions(SOURCE_DIR, CONTAINER_USER, PERMISSION) # Will call exit if not running as root
                 else:
                     script.err_msg("Permission configuration sanity check failed. Program will now exit...")
+                    if NTFY_ENABLED == True:
+                        ntfy.ntfy_err_recovery_unknown_error()
                     exit()
             else:
                 script.info_msg("Fix permissions was not specified - skipping... (use -fp and try again as root if this was unintentional)")
@@ -223,6 +258,8 @@
 
 else:
     script.err_msg("Invalid run mode selected.\nRun with -m [bl,rl] or --help for more.\n\n")
+    if NTFY_ENABLED == True:
+        ntfy.ntfy_err_invalid_run_mode()
     exit()
 
 
diff --git a/src/functions/file.py b/src/functions/file.py
index 8cdd020..fd34521 100644
--- a/src/functions/file.py
+++ b/src/functions/file.py
@@ -9,6 +9,8 @@
 import pwd
 import re
 from functions import script
+from functions import ntfy
+
 
 
 
@@ -44,7 +46,7 @@ def create_hostname_subdirs(HOSTS, DEST_DIR, RUN_MODE):
 
 
 
-def perform_backup_local(SOURCE_DIR, DEST_DIR):
+def perform_backup_local(SOURCE_DIR, DEST_DIR, NTFY_ENABLED):
     datetime_str = datetime.datetime.now().strftime('%Y-%m-%d---%H-%M-%S')
     hostname = socket.gethostname()
     print("\n\n")
@@ -69,12 +71,14 @@ def perform_backup_local(SOURCE_DIR, DEST_DIR):
     except zipfile.BadZipFile:
         archive = "bad"
         script.warn_msg("The backup archive is potentially corrupt: " + validate_me)
+        ntfy.ntfy_warn_ZIP_validation_fail()
     if archive == 'good':
         script.ok_msg("Backup archive passed validation check.")
         os.system("rm -rf /tmp/crate-scratch")
         script.ok_msg("Backup task complete!\n\n")
     else:
         script.err_msg("Backup archive FAILED validation check.")
+        ntfy.ntfy_err_ZIP_validation_fail()
 
     # This is the path of the created backup archive, returned so offsite def can use it
     return validate_me 
@@ -118,11 +122,13 @@ def perform_recovery_local(RECOVERY_ZIP, SOURCE_DIR, SKIP_RM):
 
 
 
-def copy_to_offsite_dir(zip_to_offsite, OFFSITE_PATH, HOSTS): 
+def copy_to_offsite_dir(zip_to_offsite, OFFSITE_PATH, HOSTS, NTFY_ENABLED): 
 
 # Check if the ZIP file exists
     if not os.path.isfile(zip_to_offsite):
         script.err_msg(f"The file {zip_to_offsite} does not exist.")
+        if NTFY_ENABLED == True:
+            ntfy.ntfy_err_offsite_fail_no_zip()
         return
     
     # Check if the destination path is a directory
@@ -139,9 +145,13 @@ def copy_to_offsite_dir(zip_to_offsite, OFFSITE_PATH, HOSTS):
             os.makedirs(hostname_path)
         except PermissionError as e:
             script.err_msg(f"Permission error while creating directory {hostname_path}: {e}")
+            if NTFY_ENABLED == True:
+                ntfy.ntfy_err_offsite_local_fail_dir_create()
             return
         except Exception as e:
             script.err_msg(f"An error occurred while creating directory {hostname_path}: {e}")
+            if NTFY_ENABLED == True:
+                ntfy.ntfy_err_offsite_local_fail_dir_create()
             return
     
     # Construct the destination path for the copied file
@@ -152,10 +162,16 @@ def copy_to_offsite_dir(zip_to_offsite, OFFSITE_PATH, HOSTS):
         # Copy the file to the destination directory
         shutil.copy(zip_to_offsite, destination)
         script.ok_msg(f"Successfully copied {zip_to_offsite} to {destination}.")
+        if NTFY_ENABLED == True:
+                ntfy.ntfy_ok_offsite_local_complete() 
     except PermissionError as e:
         script.err_msg(f"Permission error: {e}")
+        if NTFY_ENABLED == True:
+                ntfy.ntfy_err_offsite_local_fail_copy()
     except Exception as e:
         script.err_msg(f"An error occurred: {e}")
+        if NTFY_ENABLED == True:
+                ntfy.ntfy_err_offsite_local_fail_copy()
 
 
 
diff --git a/src/functions/ntfy.py b/src/functions/ntfy.py
new file mode 100644
index 0000000..edc2987
--- /dev/null
+++ b/src/functions/ntfy.py
@@ -0,0 +1,278 @@
+import requests
+import socket
+from configparser import ConfigParser 
+
+
+############################
+# Parse NTFY Config
+############################
+config = ConfigParser()
+config.read('/usr/share/ContainerCleaner/conf/config.ini')
+
+NTFY_HOST = str(config['NTFY']['NTFY_HOST'])
+NTFY_TOPIC = str(config['NTFY']['NTFY_TOPIC'])
+############################
+
+
+hostname = str(socket.gethostname())
+
+
+
+
+
+# Start of script argument sanity check errors 
+def ntfy_err_FP_wrong_mode_fail():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Run failed on " + hostname + " - Fix Permissions argument specificed but CRATE is running in recovery mode. Mode/Arg are incompatible - check run command.",
+        headers={
+            "Title": "CRATE",
+            "Priority": "urgent",
+            "Tags": "x,pensive"
+        })
+
+def ntfy_err_ZIP_wrong_mode_fail():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Run failed on " + hostname + " - Zip file argument specificed but CRATE is running in backup mode. Mode/Arg are incompatible - check run command.",
+        headers={
+            "Title": "CRATE",
+            "Priority": "urgent",
+            "Tags": "x,pensive"
+        })   
+
+def ntfy_err_SRM_wrong_mode_fail():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Run failed on " + hostname + " - Skip remove argument specificed but CRATE is running in backup mode. Mode/Arg are incompatible - check run command.",
+        headers={
+            "Title": "CRATE",
+            "Priority": "urgent",
+            "Tags": "x,pensive"
+        })       
+
+def ntfy_err_FP_wrong_mode_fail():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Run failed on " + hostname + " - Fix permissions argument specificed but CRATE is running in backup mode. Mode/Arg are incompatible - check run command.",
+        headers={
+            "Title": "CRATE",
+            "Priority": "urgent",
+            "Tags": "x,pensive"
+        })     
+
+def ntfy_err_ZIP_missing_fail():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Run failed on " + hostname + " - Zip file not specificed but CRATE is running in recovery mode. Mode/Arg are incompatible - check run command.",
+        headers={
+            "Title": "CRATE",
+            "Priority": "urgent",
+            "Tags": "x,pensive"
+        })        
+
+def ntfy_err_invalid_run_mode():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Run failed on " + hostname + " - Invalid run mode. Check run command.",
+        headers={
+            "Title": "CRATE",
+            "Priority": "urgent",
+            "Tags": "x,pensive"
+        })         
+
+
+
+
+# Backup Local mode - Local backup
+def ntfy_err_source_dir_unreadable_fail():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Run failed on " + hostname + " - Source dir is not readable.",
+        headers={
+            "Title": "CRATE",
+            "Priority": "urgent",
+            "Tags": "x,pensive"
+        })  
+
+
+def ntfy_err_dest_dir_unwriteable_fail():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Run failed on " + hostname + " - Destination dir is not readable.",
+        headers={
+            "Title": "CRATE",
+            "Priority": "urgent",
+            "Tags": "x,pensive"
+        })      
+
+
+
+def ntfy_err_ZIP_validation_fail():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data=hostname + " - Backup ZIP failed validity check. Needs manual investigation to ensure file integrity.",
+        headers={
+            "Title": "CRATE",
+            "Priority": "high",
+            "Tags": "x,pensive"
+        }) 
+
+def ntfy_warn_ZIP_validation_fail():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data=hostname + " - Backup ZIP validity check shows potential error. Needs manual investigation to ensure file integrity.",
+        headers={
+            "Title": "CRATE",
+            "Priority": "high",
+            "Tags": "exclamation,raised_eyebrow"
+        })      
+
+
+def ntfy_ok_local_backup_complete():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Local backup complete on " + hostname,
+        headers={
+            "Title": "CRATE",
+            "Priority": "default",
+            "Tags": "white_check_mark,slightly_smiling_face"
+        })       
+
+def ntfy_warn_local_retention_error():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Local backup retention error on " + hostname + " - Possible permission issue?: ",
+        headers={
+            "Title": "CRATE",
+            "Priority": "high",
+            "Tags": "exclamation,raised_eyebrow"
+        })      
+
+
+
+
+# Backup Local mode - offsite global
+def ntfy_err_offsite_fail_no_zip():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Offsite backup failed on " + hostname + " - Local source ZIP doesn't exist / not readable",
+        headers={
+            "Title": "CRATE",
+            "Priority": "high",
+            "Tags": "x,pensive"
+        })      
+
+
+
+
+# Backup Local mode - Local offsite
+def ntfy_err_offsite_local_fail_dir_create():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Local offsite backup failed on " + hostname + " - Unable to create destination directory. Permission error?",
+        headers={
+            "Title": "CRATE",
+            "Priority": "high",
+            "Tags": "x,pensive"
+        })     
+
+def ntfy_err_offsite_local_fail_copy():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Local offsite backup failed on " + hostname + " - Unable to copy zip to destination directory. Permission error/Out of storage?",
+        headers={
+            "Title": "CRATE",
+            "Priority": "high",
+            "Tags": "x,pensive"
+        })   
+
+def ntfy_ok_offsite_local_complete():
+        requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Offsite (local mode) backup complete on " + hostname,
+        headers={
+            "Title": "CRATE",
+            "Priority": "default",
+            "Tags": "white_check_mark,slightly_smiling_face"
+        })  
+
+
+
+# Backup Local mode - SFTP offsite 
+def ntfy_err_offsite_sftp_connection(e):
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="SFTP offsite backup failed on " + hostname + " - Unable to connect to SFTP server: " + e,
+        headers={
+            "Title": "CRATE",
+            "Priority": "high",
+            "Tags": "x,pensive"
+        })   
+
+def ntfy_warn_sftp_retention_error():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="SFTP offsite backup retention error on " + hostname + " - Possible SFTP permission issue?",
+        headers={
+            "Title": "CRATE",
+            "Priority": "high",
+            "Tags": "exclamation,raised_eyebrow"
+        })  
+
+def ntfy_ok_offsite_sftp_complete():
+        requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Offsite (sftp mode) backup complete on " + hostname,
+        headers={
+            "Title": "CRATE",
+            "Priority": "default",
+            "Tags": "white_check_mark,slightly_smiling_face"
+        })  
+
+
+
+# Recover Local mode
+def ntfy_ok_recovery_sucess():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Recovery complete on " + hostname,
+        headers={
+            "Title": "CRATE",
+            "Priority": "default",
+            "Tags": "white_check_mark,slightly_smiling_face"
+        })  
+
+
+def ntfy_err_recovery_zip_error():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Recovery failed on " + hostname + " - ZIP error. Try again with root or use a different archive",
+        headers={
+            "Title": "CRATE",
+            "Priority": "urgent",
+            "Tags": "x,pensive"
+        })   
+
+def ntfy_err_recovery_unknown_error():
+    requests.post(NTFY_HOST + "/" + NTFY_TOPIC,
+        data="Recovery failed on " + hostname + " - Unknown error. Try again with root?",
+        headers={
+            "Title": "CRATE",
+            "Priority": "urgent",
+            "Tags": "x,pensive"
+        })   
+
+
+
+
+
+
+# ✅🙂 - "Tags": "white_check_mark,slightly_smiling_face"
+# ❗🤨 - "Tags": "exclamation,raised_eyebrow"
+# ❌😔 - "Tags": "x,pensive"
+# https://docs.ntfy.sh/emojis/
+
+# Priorities: urgent (5), high (4), default (none) (3), low (2), min (1)
+# https://docs.ntfy.sh/publish/
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/src/functions/retention.py b/src/functions/retention.py
index 3e92fd4..f3eebb4 100644
--- a/src/functions/retention.py
+++ b/src/functions/retention.py
@@ -2,10 +2,11 @@
 import os
 import pysftp
 from functions import script
+from functions import ntfy
 
 
 
-def localhost_basic_retention(DEST_DIR, LOCAL_RETENTION_PERIOD_DAYS):
+def localhost_basic_retention(DEST_DIR, LOCAL_RETENTION_PERIOD_DAYS, NTFY_ENABLED):
     today = datetime.datetime.now()
     retention_delta = datetime.timedelta(days=LOCAL_RETENTION_PERIOD_DAYS)
 
@@ -42,13 +43,15 @@ def localhost_basic_retention(DEST_DIR, LOCAL_RETENTION_PERIOD_DAYS):
 
     if delete_errors:
         print("RETENTION DEBUG - " + str(delete_errors))
+        if NTFY_ENABLED == True:
+            ntfy.ntfy_warn_local_retention_error()
 
     script.ok_msg("Retention cleanup complete\n\n")
 
 
 
 
-def sftp_basic_retention(HOSTS, SFTP_USER, SFTP_PASS, SFTP_HOST, SFTP_PATH, OFFSITE_RETENTION_PERIOD_DAYS):
+def sftp_basic_retention(HOSTS, SFTP_USER, SFTP_PASS, SFTP_HOST, SFTP_PATH, OFFSITE_RETENTION_PERIOD_DAYS, NTFY_ENABLED):
     # Combine the SFTP path with the HOSTS variable to form the full path
     directory_path = os.path.join(SFTP_PATH, HOSTS)
 
@@ -87,6 +90,8 @@ def sftp_basic_retention(HOSTS, SFTP_USER, SFTP_PASS, SFTP_HOST, SFTP_PATH, OFFS
             print(f"Directory {directory_path} not found.")
         except Exception as e:
             print(f"An error occurred: {e}")
+            if NTFY_ENABLED == True:
+                ntfy.ntfy_warn_sftp_retention_error()
 
         script.ok_msg("Retention cleanup complete\n\n")
 
diff --git a/src/functions/sftp.py b/src/functions/sftp.py
index 4d535b3..deccd58 100644
--- a/src/functions/sftp.py
+++ b/src/functions/sftp.py
@@ -1,9 +1,10 @@
 import pysftp
 import os
 from functions import script
+from functions import ntfy
 
 
-def copy_to_sftp_dir(zip_to_offsite, HOSTS, SFTP_USER, SFTP_PASS, SFTP_HOST, SFTP_PATH):
+def copy_to_sftp_dir(zip_to_offsite, HOSTS, SFTP_USER, SFTP_PASS, SFTP_HOST, SFTP_PATH, NTFY_ENABLED):
     # Construct SFTP destination path
     sftp_path_host_added = str(SFTP_PATH + "/" + HOSTS)
 
@@ -11,6 +12,8 @@ def copy_to_sftp_dir(zip_to_offsite, HOSTS, SFTP_USER, SFTP_PASS, SFTP_HOST, SFT
     # Check if the source ZIP file exists
     if not os.path.isfile(zip_to_offsite):
         script.err_msg(f"The file {zip_to_offsite} does not exist.")
+        if NTFY_ENABLED == True:
+            ntfy.ntfy_err_offsite_fail_no_zip()
         return
 
 
@@ -29,10 +32,14 @@ def copy_to_sftp_dir(zip_to_offsite, HOSTS, SFTP_USER, SFTP_PASS, SFTP_HOST, SFT
             sftp.put(zip_to_offsite)
 
             script.ok_msg("Offsite backup zip successfully uploaded to the SFTP server.\n\n")
+            if NTFY_ENABLED == True:
+                ntfy.ntfy_ok_offsite_sftp_complete()
     
 
     except Exception as e:
         script.err_msg("An error occured, unable to backup to the SFTP server:\n")
         print(e)
+        if NTFY_ENABLED == True:
+            ntfy.ntfy_err_offsite_sftp_connection(e)