diff --git a/cleanup.py b/cleanup.py new file mode 100644 index 0000000..fc92fac --- /dev/null +++ b/cleanup.py @@ -0,0 +1,66 @@ +import os +import logging +from datetime import datetime, timedelta + +# Define the path for the log file +primary_log_directory = "/var/log/reform" + +if not os.path.exists(primary_log_directory): + local_log_dirs = "./cleanuplog" # If not, use the local log directory + if not os.path.exists(local_log_dirs): + os.makedirs(local_log_dirs) + log_file_path = os.path.join(local_log_dirs, "cleanup.log") +else: + log_file_path = os.path.join(primary_log_directory, "cleanup.log") # create cleanup.log in /var/log/reform + +# Configure logging +logging.basicConfig( + level=logging.INFO, # INFO, WARNING, ERROR, and CRITICAL + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.FileHandler(log_file_path), + logging.StreamHandler() + ] +) + +# Define the path to the downloads directory +download_folder = "./downloads" +# cutoff time = 72 hours ago +cutoff = datetime.now() - timedelta(hours=72) + +# Traverse the download folder +# directory path, a list of sub-directories inside the current directory, and filenames +for root, dirs, files in os.walk(download_folder): + for dir_name in dirs: + dirpath = os.path.join(root, dir_name) + # Get modification time + dir_modified_time = datetime.fromtimestamp(os.path.getmtime(dirpath)) + # Check if the directory is older than 72 hours + if dir_modified_time < cutoff: + try: + # Attempt to remove the directory + os.rmdir(dirpath) + logging.info(f"Removed old directory: {dirpath}") # log: remove folders + except OSError as e: + # If directory not empty, remove files inside first + for root_dir, sub_dirs, sub_files in os.walk(dirpath, topdown=False): + for name in sub_files: + filepath = os.path.join(root_dir, name) + try: + os.remove(filepath) + logging.info(f"Removed file: {filepath}") # log: remove files + except Exception as e: + logging.error(f"Error removing file {filepath}: {e}") + for name in sub_dirs: + sub_dirpath = os.path.join(root_dir, name) + try: + os.rmdir(sub_dirpath) + logging.info(f"Removed directory: {sub_dirpath}") + except Exception as e: + logging.error(f"Error removing directory {sub_dirpath}: {e}") + # Finally, remove the main directory + try: + os.rmdir(dirpath) + logging.info(f"Removed old directory: {dirpath}") + except Exception as e: + logging.error(f"Error removing directory {dirpath}: {e}") \ No newline at end of file diff --git a/job.py b/job.py index 8dde77a..7739b8f 100644 --- a/job.py +++ b/job.py @@ -5,6 +5,7 @@ import wget from flask import request, flash, Flask from flask_mail import Message, Mail +from datetime import datetime, timedelta from werkzeug.utils import secure_filename from forms import ALLOWED_EXTENSIONS @@ -173,10 +174,18 @@ def runReform(target_dir, ref_fasta, ref_gff, timestamp, position, chrom, in_fas def send_email(email, timestamp): + # calculate 72h DDL + deadline = datetime.now() + timedelta(hours=72) + deadline_str = deadline.strftime('%Y-%m-%d %H:%M:%S') + with j.app_context(): - msg = Message('reform results', sender='reform@nyu.edu', recipients=[email]) - msg.html = "reform job complete. Click here to download results. " + subject = f"Reform Results - Download Deadline: {deadline_str}" + msg = Message(subject, sender='reform@nyu.edu', recipients=[email]) + msg.html = f"""Reform job complete. + Click here to download results. + The file will be available for the next 72 hours. + The deadline to download the file is {deadline_str}. + If you do not download the file before this time, it will be deleted.""" mail.send(msg) diff --git a/run.sh b/run.sh index e91dcfb..32e52df 100644 --- a/run.sh +++ b/run.sh @@ -105,6 +105,10 @@ mkdir -p ./downloads/$timestamp echo "tar cf - ./results/$timestamp/ | pigz > ./downloads/$timestamp/reformed.tar.gz" tar cf - ./results/$timestamp/ | pigz > ./downloads/$timestamp/reformed.tar.gz +# remove results folder +echo "rm -Rf ./results/$timestamp" +rm -Rf ./results/$timestamp/ + echo "########################################" echo "[$(date "+%D %T")] END $timestamp" echo "########################################"