Skip to content

Commit

Permalink
Merge pull request #19 from gencorefacility/del-old-files
Browse files Browse the repository at this point in the history
delete old files & add DDL in email
  • Loading branch information
genericdata authored May 30, 2024
2 parents c620a21 + d07610f commit b187af5
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 3 deletions.
66 changes: 66 additions & 0 deletions cleanup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
import os
import logging
from datetime import datetime, timedelta

# Define the path for the log file
primary_log_directory = "/var/log/reform"

if not os.path.exists(primary_log_directory):
local_log_dirs = "./cleanuplog" # If not, use the local log directory
if not os.path.exists(local_log_dirs):
os.makedirs(local_log_dirs)
log_file_path = os.path.join(local_log_dirs, "cleanup.log")
else:
log_file_path = os.path.join(primary_log_directory, "cleanup.log") # create cleanup.log in /var/log/reform

# Configure logging
logging.basicConfig(
level=logging.INFO, # INFO, WARNING, ERROR, and CRITICAL
format='%(asctime)s - %(levelname)s - %(message)s',
handlers=[
logging.FileHandler(log_file_path),
logging.StreamHandler()
]
)

# Define the path to the downloads directory
download_folder = "./downloads"
# cutoff time = 72 hours ago
cutoff = datetime.now() - timedelta(hours=72)

# Traverse the download folder
# directory path, a list of sub-directories inside the current directory, and filenames
for root, dirs, files in os.walk(download_folder):
for dir_name in dirs:
dirpath = os.path.join(root, dir_name)
# Get modification time
dir_modified_time = datetime.fromtimestamp(os.path.getmtime(dirpath))
# Check if the directory is older than 72 hours
if dir_modified_time < cutoff:
try:
# Attempt to remove the directory
os.rmdir(dirpath)
logging.info(f"Removed old directory: {dirpath}") # log: remove folders
except OSError as e:
# If directory not empty, remove files inside first
for root_dir, sub_dirs, sub_files in os.walk(dirpath, topdown=False):
for name in sub_files:
filepath = os.path.join(root_dir, name)
try:
os.remove(filepath)
logging.info(f"Removed file: {filepath}") # log: remove files
except Exception as e:
logging.error(f"Error removing file {filepath}: {e}")
for name in sub_dirs:
sub_dirpath = os.path.join(root_dir, name)
try:
os.rmdir(sub_dirpath)
logging.info(f"Removed directory: {sub_dirpath}")
except Exception as e:
logging.error(f"Error removing directory {sub_dirpath}: {e}")
# Finally, remove the main directory
try:
os.rmdir(dirpath)
logging.info(f"Removed old directory: {dirpath}")
except Exception as e:
logging.error(f"Error removing directory {dirpath}: {e}")
15 changes: 12 additions & 3 deletions job.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import wget
from flask import request, flash, Flask
from flask_mail import Message, Mail
from datetime import datetime, timedelta
from werkzeug.utils import secure_filename

from forms import ALLOWED_EXTENSIONS
Expand Down Expand Up @@ -173,10 +174,18 @@ def runReform(target_dir, ref_fasta, ref_gff, timestamp, position, chrom, in_fas


def send_email(email, timestamp):
# calculate 72h DDL
deadline = datetime.now() + timedelta(hours=72)
deadline_str = deadline.strftime('%Y-%m-%d %H:%M:%S')

with j.app_context():
msg = Message('reform results', sender='[email protected]', recipients=[email])
msg.html = "reform job complete. <a href='https://reform.bio.nyu.edu/download/" + timestamp \
+ "'> Click here to download results.</a> "
subject = f"Reform Results - Download Deadline: {deadline_str}"
msg = Message(subject, sender='[email protected]', recipients=[email])
msg.html = f"""Reform job complete.
<a href='https://reform.bio.nyu.edu/download/{timestamp}'>Click here to download results</a>.
The file will be available for the next 72 hours.
The deadline to download the file is {deadline_str}.
If you do not download the file before this time, it will be deleted."""
mail.send(msg)


Expand Down
4 changes: 4 additions & 0 deletions run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,10 @@ mkdir -p ./downloads/$timestamp
echo "tar cf - ./results/$timestamp/ | pigz > ./downloads/$timestamp/reformed.tar.gz"
tar cf - ./results/$timestamp/ | pigz > ./downloads/$timestamp/reformed.tar.gz

# remove results folder
echo "rm -Rf ./results/$timestamp"
rm -Rf ./results/$timestamp/

echo "########################################"
echo "[$(date "+%D %T")] END $timestamp"
echo "########################################"

0 comments on commit b187af5

Please sign in to comment.