diff --git a/assets/mainwindow.ui b/assets/mainwindow.ui index cdfc422..910a38b 100644 --- a/assets/mainwindow.ui +++ b/assets/mainwindow.ui @@ -7,7 +7,7 @@ 0 0 990 - 630 + 794 @@ -714,7 +714,7 @@ 840 - 590 + 760 141 32 @@ -792,6 +792,9 @@ 0 + + 5000 + @@ -825,6 +828,9 @@ 0 + + 5000 + @@ -874,6 +880,9 @@ 0 + + 5000 + @@ -897,6 +906,117 @@ + + + + 10 + 590 + 971 + 171 + + + + + + + + + 10 + 10 + 291 + 16 + + + + + 10 + + + + Safelisted File Type Extensions + + + + + + 10 + 40 + 951 + 116 + + + + + + + Raw Videos + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + + + + 5000 + + + + + + + 5000 + + + + + + + Regular Images + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + + + + 5000 + + + + + + + Regular Videos + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + + + + Raw Images + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + + + + + 5000 + + + + + + diff --git a/pie/core/index_db.py b/pie/core/index_db.py index 4128cfc..2f80dc8 100644 --- a/pie/core/index_db.py +++ b/pie/core/index_db.py @@ -2,7 +2,6 @@ import logging import os from logging import Logger -from types import SimpleNamespace from typing import Dict, List from sqlalchemy import create_engine @@ -69,7 +68,6 @@ def get_all_media_files_by_path(self) -> Dict[str, MediaFile]: def get_settings(self): settings_path = MiscUtils.get_settings_path() - save_record: bool = False settings: Settings = None if os.path.exists(settings_path) and os.path.isfile(settings_path): @@ -78,83 +76,13 @@ def get_settings(self): settings_dict = json.load(file) settings = Settings() for key in settings_dict: - settings.__dict__[key] = settings_dict[key] + if key in settings.__dict__: # Don't keep stale keys + settings.__dict__[key] = settings_dict[key] except: logging.exception("Failed to load settings from JSON file. Restoring defaults.") if settings is None: settings = Settings() - save_record = True - - # Apply defaults if they are not already set - if settings.dirs_to_exclude is None: - settings.dirs_to_exclude = '[]' - save_record = True - if settings.output_dir_path_type is None: - settings.output_dir_path_type = "Use Original Paths" - save_record = True - if settings.unknown_output_dir_path_type is None: - settings.unknown_output_dir_path_type = "Use Original Paths" - save_record = True - if settings.skip_same_name_video is None: - settings.skip_same_name_video = True - save_record = True - if settings.skip_same_name_raw is None: - settings.skip_same_name_raw = True - save_record = True - if settings.convert_unknown is None: - settings.convert_unknown = False - save_record = True - if settings.overwrite_output_files is None: - settings.overwrite_output_files = False - save_record = True - if settings.indexing_workers is None: - settings.indexing_workers = MiscUtils.get_default_worker_count() - save_record = True - if settings.conversion_workers is None: - settings.conversion_workers = MiscUtils.get_default_worker_count() - save_record = True - if settings.gpu_workers is None: - settings.gpu_workers = 1 - save_record = True - if settings.gpu_count is None: - settings.gpu_count = 0 - save_record = True - if settings.image_compression_quality is None: - settings.image_compression_quality = 75 - save_record = True - if settings.image_max_dimension is None: - settings.image_max_dimension = 1920 - save_record = True - if settings.video_max_dimension is None: - settings.video_max_dimension = 1920 - save_record = True - if settings.video_crf is None: - settings.video_crf = 28 - save_record = True - if settings.video_nvenc_preset is None: - settings.video_nvenc_preset = "fast" - save_record = True - if settings.video_audio_bitrate is None: - settings.video_audio_bitrate = 128 - save_record = True - if settings.path_ffmpeg is None: - settings.path_ffmpeg = "/usr/local/bin/ffmpeg" if not MiscUtils.is_platform_win() else "ffmpeg" - save_record = True - if settings.path_magick is None: - settings.path_magick = "/usr/local/bin/magick" if not MiscUtils.is_platform_win() else "magick" - save_record = True - if settings.path_exiftool is None: - settings.path_exiftool = "/usr/local/bin/exiftool" if not MiscUtils.is_platform_win() else "exiftool" - save_record = True - if settings.auto_update_check is None: - settings.auto_update_check = True - save_record = True - if settings.auto_show_log_window is None: - settings.auto_show_log_window = True - save_record = True - - if save_record: self.save_settings(settings) return settings diff --git a/pie/core/indexing_helper.py b/pie/core/indexing_helper.py index 6c9000f..2023ac7 100644 --- a/pie/core/indexing_helper.py +++ b/pie/core/indexing_helper.py @@ -19,9 +19,22 @@ class IndexingHelper: def __init__(self, indexing_task: IndexingTask, log_queue: Queue, indexing_stop_event: Event): self.__indexing_task = indexing_task + self.__image_extensions: Set[str] = IndexingHelper.parse_file_type_extension_str(self.__indexing_task.settings.image_extensions) + self.__image_raw_extensions: Set[str] = IndexingHelper.parse_file_type_extension_str(self.__indexing_task.settings.image_raw_extensions) + self.__video_extensions: Set[str] = IndexingHelper.parse_file_type_extension_str(self.__indexing_task.settings.video_extensions) + self.__video_raw_extensions: Set[str] = IndexingHelper.parse_file_type_extension_str(self.__indexing_task.settings.video_raw_extensions) self.__log_queue = log_queue self.__indexing_stop_event = indexing_stop_event + @staticmethod + def parse_file_type_extension_str(comma_delimited_list: str) -> Set[str]: + ext_set = set() + for item in comma_delimited_list.split(','): + stripped_item = item.strip() + if len(stripped_item) > 0: + ext_set.add(stripped_item) + return ext_set + def lookup_already_indexed_files(self, indexDB: IndexDB, scanned_files: List[ScannedFile]): IndexingHelper.__logger.info("BEGIN:: IndexDB lookup for indexed files") total_scanned_files = len(scanned_files) @@ -103,7 +116,7 @@ def __scan_dir(self, dir_path, file_names, scanned_files): file_name_tuple = os.path.splitext(file_name) file_name_without_extension = file_name_tuple[0] extension = file_name_tuple[1].replace(".", "").upper() - (scanned_file_type, is_raw) = ScannedFileType.get_type(extension) + (scanned_file_type, is_raw) = ScannedFileType.get_type(self.__image_extensions, self.__image_raw_extensions, self.__video_extensions, self.__video_raw_extensions, extension) file_path = os.path.join(dir_path, file_name) if ScannedFileType.UNKNOWN != scanned_file_type: creation_time = datetime.fromtimestamp(os.path.getctime(file_path)) @@ -156,7 +169,7 @@ def create_media_files(self, scanned_files: List[ScannedFile]) -> List[str]: IndexingHelper.__logger.info("BEGIN:: Media file creation and indexing") pool = PyProcessPool(pool_name="IndexingWorker", process_count=self.__indexing_task.settings.indexing_workers, log_queue=self.__log_queue, target=IndexingHelper.indexing_process_exec, initializer=IndexDB.create_instance, terminator=IndexDB.destroy_instance, stop_event=self.__indexing_stop_event) - db_write_lock: Lock = Manager().Lock() # pylint: disable=maybe-no-member + db_write_lock: Lock = Manager().Lock() # pylint: disable=maybe-no-member tasks = list(map(lambda scanned_file: (self.__indexing_task.indexing_time, self.__indexing_task.settings.output_dir, self.__indexing_task.settings.unknown_output_dir, self.__indexing_task.settings.path_exiftool, scanned_file, db_write_lock), scanned_files)) saved_file_paths = pool.submit_and_wait(tasks) diff --git a/pie/domain/file_model.py b/pie/domain/file_model.py index 2b0428d..1474071 100644 --- a/pie/domain/file_model.py +++ b/pie/domain/file_model.py @@ -1,6 +1,9 @@ import hashlib +import multiprocessing +import sys from datetime import datetime from enum import Enum +from typing import Set from sqlalchemy import Boolean, Column, DateTime, Float, Integer, String @@ -8,24 +11,19 @@ class ScannedFileType(Enum): - __IMAGE_EXTENSIONS__ = ["JPEG", "JPG", "TIF", "TIFF", "PNG", "BMP", "HEIC"] - __RAW_IMAGE_EXTENSIONS__ = ["CR2", "DNG"] - __VIDEO_EXTENSIONS__ = ["MOV", "MP4", "M4V", "3G2", "3GP", "AVI", "MTS", "MPG", "MPEG"] - __RAW_VIDEO_EXTENSIONS__ = [] - IMAGE = 1 VIDEO = 2 UNKNOWN = 3 @staticmethod - def get_type(extension): - if extension in ScannedFileType.__IMAGE_EXTENSIONS__: + def get_type(image_extensions: Set[str], image_raw_extensions: Set[str], video_extensions: Set[str], video_raw_extensions: Set[str], file_extension: str): + if file_extension in image_extensions: return (ScannedFileType.IMAGE, False) - elif extension in ScannedFileType.__RAW_IMAGE_EXTENSIONS__: + elif file_extension in image_raw_extensions: return (ScannedFileType.IMAGE, True) - elif extension in ScannedFileType.__VIDEO_EXTENSIONS__: + elif file_extension in video_extensions: return (ScannedFileType.VIDEO, False) - elif extension in ScannedFileType.__RAW_VIDEO_EXTENSIONS__: + elif file_extension in video_raw_extensions: return (ScannedFileType.VIDEO, True) else: return (ScannedFileType.UNKNOWN, False) @@ -82,30 +80,45 @@ class Settings: def __init__(self) -> None: self.monitored_dir: str = None - self.dirs_to_exclude: str = None + self.dirs_to_exclude: str = '[]' self.output_dir: str = None self.unknown_output_dir: str = None - self.output_dir_path_type: str = None - self.unknown_output_dir_path_type: str = None - self.skip_same_name_video: bool = None - self.skip_same_name_raw: bool = None - self.convert_unknown: bool = None - self.overwrite_output_files: bool = None - self.indexing_workers: int = None - self.conversion_workers: int = None - self.gpu_workers: int = None - self.gpu_count: int = None - self.image_compression_quality: int = None - self.image_max_dimension: int = None - self.video_max_dimension: int = None - self.video_crf: int = None - self.video_nvenc_preset: str = None - self.video_audio_bitrate: int = None - self.path_ffmpeg: str = None - self.path_magick: str = None - self.path_exiftool: str = None - self.auto_update_check: bool = None - self.auto_show_log_window: bool = None + self.output_dir_path_type: str = "Use Original Paths" + self.unknown_output_dir_path_type: str = "Use Original Paths" + self.skip_same_name_video: bool = True + self.skip_same_name_raw: bool = True + self.convert_unknown: bool = False + self.overwrite_output_files: bool = False + self.indexing_workers: int = Settings.get_default_worker_count() + self.conversion_workers: int = Settings.get_default_worker_count() + self.gpu_workers: int = 1 + self.gpu_count: int = 0 + self.image_compression_quality: int = 75 + self.image_max_dimension: int = 1920 + self.video_max_dimension: int = 1920 + self.video_crf: int = 28 + self.video_nvenc_preset: str = "fast" + self.video_audio_bitrate: int = 128 + self.path_ffmpeg: str = "/usr/local/bin/ffmpeg" if not Settings.is_platform_win() else "ffmpeg" + self.path_magick: str = "/usr/local/bin/magick" if not Settings.is_platform_win() else "magick" + self.path_exiftool: str = "/usr/local/bin/exiftool" if not Settings.is_platform_win() else "exiftool" + self.auto_update_check: bool = True + self.auto_show_log_window: bool = True + self.image_extensions: str = "JPEG, JPG, TIF, TIFF, PNG, BMP, HEIC" + self.image_raw_extensions: str = "CRW, CR2, CR3, NRW, NEF, ARW, SRF, SR2, DNG" + self.video_extensions: str = "MOV, MP4, M4V, 3G2, 3GP, AVI, MTS, MPG, MPEG" + self.video_raw_extensions: str = "" + + @staticmethod + def is_platform_win() -> bool: + return sys.platform == 'win32' or sys.platform == 'cygwin' + + @staticmethod + def get_default_worker_count() -> int: + try: + return multiprocessing.cpu_count() + except: + return 1 def generate_image_settings_hash(self): settings_hash = hashlib.sha1() diff --git a/pie/preferences_window.py b/pie/preferences_window.py index a03cae1..05ab646 100644 --- a/pie/preferences_window.py +++ b/pie/preferences_window.py @@ -26,7 +26,7 @@ def __init__(self, apply_process_changed_setting: Callable[[], None]): ui_file.close() self.window.setWindowTitle("Edit Preferences") - self.window.setFixedSize(self.window.size()) # TODO: Disable maximize button on OSX + self.window.setFixedSize(self.window.size()) # TODO: Disable maximize button on OSX self.txtMonitoredDir: QtWidgets.QLineEdit = self.window.findChild(QtWidgets.QLineEdit, 'txtMonitoredDir') self.btnPickMonitoredDir: QtWidgets.QPushButton = self.window.findChild(QtWidgets.QPushButton, 'btnPickMonitoredDir') @@ -63,6 +63,11 @@ def __init__(self, apply_process_changed_setting: Callable[[], None]): self.txtPathMagick: QtWidgets.QLineEdit = self.window.findChild(QtWidgets.QLineEdit, 'txtPathMagick') self.txtPathExiftool: QtWidgets.QLineEdit = self.window.findChild(QtWidgets.QLineEdit, 'txtPathExiftool') + self.txtImageExt: QtWidgets.QLineEdit = self.window.findChild(QtWidgets.QLineEdit, 'txtImageExt') + self.txtImageRawExt: QtWidgets.QLineEdit = self.window.findChild(QtWidgets.QLineEdit, 'txtImageRawExt') + self.txtVideoExt: QtWidgets.QLineEdit = self.window.findChild(QtWidgets.QLineEdit, 'txtVideoExt') + self.txtVideoRawExt: QtWidgets.QLineEdit = self.window.findChild(QtWidgets.QLineEdit, 'txtVideoRawExt') + self.btnPickMonitoredDir.clicked.connect(self.btnPickMonitoredDir_click) self.lwDirsToExclude.itemSelectionChanged.connect(self.lwDirsToExclude_itemSelectionChanged) self.btnAddDirToExclude.clicked.connect(self.btnAddDirToExclude_click) @@ -93,6 +98,11 @@ def __init__(self, apply_process_changed_setting: Callable[[], None]): self.txtPathMagick.textChanged.connect(self.txtPathMagick_textChanged) self.txtPathExiftool.textChanged.connect(self.txtPathExiftool_textChanged) + self.txtImageExt.textChanged.connect(self.txtImageExt_textChanged) + self.txtImageRawExt.textChanged.connect(self.txtImageRawExt_textChanged) + self.txtVideoExt.textChanged.connect(self.txtVideoExt_textChanged) + self.txtVideoRawExt.textChanged.connect(self.txtVideoRawExt_textChanged) + self.cbVideoNvencPreset: QtWidgets.QComboBox = self.window.findChild(QtWidgets.QComboBox, 'cbVideoNvencPreset') self.__indexDB.save_settings(self.settings) @@ -184,6 +194,10 @@ def apply_settings(self): self.txtPathFfmpeg.setText(self.settings.path_ffmpeg) self.txtPathMagick.setText(self.settings.path_magick) self.txtPathExiftool.setText(self.settings.path_exiftool) + self.txtImageExt.setText(self.settings.image_extensions) + self.txtImageRawExt.setText(self.settings.image_raw_extensions) + self.txtVideoExt.setText(self.settings.video_extensions) + self.txtVideoRawExt.setText(self.settings.video_raw_extensions) def cleanup(self): self.__logger.info("Performing cleanup") @@ -281,3 +295,19 @@ def txtPathExiftool_textChanged(self, new_text: str): self.__indexDB.save_settings(self.settings) except: self.txtPathExiftool.setStyleSheet(PreferencesWindow.__QLINEEDIT_INVALID_VALUE_STYLESHEET) + + def txtImageExt_textChanged(self, new_text: str): + self.settings.image_extensions = new_text + self.__indexDB.save_settings(self.settings) + + def txtImageRawExt_textChanged(self, new_text: str): + self.settings.image_raw_extensions = new_text + self.__indexDB.save_settings(self.settings) + + def txtVideoExt_textChanged(self, new_text: str): + self.settings.video_extensions = new_text + self.__indexDB.save_settings(self.settings) + + def txtVideoRawExt_textChanged(self, new_text: str): + self.settings.video_raw_extensions = new_text + self.__indexDB.save_settings(self.settings)