diff --git a/app/brushtask.py b/app/brushtask.py index 7e2cef7970..dec2b838e3 100644 --- a/app/brushtask.py +++ b/app/brushtask.py @@ -103,6 +103,7 @@ def get_brushtask_info(self, taskid=None): "site_id": task.SITE, "interval": task.INTEVAL, "label": task.LABEL, + "savepath": task.SAVEPATH, "state": True if task.STATE == "Y" else False, "downloader": task.DOWNLOADER, "downloader_name": downloader_info.get("name") if downloader_info else None, @@ -156,6 +157,8 @@ def check_task_rss(self, taskid): if not site_info: log.error("【Brush】刷流任务 %s 的站点已不存在，无法刷流！" % task_name) return + # 站点属性 + site_id = site_info.get("id") site_name = site_info.get("name") site_proxy = site_info.get("proxy") @@ -217,6 +220,7 @@ def check_task_rss(self, taskid): torrent_url=page_url, torrent_size=size, pubdate=pubdate, + siteid=site_id, cookie=cookie, ua=ua, proxy=site_proxy): @@ -303,7 +307,7 @@ def __send_message(_task_name, _delete_type, _torrent_name): set(torrent_ids).difference( set([(torrent.get("hash") if downloader_type == 'qbittorrent' - else str(torrent.id)) for torrent in torrents]))) + else str(torrent.hashString)) for torrent in torrents]))) # 完成的种子 for torrent in torrents: torrent_info = self.__get_torrent_dict(downloader_type=downloader_type, @@ -345,7 +349,7 @@ def __send_message(_task_name, _delete_type, _torrent_name): set(remove_torrent_ids).difference( set([(torrent.get("hash") if downloader_type == 'qbittorrent' - else str(torrent.id)) for torrent in torrents]))) + else str(torrent.hashString)) for torrent in torrents]))) # 下载中的种子 for torrent in torrents: torrent_info = self.__get_torrent_dict(downloader_type=downloader_type, @@ -475,6 +479,9 @@ def __download_torrent(self, """ if not enclosure: return False + # 站点流控 + if self.sites.check_ratelimit(site_info.get("id")): + return False taskid = taskinfo.get("id") taskname = taskinfo.get("name") transfer = taskinfo.get("transfer") @@ -482,6 +489,7 @@ def __download_torrent(self, downloader_id = taskinfo.get("downloader") download_limit = rss_rule.get("downspeed") upload_limit = rss_rule.get("upspeed") + download_dir = taskinfo.get("savepath") tag = taskinfo.get("label").split(',') if taskinfo.get("label") else None # 标签 if not transfer: @@ -498,9 +506,11 @@ def __download_torrent(self, media_info=meta_info, tag=tag, downloader_id=downloader_id, + download_dir=download_dir, download_setting="-2", download_limit=download_limit, upload_limit=upload_limit, + is_auto=False if download_dir else None ) if not download_id: # 下载失败 @@ -536,6 +546,7 @@ def __check_rss_rule(self, torrent_url, torrent_size, pubdate, + siteid, cookie, ua, proxy): @@ -546,6 +557,7 @@ def __check_rss_rule(self, :param torrent_url: 种子页面地址 :param torrent_size: 种子大小 :param pubdate: 发布时间 + :param siteid: 站点ID :param cookie: Cookie :param ua: User-Agent :return: 是否命中 @@ -582,6 +594,10 @@ def __check_rss_rule(self, if re.search(r"%s" % rss_rule.get("exclude"), title): return False + # 站点流控 + if self.sites.check_ratelimit(siteid): + return False + torrent_attr = self.siteconf.check_torrent_attr(torrent_url=torrent_url, cookie=cookie, ua=ua, @@ -742,7 +758,7 @@ def __get_torrent_dict(downloader_type, torrent): total_size = torrent.get("total_size") else: # ID - torrent_id = torrent.id + torrent_id = torrent.hashString # 做种时间 date_done = torrent.date_done or torrent.date_added # 下载耗时 diff --git a/app/db/models.py b/app/db/models.py index 7c5bb18c57..7ea73959fa 100644 --- a/app/db/models.py +++ b/app/db/models.py @@ -389,6 +389,7 @@ class SITEBRUSHTASK(Base): SEED_SIZE = Column(Text) INTEVAL = Column(Text) LABEL = Column(Text) + SAVEPATH = Column(Text) DOWNLOADER = Column(Text) TRANSFER = Column(Text) DOWNLOAD_COUNT = Column(Integer) diff --git a/app/downloader/client/_base.py b/app/downloader/client/_base.py index 7a2a6a47e1..76fc1fb276 100644 --- a/app/downloader/client/_base.py +++ b/app/downloader/client/_base.py @@ -184,3 +184,10 @@ def set_speed_limit(self, **kwargs): 设置速度限制 """ pass + + @abstractmethod + def recheck_torrents(self, ids): + """ + 下载控制：重新校验 + """ + pass diff --git a/app/downloader/client/qbittorrent.py b/app/downloader/client/qbittorrent.py index 9a3b014cec..043b83bbeb 100644 --- a/app/downloader/client/qbittorrent.py +++ b/app/downloader/client/qbittorrent.py @@ -12,7 +12,6 @@ class Qbittorrent(_IDownloadClient): - # 下载器ID client_id = "qbittorrent" # 下载器类型 @@ -317,6 +316,7 @@ def get_torrent_id_by_tag(self, tag, status=None): def add_torrent(self, content, is_paused=False, + is_auto=None, download_dir=None, tag=None, category=None, @@ -331,6 +331,7 @@ def add_torrent(self, 添加种子 :param content: 种子urls或文件 :param is_paused: 添加后暂停 + :param is_auto: 自动管理 :param tag: 标签 :param download_dir: 下载路径 :param category: 分类 @@ -379,7 +380,9 @@ def add_torrent(self, else: seeding_time_limit = None try: - if self._torrent_management: + if is_auto is None: + is_auto = self._torrent_management + if is_auto: save_path = None qbc_ret = self.qbc.torrents_add(urls=urls, torrent_files=torrent_files, @@ -392,7 +395,7 @@ def add_torrent(self, download_limit=download_limit, ratio_limit=ratio_limit, seeding_time_limit=seeding_time_limit, - use_auto_torrent_management=self._torrent_management, + use_auto_torrent_management=is_auto, cookie=cookie) return True if qbc_ret and str(qbc_ret).find("Ok") != -1 else False except Exception as err: @@ -545,3 +548,12 @@ def set_speed_limit(self, download_limit=None, upload_limit=None): except Exception as err: ExceptionUtils.exception_traceback(err) return False + + def recheck_torrents(self, ids): + if not self.qbc: + return False + try: + return self.qbc.torrents_recheck(torrent_hashes=ids) + except Exception as err: + ExceptionUtils.exception_traceback(err) + return False diff --git a/app/downloader/client/transmission.py b/app/downloader/client/transmission.py index c0da6e79c6..8b5a1942b1 100644 --- a/app/downloader/client/transmission.py +++ b/app/downloader/client/transmission.py @@ -80,6 +80,17 @@ def __login_transmission(self): def get_status(self): return True if self.trc else False + @staticmethod + def __parse_ids(ids): + """ + 统一处理种子ID + """ + if isinstance(ids, list) and any([str(x).isdigit() for x in ids]): + ids = [int(x) for x in ids if str(x).isdigit()] + elif not isinstance(ids, list) and str(ids).isdigit(): + ids = int(ids) + return ids + def get_torrents(self, ids=None, status=None, tag=None): """ 获取种子列表 @@ -87,10 +98,7 @@ def get_torrents(self, ids=None, status=None, tag=None): """ if not self.trc: return [], True - if isinstance(ids, list): - ids = [int(x) for x in ids if str(x).isdigit()] - elif str(ids).isdigit(): - ids = int(ids) + ids = self.__parse_ids(ids) try: torrents = self.trc.get_torrents(ids=ids, arguments=self._trarg) except Exception as err: @@ -151,10 +159,7 @@ def set_torrents_status(self, ids, tags=None): """ if not self.trc: return - if isinstance(ids, list): - ids = [int(x) for x in ids if str(x).isdigit()] - elif str(ids).isdigit(): - ids = int(ids) + ids = self.__parse_ids(ids) # 合成标签 if tags: if not isinstance(tags, list): @@ -176,8 +181,9 @@ def set_torrent_tag(self, tid, tag): """ if not tid or not tag: return + ids = self.__parse_ids(tid) try: - self.trc.change_torrent(labels=tag, ids=int(tid)) + self.trc.change_torrent(labels=tag, ids=ids) except Exception as err: ExceptionUtils.exception_traceback(err) @@ -201,7 +207,7 @@ def change_torrent(self, if not tid: return else: - ids = int(tid) + ids = self.__parse_ids(tid) if tag: if isinstance(tag, list): labels = tag @@ -270,7 +276,7 @@ def get_transfer_task(self, tag, match_path=None): true_path = self.get_replace_path(path, self.download_dir) trans_tasks.append({ 'path': os.path.join(true_path, torrent.name).replace("\\", "/"), - 'id': torrent.id, + 'id': torrent.hashString, 'tags': torrent.labels }) return trans_tasks @@ -329,21 +335,21 @@ def get_remove_torrents(self, config=None): if tr_error_key and not re.findall(tr_error_key, torrent.error_string, re.I): continue remove_torrents.append({ - "id": torrent.id, + "id": torrent.hashString, "name": torrent.name, "site": torrent.trackers[0].get("sitename"), "size": torrent.total_size }) - remove_torrents_ids.append(torrent.id) + remove_torrents_ids.append(torrent.hashString) if config.get("samedata") and remove_torrents: remove_torrents_plus = [] for remove_torrent in remove_torrents: name = remove_torrent.get("name") size = remove_torrent.get("size") for torrent in torrents: - if torrent.name == name and torrent.total_size == size and torrent.id not in remove_torrents_ids: + if torrent.name == name and torrent.total_size == size and torrent.hashString not in remove_torrents_ids: remove_torrents_plus.append({ - "id": torrent.id, + "id": torrent.hashString, "name": torrent.name, "site": torrent.trackers[0].get("sitename") if torrent.trackers else "", "size": torrent.total_size @@ -364,11 +370,11 @@ def add_torrent(self, content, download_dir=download_dir, paused=is_paused, cookies=cookie) - if ret and ret.id: + if ret and ret.hashString: if upload_limit: - self.set_uploadspeed_limit(ret.id, int(upload_limit)) + self.set_uploadspeed_limit(ret.hashString, int(upload_limit)) if download_limit: - self.set_downloadspeed_limit(ret.id, int(download_limit)) + self.set_downloadspeed_limit(ret.hashString, int(download_limit)) return ret except Exception as err: ExceptionUtils.exception_traceback(err) @@ -377,10 +383,7 @@ def add_torrent(self, content, def start_torrents(self, ids): if not self.trc: return False - if isinstance(ids, list): - ids = [int(x) for x in ids if str(x).isdigit()] - elif str(ids).isdigit(): - ids = int(ids) + ids = self.__parse_ids(ids) try: return self.trc.start_torrent(ids=ids) except Exception as err: @@ -390,10 +393,7 @@ def start_torrents(self, ids): def stop_torrents(self, ids): if not self.trc: return False - if isinstance(ids, list): - ids = [int(x) for x in ids if str(x).isdigit()] - elif str(ids).isdigit(): - ids = int(ids) + ids = self.__parse_ids(ids) try: return self.trc.stop_torrent(ids=ids) except Exception as err: @@ -405,10 +405,7 @@ def delete_torrents(self, delete_file, ids): return False if not ids: return False - if isinstance(ids, list): - ids = [int(x) for x in ids if str(x).isdigit()] - elif str(ids).isdigit(): - ids = int(ids) + ids = self.__parse_ids(ids) try: return self.trc.remove_torrent(delete_data=delete_file, ids=ids) except Exception as err: @@ -471,10 +468,7 @@ def set_uploadspeed_limit(self, ids, limit): return if not ids or not limit: return - if not isinstance(ids, list): - ids = int(ids) - else: - ids = [int(x) for x in ids if str(x).isdigit()] + ids = self.__parse_ids(ids) self.trc.change_torrent(ids, uploadLimit=int(limit)) def set_downloadspeed_limit(self, ids, limit): @@ -485,10 +479,7 @@ def set_downloadspeed_limit(self, ids, limit): return if not ids or not limit: return - if not isinstance(ids, list): - ids = int(ids) - else: - ids = [int(x) for x in ids if str(x).isdigit()] + ids = self.__parse_ids(ids) self.trc.change_torrent(ids, downloadLimit=int(limit)) def get_downloading_progress(self, tag=None, ids=None): @@ -515,7 +506,7 @@ def get_downloading_progress(self, tag=None, ids=None): # 进度 progress = round(torrent.progress) DispTorrents.append({ - 'id': torrent.id, + 'id': torrent.hashString, 'name': torrent.name, 'speed': speed, 'state': state, @@ -551,3 +542,13 @@ def set_speed_limit(self, download_limit=None, upload_limit=None): except Exception as err: ExceptionUtils.exception_traceback(err) return False + + def recheck_torrents(self, ids): + if not self.trc: + return False + ids = self.__parse_ids(ids) + try: + return self.trc.verify_torrent(ids=ids) + except Exception as err: + ExceptionUtils.exception_traceback(err) + return False diff --git a/app/downloader/downloader.py b/app/downloader/downloader.py index 6a4dae567d..343dccef6a 100644 --- a/app/downloader/downloader.py +++ b/app/downloader/downloader.py @@ -254,7 +254,8 @@ def download(self, download_limit=None, torrent_file=None, in_from=None, - user_name=None): + user_name=None, + is_auto=None): """ 添加下载任务，根据当前使用的下载器分别调用不同的客户端处理 :param media_info: 需下载的媒体信息，含URL地址 @@ -268,6 +269,7 @@ def download(self, :param torrent_file: 种子文件路径 :param in_from: 来源 :param user_name: 用户名 + :param is_auto: 是否开始自动管理模式 :return: 下载器类型, 种子ID，错误信息 """ @@ -430,7 +432,7 @@ def __download_fail(msg): download_dir=download_dir, cookie=site_info.get("cookie")) if ret: - download_id = ret.id + download_id = ret.hashString downloader.change_torrent(tid=download_id, tag=tags, upload_limit=upload_limit, @@ -446,6 +448,7 @@ def __download_fail(msg): tags = [torrent_tag] ret = downloader.add_torrent(content, is_paused=is_paused, + is_auto=is_auto, download_dir=download_dir, tag=tags, category=category, @@ -486,7 +489,13 @@ def __download_fail(msg): subtitle_dir = visit_dir ThreadHelper().start_thread( self.sitesubtitle.download, - (media_info, site_info.get("cookie"), site_info.get("ua"), subtitle_dir) + ( + media_info, + site_info.get("id"), + site_info.get("cookie"), + site_info.get("ua"), + subtitle_dir + ) ) # 发送下载消息 if in_from: @@ -1317,3 +1326,17 @@ def get_status(self, dtype=None, config=None): if not state: log.error(f"【Downloader】下载器连接测试失败") return state + + def recheck_torrents(self, downloader_id=None, ids=None): + """ + 下载控制：重新校验种子 + :param downloader_id: 下载器ID + :param ids: 种子ID列表 + :return: 处理状态 + """ + if not ids: + return False + _client = self.__get_client(downloader_id) if downloader_id else self.default_client + if not _client: + return False + return _client.recheck_torrents(ids) diff --git a/app/filter.py b/app/filter.py index 6ce2eae4d7..9598a599f4 100644 --- a/app/filter.py +++ b/app/filter.py @@ -1,5 +1,6 @@ import re +import log from app.conf import ModuleConf from app.helper import DbHelper from app.media.meta import ReleaseGroupsMatcher @@ -123,76 +124,79 @@ def check_rules(self, meta_info, rulegroup=None): # 当前规则组是否命中 group_match = True for filter_info in filters: - # 当前规则是否命中 - rule_match = True - # 命中规则的序号 - order_seq = 100 - int(filter_info.get('pri')) - # 必须包括的项 - includes = filter_info.get('include') - if includes and rule_match: - include_flag = True - for include in includes: - if not include: - continue - if not re.search(r'%s' % include.strip(), title, re.IGNORECASE): - include_flag = False - break - if not include_flag: - rule_match = False + try: + # 当前规则是否命中 + rule_match = True + # 命中规则的序号 + order_seq = 100 - int(filter_info.get('pri')) + # 必须包括的项 + includes = filter_info.get('include') + if includes and rule_match: + include_flag = True + for include in includes: + if not include: + continue + if not re.search(r'%s' % include.strip(), title, re.IGNORECASE): + include_flag = False + break + if not include_flag: + rule_match = False - # 不能包含的项 - excludes = filter_info.get('exclude') - if excludes and rule_match: - exclude_flag = False - exclude_count = 0 - for exclude in excludes: - if not exclude: - continue - exclude_count += 1 - if not re.search(r'%s' % exclude.strip(), title, re.IGNORECASE): - exclude_flag = True - if exclude_count > 0 and not exclude_flag: - rule_match = False - # 大小 - sizes = filter_info.get('size') - if sizes and rule_match and meta_info.size: - meta_info.size = StringUtils.num_filesize(meta_info.size) - if sizes.find(',') != -1: - sizes = sizes.split(',') - if sizes[0].isdigit(): - begin_size = int(sizes[0].strip()) + # 不能包含的项 + excludes = filter_info.get('exclude') + if excludes and rule_match: + exclude_flag = False + exclude_count = 0 + for exclude in excludes: + if not exclude: + continue + exclude_count += 1 + if not re.search(r'%s' % exclude.strip(), title, re.IGNORECASE): + exclude_flag = True + if exclude_count > 0 and not exclude_flag: + rule_match = False + # 大小 + sizes = filter_info.get('size') + if sizes and rule_match and meta_info.size: + meta_info.size = StringUtils.num_filesize(meta_info.size) + if sizes.find(',') != -1: + sizes = sizes.split(',') + if sizes[0].isdigit(): + begin_size = int(sizes[0].strip()) + else: + begin_size = 0 + if sizes[1].isdigit(): + end_size = int(sizes[1].strip()) + else: + end_size = 0 else: begin_size = 0 - if sizes[1].isdigit(): - end_size = int(sizes[1].strip()) + if sizes.isdigit(): + end_size = int(sizes.strip()) + else: + end_size = 0 + if meta_info.type == MediaType.MOVIE: + if not begin_size * 1024 ** 3 <= int(meta_info.size) <= end_size * 1024 ** 3: + rule_match = False else: - end_size = 0 - else: - begin_size = 0 - if sizes.isdigit(): - end_size = int(sizes.strip()) - else: - end_size = 0 - if meta_info.type == MediaType.MOVIE: - if not begin_size * 1024 ** 3 <= int(meta_info.size) <= end_size * 1024 ** 3: - rule_match = False - else: - if meta_info.total_episodes \ - and not begin_size * 1024 ** 3 <= int(meta_info.size) / int(meta_info.total_episodes) <= end_size * 1024 ** 3: - rule_match = False + if meta_info.total_episodes \ + and not begin_size * 1024 ** 3 <= int(meta_info.size) / int(meta_info.total_episodes) <= end_size * 1024 ** 3: + rule_match = False - # 促销 - free = filter_info.get("free") - if free and meta_info.upload_volume_factor is not None and meta_info.download_volume_factor is not None: - ul_factor, dl_factor = free.split() - if float(ul_factor) > meta_info.upload_volume_factor \ - or float(dl_factor) < meta_info.download_volume_factor: - rule_match = False + # 促销 + free = filter_info.get("free") + if free and meta_info.upload_volume_factor is not None and meta_info.download_volume_factor is not None: + ul_factor, dl_factor = free.split() + if float(ul_factor) > meta_info.upload_volume_factor \ + or float(dl_factor) < meta_info.download_volume_factor: + rule_match = False - if rule_match: - return True, order_seq, rulegroup.get("name") - else: - group_match = False + if rule_match: + return True, order_seq, rulegroup.get("name") + else: + group_match = False + except Exception as err: + log.error(f"【Filter】过滤规则出现严重错误 {err}，请检查：{filter_info}") if not group_match: return False, 0, rulegroup.get("name") return True, order_seq, rulegroup.get("name") diff --git a/app/helper/__init__.py b/app/helper/__init__.py index eac7702fe1..5c72e97350 100644 --- a/app/helper/__init__.py +++ b/app/helper/__init__.py @@ -11,6 +11,6 @@ from .ocr_helper import OcrHelper from .words_helper import WordsHelper from .submodule_helper import SubmoduleHelper -from .cookiecloud_helper import CookieCloudHelper from .ffmpeg_helper import FfmpegHelper from .redis_helper import RedisHelper +from .iyuu_helper import IyuuHelper diff --git a/app/helper/chrome_helper.py b/app/helper/chrome_helper.py index 5cfafb387d..ef8a9b6cd3 100644 --- a/app/helper/chrome_helper.py +++ b/app/helper/chrome_helper.py @@ -74,6 +74,8 @@ def __get_browser(self): options.add_argument('--no-service-autorun') options.add_argument('--no-default-browser-check') options.add_argument('--password-store=basic') + if SystemUtils.is_windows() or SystemUtils.is_macos(): + options.add_argument("--window-position=-32000,-32000") if self._proxy: proxy = Config().get_proxies().get("https") if proxy: diff --git a/app/helper/cookiecloud_helper.py b/app/helper/cookiecloud_helper.py deleted file mode 100644 index ca88d9716f..0000000000 --- a/app/helper/cookiecloud_helper.py +++ /dev/null @@ -1,39 +0,0 @@ -from app.utils import RequestUtils - - -class CookieCloudHelper(object): - _req = None - _server = None - _key = None - _password = None - - def __init__(self, server, key, password): - self._server = server - if self._server: - if not self._server.startswith("http"): - self._server = "http://%s" % self._server - if self._server.endswith("/"): - self._server = self._server[:-1] - self._key = key - self._password = password - self._req = RequestUtils(content_type="application/json") - - def download_data(self): - """ - 从CookieCloud下载数据 - """ - if not self._server or not self._key or not self._password: - return {}, "CookieCloud参数不正确" - req_url = "%s/get/%s" % (self._server, self._key) - ret = self._req.post_res(url=req_url, json={"password": self._password}) - if ret and ret.status_code == 200: - result = ret.json() - if not result: - return {}, "" - if result.get("cookie_data"): - return result.get("cookie_data"), "" - return result, "" - elif ret: - return {}, "同步CookieCloud失败，错误码：%s" % ret.status_code - else: - return {}, "CookieCloud请求失败，请检查服务器地址、用户KEY及加密密码是否正确" diff --git a/app/helper/db_helper.py b/app/helper/db_helper.py index 440263d2a7..a14e1a711e 100644 --- a/app/helper/db_helper.py +++ b/app/helper/db_helper.py @@ -558,7 +558,8 @@ def get_site_by_name(self, name): return self._db.query(CONFIGSITE).filter(CONFIGSITE.NAME == name).all() @DbPersist(_db) - def insert_config_site(self, name, site_pri, rssurl, signurl, cookie, note, rss_uses): + def insert_config_site(self, name, site_pri, + rssurl=None, signurl=None, cookie=None, note=None, rss_uses=None): """ 插入站点信息 """ @@ -1716,6 +1717,7 @@ def insert_brushtask(self, brush_id, item): INTEVAL=item.get('interval'), DOWNLOADER=item.get('downloader'), LABEL=item.get('label'), + SAVEPATH=item.get('savepath'), TRANSFER=item.get('transfer'), DOWNLOAD_COUNT=0, REMOVE_COUNT=0, @@ -1738,6 +1740,7 @@ def insert_brushtask(self, brush_id, item): "INTEVAL": item.get('interval'), "DOWNLOADER": item.get('downloader'), "LABEL": item.get('label'), + "SAVEPATH": item.get('savepath'), "TRANSFER": item.get('transfer'), "STATE": item.get('state'), "LST_MOD_DATE": time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), diff --git a/app/helper/indexer_helper.py b/app/helper/indexer_helper.py index 2e3f4b3861..dc1cdc8d24 100644 --- a/app/helper/indexer_helper.py +++ b/app/helper/indexer_helper.py @@ -25,8 +25,15 @@ def init_config(self): def get_all_indexers(self): return self._indexers + def get_indexer_info(self, url): + for indexer in self._indexers: + if StringUtils.url_equal(indexer.get("domain"), url): + return indexer + return None + def get_indexer(self, url, + siteid=None, cookie=None, name=None, rule=None, @@ -44,6 +51,7 @@ def get_indexer(self, continue if StringUtils.url_equal(indexer.get("domain"), url): return IndexerConf(datas=indexer, + siteid=siteid, cookie=cookie, name=name, rule=rule, @@ -62,6 +70,7 @@ class IndexerConf(object): def __init__(self, datas=None, + siteid=None, cookie=None, name=None, rule=None, @@ -75,7 +84,7 @@ def __init__(self, pri=None): if not datas: return - # ID + # 索引ID self.id = datas.get('id') # 名称 self.name = datas.get('name') if not name else name @@ -97,6 +106,8 @@ def __init__(self, self.torrents = datas.get('torrents', {}) # 分类 self.category = datas.get('category', {}) + # 站点ID + self.siteid = siteid # Cookie self.cookie = cookie # User-Agent diff --git a/app/helper/iyuu_helper.py b/app/helper/iyuu_helper.py new file mode 100644 index 0000000000..84b707e848 --- /dev/null +++ b/app/helper/iyuu_helper.py @@ -0,0 +1,130 @@ +import hashlib +import json +import time + +from app.utils import RequestUtils +from app.utils.commons import singleton + + +@singleton +class IyuuHelper(object): + _version = "2.0.0" + _api_base = "https://api.iyuu.cn/%s" + _sites = {} + _token = None + + def __init__(self, token): + self._token = token + if self._token: + self.init_config() + + def init_config(self): + self._sites = self.__get_sites() + + def __request_iyuu(self, url, method="get", params=None): + """ + 向IYUUApi发送请求 + """ + if params: + if not params.get("sign"): + params.update({"sign": self._token}) + if not params.get("version"): + params.update({"version": self._version}) + else: + params = {"sign": self._token, "version": self._version} + # 开始请求 + if method == "get": + ret = RequestUtils( + accept_type="application/json" + ).get_res(f"{url}", params=params) + else: + ret = RequestUtils( + accept_type="application/json" + ).post_res(f"{url}", data=params) + if ret: + result = ret.json() + if result.get('ret') == 200: + return result.get('data'), "" + else: + return None, f"请求IYUU失败，状态码：{result.get('ret')}，返回信息：{result.get('msg')}" + elif ret is not None: + return None, f"请求IYUU失败，状态码：{ret.status_code}，错误原因：{ret.reason}" + else: + return None, f"请求IYUU失败，未获取到返回信息" + + def get_torrent_url(self, sid): + if not sid: + return None, None + if not self._sites.get(sid): + return None, None + site = self._sites.get(sid) + return site.get('base_url'), site.get('download_page') + + def __get_sites(self): + """ + 返回支持辅种的全部站点 + :return: 站点列表、错误信息 + { + "ret": 200, + "data": { + "sites": [ + { + "id": 1, + "site": "keepfrds", + "nickname": "朋友", + "base_url": "pt.keepfrds.com", + "download_page": "download.php?id={}&passkey={passkey}", + "reseed_check": "passkey", + "is_https": 2 + }, + ] + } + } + """ + result, msg = self.__request_iyuu(url=self._api_base % 'api/sites') + if result: + ret_sites = {} + sites = result.get('sites') or [] + for site in sites: + ret_sites[site.get('id')] = site + return ret_sites + else: + print(msg) + return {} + + def get_seed_info(self, info_hashs: list): + """ + 返回info_hash对应的站点id、种子id + { + "ret": 200, + "data": [ + { + "sid": 3, + "torrent_id": 377467, + "info_hash": "a444850638e7a6f6220e2efdde94099c53358159" + }, + { + "sid": 7, + "torrent_id": 35538, + "info_hash": "cf7d88fd656d10fe5130d13567aec27068b96676" + } + ], + "msg": "", + "version": "1.0.0" + } + """ + info_hashs.sort() + json_data = json.dumps(info_hashs, separators=(',', ':'), ensure_ascii=False) + sha1 = self.get_sha1(json_data) + result, msg = self.__request_iyuu(url=self._api_base % 'api/infohash', + method="post", + params={ + "timestamp": time.time(), + "hash": json_data, + "sha1": sha1 + }) + return result, msg + + @staticmethod + def get_sha1(json_str) -> str: + return hashlib.sha1(json_str.encode('utf-8')).hexdigest() diff --git a/app/indexer/client/builtin.py b/app/indexer/client/builtin.py index d27dead67b..8c94380320 100644 --- a/app/indexer/client/builtin.py +++ b/app/indexer/client/builtin.py @@ -67,6 +67,7 @@ def get_indexers(self, check=True, indexer_id=None): continue render = False if not chrome_ok else site.get("chrome") indexer = IndexerHelper().get_indexer(url=url, + siteid=site.get("id"), cookie=cookie, ua=site.get("ua"), name=site.get("name"), @@ -101,6 +102,10 @@ def search(self, order_seq, indexer_sites = Config().get_config("pt").get("indexer_sites") or [] if indexer_sites and indexer.id not in indexer_sites: return [] + # 站点流控 + if self.sites.check_ratelimit(indexer.siteid): + self.progress.update(ptype=ProgressKey.Search, text=f"{indexer.name} 触发站点流控，跳过 ...") + return [] # fix 共用同一个dict时会导致某个站点的更新全局全效 if filter_args is None: _filter_args = {} diff --git a/app/media/media.py b/app/media/media.py index b75226f0e5..f69729749c 100644 --- a/app/media/media.py +++ b/app/media/media.py @@ -436,7 +436,8 @@ def __search_multi_tmdb(self, file_media_name): info = tv_info # 返回 if info: - info['media_type'] = MediaType.MOVIE if info.get('media_type') == 'movie' else MediaType.TV + info['media_type'] = MediaType.MOVIE if info.get('media_type') in ['movie', + MediaType.MOVIE] else MediaType.TV else: log.info("【Meta】%s 在TMDB中未找到媒体信息!" % file_media_name) return info @@ -2010,6 +2011,18 @@ def get_tmdb_discover(self, mtype, params=None, page=1): print(str(e)) return [] + def get_tmdb_discover_movies_pages(self, params=None): + """ + 获取电影浏览的总页数 + """ + if not self.discover: + return 0 + try: + return self.discover.discover_movies_pages(params=params) + except Exception as e: + print(str(e)) + return 0 + def get_person_medias(self, personid, mtype=None, page=1): """ 查询人物相关影视作品 diff --git a/app/media/scraper.py b/app/media/scraper.py index 72b972f378..4619afc9e8 100644 --- a/app/media/scraper.py +++ b/app/media/scraper.py @@ -7,9 +7,10 @@ import log from app.helper import FfmpegHelper from app.media.douban import DouBan +from app.media.meta import MetaInfo from app.utils.commons import retry -from config import Config -from app.utils import DomUtils, RequestUtils, ExceptionUtils +from config import Config, RMT_MEDIAEXT +from app.utils import DomUtils, RequestUtils, ExceptionUtils, NfoReader from app.utils.types import MediaType from app.media import Media @@ -27,6 +28,110 @@ def __init__(self): self._scraper_nfo = Config().get_config('scraper_nfo') self._scraper_pic = Config().get_config('scraper_pic') + def folder_scraper(self, path, exclude_path=None, mode=None): + """ + 刮削指定文件夹或文件 + :param path: 文件夹或文件路径 + :param exclude_path: 排除路径 + :param mode: 刮削模式，可选值：force_nfo, force_all + :return: + """ + # 模式 + force_nfo = True if mode in ["force_nfo", "force_all"] else False + force_pic = True if mode in ["force_all"] else False + # 每个媒体库下的所有文件 + for file in self.__get_library_files(path, exclude_path): + if not file: + continue + log.info(f"【Scraper】开始刮削媒体库文件：{file} ...") + # 识别媒体文件 + meta_info = MetaInfo(os.path.basename(file)) + # 优先读取本地文件 + tmdbid = None + if meta_info.type == MediaType.MOVIE: + # 电影 + movie_nfo = os.path.join(os.path.dirname(file), "movie.nfo") + if os.path.exists(movie_nfo): + tmdbid = self.__get_tmdbid_from_nfo(movie_nfo) + file_nfo = os.path.join(os.path.splitext(file)[0] + ".nfo") + if not tmdbid and os.path.exists(file_nfo): + tmdbid = self.__get_tmdbid_from_nfo(file_nfo) + else: + # 电视剧 + tv_nfo = os.path.join(os.path.dirname(os.path.dirname(file)), "tvshow.nfo") + if os.path.exists(tv_nfo): + tmdbid = self.__get_tmdbid_from_nfo(tv_nfo) + if tmdbid and not force_nfo: + log.info(f"【Scraper】读取到本地nfo文件的tmdbid：{tmdbid}") + meta_info.set_tmdb_info(self.media.get_tmdb_info(mtype=meta_info.type, + tmdbid=tmdbid, + append_to_response='all')) + media_info = meta_info + else: + medias = self.media.get_media_info_on_files(file_list=[file], + append_to_response="all") + if not medias: + continue + media_info = None + for _, media in medias.items(): + media_info = media + break + if not media_info or not media_info.tmdb_info: + continue + self.gen_scraper_files(media=media_info, + dir_path=os.path.dirname(file), + file_name=os.path.splitext(os.path.basename(file))[0], + file_ext=os.path.splitext(file)[-1], + force=True, + force_nfo=force_nfo, + force_pic=force_pic) + log.info(f"【Scraper】{file} 刮削完成") + + @staticmethod + def __get_library_files(in_path, exclude_path=None): + """ + 获取媒体库文件列表 + """ + if not os.path.isdir(in_path): + yield in_path + return + + for root, dirs, files in os.walk(in_path): + if exclude_path and any(os.path.abspath(root).startswith(os.path.abspath(path)) + for path in exclude_path.split(",")): + continue + + for file in files: + cur_path = os.path.join(root, file) + # 检查后缀 + if os.path.splitext(file)[-1].lower() in RMT_MEDIAEXT: + yield cur_path + + @staticmethod + def __get_tmdbid_from_nfo(file_path): + """ + 从nfo文件中获取信息 + :param file_path: + :return: tmdbid + """ + if not file_path: + return None + xpaths = [ + "uniqueid[@type='Tmdb']", + "uniqueid[@type='tmdb']", + "uniqueid[@type='TMDB']", + "tmdbid" + ] + reader = NfoReader(file_path) + for xpath in xpaths: + try: + tmdbid = reader.get_element_value(xpath) + if tmdbid: + return tmdbid + except Exception as err: + print(str(err)) + return None + def __gen_common_nfo(self, tmdbinfo: dict, doubaninfo: dict, diff --git a/app/media/tmdbv3api/objs/discover.py b/app/media/tmdbv3api/objs/discover.py index b42b648186..d6d5f1bb24 100644 --- a/app/media/tmdbv3api/objs/discover.py +++ b/app/media/tmdbv3api/objs/discover.py @@ -31,6 +31,20 @@ def discover_movies(self, params, page=1): "results" ) + def discover_movies_pages(self, params): + """ + Discover movies by different types of data like average rating, number of votes, genres and certifications. + :param params: dict + :return: total_pages + """ + if not params: + params = {} + result = self._call( + self._urls["movies"], + urlencode(params) + ) + return result.get("total_pages") or 0 + def discover_tv_shows(self, params, page=1): """ Discover TV shows by different types of data like average rating, number of votes, genres, diff --git a/app/plugins/event_manager.py b/app/plugins/event_manager.py index 970b11651b..9720b333e8 100644 --- a/app/plugins/event_manager.py +++ b/app/plugins/event_manager.py @@ -83,6 +83,9 @@ def decorator(f): if isinstance(etype, list): for et in etype: self.add_event_listener(et, f) + elif type(etype) == type(EventType): + for et in etype.__members__.values(): + self.add_event_listener(et, f) else: self.add_event_listener(etype, f) return f diff --git a/app/plugins/modules/_base.py b/app/plugins/modules/_base.py index f49dd58375..4900d6371b 100644 --- a/app/plugins/modules/_base.py +++ b/app/plugins/modules/_base.py @@ -6,7 +6,21 @@ class _IPluginModule(metaclass=ABCMeta): """ - 插件模块基类 + 插件模块基类，通过继续该类实现插件功能 + 除内置属性外，还有以下方法可以扩展： + - get_fields() 获取配置字典，用于生成插件配置表单 + - get_state() 获取插件启用状态，用于展示运行状态 + - stop_service() 停止插件服务 + - get_config() 获取配置信息 + - update_config() 更新配置信息 + - init_config() 生效配置信息 + - info(msg) 记录INFO日志 + - warn(msg) 记录插件WARN日志 + - error(msg) 记录插件ERROR日志 + - debug(msg) 记录插件DEBUG日志 + - get_page() 插件额外页面数据，在插件配置页面左下解按钮展示 + - get_script() 插件额外脚本（Javascript），将会写入插件页面，可在插件元素中绑定使用 + """ # 插件名称 module_name = "" diff --git a/app/plugins/modules/autosub.py b/app/plugins/modules/autosub.py index 5c7fcc2af6..bc0764fec2 100644 --- a/app/plugins/modules/autosub.py +++ b/app/plugins/modules/autosub.py @@ -5,9 +5,12 @@ import tempfile import time import traceback +from datetime import timedelta import iso639 +import psutil import srt +from lxml import etree from app.helper import FfmpegHelper from app.helper.openai_helper import OpenAiHelper @@ -43,7 +46,7 @@ class AutoSub(_IPluginModule): _running = False # 语句结束符 _end_token = ['.', '!', '?', '。', '！', '？', '。"', '！"', '？"', '."', '!"', '?"'] - _noisy_token = [('(', ')'), ('[', ']'), ('{', '}'), ('<', '>'), ('【', '】'), ('♪', '♪'), ('♫', '♫'), ('♪♪', '♪♪')] + _noisy_token = [('(', ')'), ('[', ']'), ('{', '}'), ('【', '】'), ('♪', '♪'), ('♫', '♫'), ('♪♪', '♪♪')] def __init__(self): self.additional_args = '-t 4 -p 1' @@ -57,6 +60,9 @@ def __init__(self): self.fail_count = 0 self.success_count = 0 self.send_notify = False + self.asr_engine = 'whisper.cpp' + self.faster_whisper_model = 'base' + self.faster_whisper_model_path = None @staticmethod def get_fields(): @@ -65,11 +71,64 @@ def get_fields(): { 'type': 'div', 'content': [ - # 同一行 [ { - 'title': 'whisper.cpp路径', + 'title': '媒体路径', + 'required': '', + 'tooltip': '要进行字幕生成的路径，每行一个路径，请确保路径正确', + 'type': 'textarea', + 'content': + { + 'id': 'path_list', + 'placeholder': '文件路径', + 'rows': 5 + } + } + ], + # asr 引擎 + [ + { + 'title': '文件大小（MB）', + 'required': "required", + 'tooltip': '单位 MB, 大于该大小的文件才会进行字幕生成', + 'type': 'text', + 'content': + [{ + 'id': 'file_size', + 'placeholder': '文件大小, 单位MB' + }] + }, + { + 'title': 'ASR引擎', 'required': "required", + 'tooltip': '自动语音识别引擎选择', + 'type': 'select', + 'content': [ + { + 'id': 'asr_engine', + 'options': { + 'whisper.cpp': 'whisper.cpp', + 'faster-whisper': 'faster-whisper' + }, + 'default': 'whisper.cpp', + 'onchange': 'AutoSub_asr_engine_change(this)' + } + ] + } + ] + ] + }, + { + 'type': 'details', + 'id': 'whisper_config', + 'summary': 'whisper.cpp 配置', + 'tooltip': '使用 whisper.cpp 引擎时的配置', + 'hidden': False, + 'content': [ + [ + { + 'title': 'whisper.cpp路径', + 'required': "", 'tooltip': '填写whisper.cpp主程序路径，如/config/plugin/autosub/main \n' '推荐教程 https://ddsrem.com/autosub', 'type': 'text', @@ -81,11 +140,10 @@ def get_fields(): ] } ], - # 模型路径 [ { 'title': 'whisper.cpp模型路径', - 'required': "required", + 'required': "", 'tooltip': '填写whisper.cpp模型路径，如/config/plugin/autosub/models/ggml-base.en.bin\n' '可从https://github.com/ggerganov/whisper.cpp/tree/master/models处下载', 'type': 'text', @@ -96,34 +154,77 @@ def get_fields(): }] } ], - # 文件大小 [ { - 'title': '文件大小（MB）', - 'required': "required", - 'tooltip': '单位 MB, 大于该大小的文件才会进行字幕生成', + 'title': '高级参数', + 'tooltip': 'whisper.cpp的高级参数，请勿随意修改', + 'required': "", 'type': 'text', - 'content': - [{ - 'id': 'file_size', - 'placeholder': '文件大小, 单位MB' - }] + 'content': [ + { + 'id': 'additional_args', + 'placeholder': '-t 4 -p 1' + } + ] } - ], + ] + ] + }, + { + 'type': 'details', + 'id': 'faster_whisper_config', + 'summary': 'faster-whisper 配置', + 'tooltip': '使用 faster-whisper 引擎时的配置，安装参考 https://github.com/guillaumekln/faster-whisper', + 'hidden': True, + 'content': [ [ { - 'title': '媒体路径', - 'required': '', - 'tooltip': '要进行字幕生成的路径，每行一个路径，请确保路径正确', - 'type': 'textarea', - 'content': + 'title': '模型', + 'required': "", + 'tooltip': '选择模型后第一次运行会从Hugging Face Hub下载模型，可能需要一段时间', + 'type': 'select', + 'content': [ { - 'id': 'path_list', - 'placeholder': '文件路径', - 'rows': 5 + 'id': 'faster_whisper_model', + 'options': { + # tiny, tiny.en, base, base.en, + # small, small.en, medium, medium.en, + # large-v1, or large-v2 + 'tiny': 'tiny', + 'tiny.en': 'tiny.en', + 'base': 'base', + 'base.en': 'base.en', + 'small': 'small', + 'small.en': 'small.en', + 'medium': 'medium', + 'medium.en': 'medium.en', + 'large-v1': 'large-v1', + 'large-v2': 'large-v2', + }, + 'default': 'base' } + ] } ], + [ + { + 'title': '模型保存路径', + 'required': "", + 'tooltip': '配置模型保存路径，如/config/plugin/autosub/faster-whisper/models', + 'type': 'text', + 'content': [ + { + 'id': 'faster_whisper_model_path', + 'placeholder': 'faster-whisper配置模型保存路径' + } + ] + } + ] + ] + }, + { + 'type': 'div', + 'content': [ [ { 'title': '立即运行一次', @@ -157,28 +258,26 @@ def get_fields(): } ] ] - }, - { - 'type': 'details', - 'summary': '高级参数', - 'tooltip': 'whisper.cpp的高级参数，请勿随意修改', - 'content': [ - [ - { - 'required': "", - 'type': 'text', - 'content': [ - { - 'id': 'additional_args', - 'placeholder': '-t 4 -p 1' - } - ] - } - ] - ] } ] + @staticmethod + def get_script(): + """ + 返回插件额外的JS代码 + """ + return """ + function AutoSub_asr_engine_change(obj) { + if ($(obj).val() == 'faster-whisper') { + $('#autosubwhisper_config').hide(); + $('#autosubfaster_whisper_config').show(); + }else{ + $('#autosubwhisper_config').show(); + $('#autosubfaster_whisper_config').hide(); + } + } + """ + def init_config(self, config=None): # 如果没有配置信息， 则不处理 if not config: @@ -194,6 +293,9 @@ def init_config(self, config=None): self.translate_only = config.get('translate_only', False) self.additional_args = config.get('additional_args', '-t 4 -p 1') self.send_notify = config.get('send_notify', False) + self.asr_engine = config.get('asr_engine', 'whisper.cpp') + self.faster_whisper_model = config.get('faster_whisper_model', 'base') + self.faster_whisper_model_path = config.get('faster_whisper_model_path') run_now = config.get('run_now') if not run_now: @@ -203,28 +305,19 @@ def init_config(self, config=None): self.update_config(config) # 如果没有配置信息， 则不处理 - if not path_list or not self.file_size or not self.whisper_main or not self.whisper_model: + if not path_list or not self.file_size: self.warn(f"配置信息不完整，不进行处理") return - if not os.path.exists(self.whisper_main): - self.warn(f"whisper.cpp主程序不存在，不进行处理") - return - - if not os.path.exists(self.whisper_model): - self.warn(f"whisper.cpp模型文件不存在，不进行处理") - return - - # 校验扩展参数是否包含异常字符 - if self.additional_args and re.search(r'[;|&]', self.additional_args): - self.warn(f"扩展参数包含异常字符，不进行处理") - return - # 校验文件大小是否为数字 if not self.file_size.isdigit(): self.warn(f"文件大小不是数字，不进行处理") return + # asr 配置检查 + if not self.translate_only and not self.__check_asr(): + return + if self._running: self.warn(f"上一次任务还未完成，不进行处理") return @@ -259,6 +352,39 @@ def init_config(self, config=None): f"成功{self.success_count} / 跳过{self.skip_count} / 失败{self.fail_count} / 共{self.process_count}") self._running = False + def __check_asr(self): + if self.asr_engine == 'whisper.cpp': + if not self.whisper_main or not self.whisper_model: + self.warn(f"配置信息不完整，不进行处理") + return + if not os.path.exists(self.whisper_main): + self.warn(f"whisper.cpp主程序不存在，不进行处理") + return False + if not os.path.exists(self.whisper_model): + self.warn(f"whisper.cpp模型文件不存在，不进行处理") + return False + # 校验扩展参数是否包含异常字符 + if self.additional_args and re.search(r'[;|&]', self.additional_args): + self.warn(f"扩展参数包含异常字符，不进行处理") + return False + elif self.asr_engine == 'faster-whisper': + if not self.faster_whisper_model_path or not self.faster_whisper_model: + self.warn(f"配置信息不完整，不进行处理") + return + if not os.path.exists(self.faster_whisper_model_path): + self.warn(f"faster-whisper模型文件夹不存在，不进行处理") + return False + try: + from faster_whisper import WhisperModel, download_model + except ImportError: + self.warn(f"faster-whisper 未安装，不进行处理") + return False + return True + else: + self.warn(f"未配置asr引擎，不进行处理") + return False + return True + def __process_folder_subtitle(self, path): """ 处理目录字幕 @@ -331,6 +457,76 @@ def __process_folder_subtitle(self, path): traceback.print_exc() self.fail_count += 1 + def __do_speech_recognition(self, audio_lang, audio_file): + """ + 语音识别, 生成字幕 + :param audio_lang: + :param audio_file: + :return: + """ + lang = audio_lang + if self.asr_engine == 'whisper.cpp': + command = [self.whisper_main] + self.additional_args.split() + command += ['-l', lang, '-m', self.whisper_model, '-osrt', '-of', audio_file, audio_file] + ret = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if ret.returncode == 0: + if lang == 'auto': + # 从output中获取语言 "whisper_full_with_state: auto-detected language: en (p = 0.973642)" + output = ret.stdout.decode('utf-8') if ret.stdout else "" + lang = re.search(r"auto-detected language: (\w+)", output) + if lang and lang.group(1): + lang = lang.group(1) + else: + lang = "en" + return True, lang + elif self.asr_engine == 'faster-whisper': + try: + from faster_whisper import WhisperModel, download_model + # 设置缓存目录, 防止缓存同目录出现 cross-device 错误 + cache_dir = os.path.join(self.faster_whisper_model_path, "cache") + if not os.path.exists(cache_dir): + os.mkdir(cache_dir) + os.environ["HUGGINGFACE_HUB_CACHE"] = cache_dir + model = WhisperModel(download_model(self.faster_whisper_model), + device="cpu", compute_type="int8", cpu_threads=psutil.cpu_count(logical=False)) + segments, info = model.transcribe(audio_file, + language=lang if lang != 'auto' else None, + word_timestamps=True, + temperature=0, + beam_size=5) + if lang == 'auto': + lang = info.language + + subs = [] + if lang in ['en', 'eng']: + # 英文先生成单词级别字幕，再合并 + idx = 0 + for segment in segments: + for word in segment.words: + idx += 1 + subs.append(srt.Subtitle(index=idx, + start=timedelta(seconds=word.start), + end=timedelta(seconds=word.end), + content=word.word)) + subs = self.__merge_srt(subs) + else: + for i, segment in enumerate(segments): + subs.append(srt.Subtitle(index=i, + start=timedelta(seconds=segment.start), + end=timedelta(seconds=segment.end), + content=segment.text)) + + self.__save_srt(f"{audio_file}.srt", subs) + return True, lang + except ImportError: + self.warn(f"faster-whisper 未安装，不进行处理") + return False, None + except Exception as e: + traceback.print_exc() + self.error(f"faster-whisper 处理异常：{e}") + return False, None + return False, None + def __generate_subtitle(self, video_file, subtitle_file, only_extract=False): """ 生成字幕 @@ -399,27 +595,16 @@ def __generate_subtitle(self, video_file, subtitle_file, only_extract=False): self.info(f"提取音频完成：{audio_file.name}") # 生成字幕 - command = [self.whisper_main] + self.additional_args.split() - command += ['-l', audio_lang, '-m', self.whisper_model, '-osrt', '-of', audio_file.name, audio_file.name] self.info(f"开始生成字幕, 语言 {audio_lang} ...") - ret = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if ret.returncode == 0: - lang = audio_lang - if lang == 'auto': - # 从output中获取语言 "whisper_full_with_state: auto-detected language: en (p = 0.973642)" - output = ret.stdout.decode('utf-8') if ret.stdout else "" - lang = re.search(r"auto-detected language: (\w+)", output) - if lang and lang.group(1): - lang = lang.group(1) - else: - lang = "en" + ret, lang = self.__do_speech_recognition(audio_lang, audio_file.name) + if ret: self.info(f"生成字幕成功，原始语言：{lang}") # 复制字幕文件 SystemUtils.copy(f"{audio_file.name}.srt", f"{subtitle_file}.{lang}.srt") self.info(f"复制字幕文件：{subtitle_file}.{lang}.srt") # 删除临时文件 os.remove(f"{audio_file.name}.srt") - return True, lang + return ret, lang else: self.error(f"生成字幕失败") return False, None @@ -469,7 +654,7 @@ def __save_srt(file_path, srt_data): def __get_video_prefer_audio(self, video_meta, prefer_lang=None): """ 获取视频的首选音轨，如果有多音轨， 优先指定语言音轨，否则获取默认音轨 - :param video_file: + :param video_meta :return: """ if type(prefer_lang) == str and prefer_lang: @@ -564,10 +749,13 @@ def __merge_srt(self, subtitle_data): merged_subtitle = [] sentence_end = True - self.info(f"开始合并字幕语句 ...") for index, item in enumerate(subtitle_data): # 当前字幕先将多行合并为一行，再去除首尾空格 content = item.content.replace('\n', ' ').strip() + # 去除html标签 + parse = etree.HTML(content) + if parse is not None: + content = parse.xpath('string(.)') if content == '': continue item.content = content @@ -593,7 +781,6 @@ def __merge_srt(self, subtitle_data): else: sentence_end = False - self.info(f"合并字幕语句完成，合并前字幕数量：{len(subtitle_data)}, 合并后字幕数量：{len(merged_subtitle)}") return merged_subtitle def __do_translate_with_retry(self, text, retry=3): @@ -626,7 +813,11 @@ def __translate_zh_subtitle(self, source_lang, source_subtitle, dest_subtitle): srt_data = self.__load_srt(source_subtitle) # 合并字幕语句，目前带标点带英文效果较好，非英文或者无标点的需要NLP处理 if source_lang in ['en', 'eng']: - srt_data = self.__merge_srt(srt_data) + self.info(f"开始合并字幕语句 ...") + merged_data = self.__merge_srt(srt_data) + self.info(f"合并字幕语句完成，合并前字幕数量：{len(srt_data)}, 合并后字幕数量：{len(merged_data)}") + srt_data = merged_data + batch = [] max_batch_tokens = 1000 for srt_item in srt_data: diff --git a/app/plugins/modules/cloudflarespeedtest.py b/app/plugins/modules/cloudflarespeedtest.py index 6b96af57d1..3ad6666fe2 100644 --- a/app/plugins/modules/cloudflarespeedtest.py +++ b/app/plugins/modules/cloudflarespeedtest.py @@ -7,6 +7,7 @@ from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.cron import CronTrigger +from app.message import Message from app.plugins import EventManager from app.plugins.modules._base import _IPluginModule from app.utils import SystemUtils, RequestUtils, IpUtils @@ -38,6 +39,7 @@ class CloudflareSpeedTest(_IPluginModule): # 私有属性 eventmanager = None + message = None _customhosts = False _cf_ip = None _scheduler = None @@ -48,6 +50,7 @@ class CloudflareSpeedTest(_IPluginModule): _version = None _additional_args = None _re_install = False + _notify = False _cf_path = 'cloudflarespeedtest' _cf_ipv4 = 'cloudflarespeedtest/ip.txt' _cf_ipv6 = 'cloudflarespeedtest/ipv6.txt' @@ -106,6 +109,19 @@ def get_fields(): 'tooltip': '优选测速ipv6；v4和v6必须其一，都不选择则默认ipv4。选择ipv6会大大加长测速时间。', 'type': 'switch', 'id': 'ipv6', + }, + { + 'title': '', + 'required': "", + 'tooltip': '', + 'type': 'text', + 'hidden': True, + 'content': [ + { + 'id': 'version', + 'placeholder': 'CloudflareSpeedTest版本', + } + ] } ], [ @@ -122,7 +138,14 @@ def get_fields(): 'tooltip': '开启后，每次会重新下载CloudflareSpeedTest，网络不好慎选', 'type': 'switch', 'id': 're_install', - } + }, + { + 'title': '运行时通知', + 'required': "", + 'tooltip': '运行任务后会发送通知（需要打开自定义消息通知）', + 'type': 'switch', + 'id': 'notify', + }, ] ] }, @@ -141,19 +164,6 @@ def get_fields(): 'placeholder': '-dd' } ] - }, - { - 'title': '', - 'required': "", - 'tooltip': '', - 'type': 'text', - 'hidden': True, - 'content': [ - { - 'id': 'version', - 'placeholder': 'CloudflareSpeedTest版本', - } - ] } ] ] @@ -162,9 +172,7 @@ def get_fields(): def init_config(self, config=None): self.eventmanager = EventManager() - - # 停止现有任务 - self.stop_service() + self.message = Message() # 读取配置 if config: @@ -176,55 +184,52 @@ def init_config(self, config=None): self._ipv6 = config.get("ipv6") self._re_install = config.get("re_install") self._additional_args = config.get("additional_args") + self._notify = config.get("notify") - # 自定义插件hosts配置 - customHosts = self.get_config("CustomHosts") - self._customhosts = customHosts and customHosts.get("enable") + # 停止现有任务 + self.stop_service() # 启动定时任务 & 立即运行一次 - if self._cron or self._onlyonce: - # 获取自定义Hosts插件，若无设置则停止 - if self._cf_ip and not customHosts or not customHosts.get("hosts"): - self.error(f"Cloudflare CDN优选依赖于自定义Hosts，请先维护hosts") - self._onlyonce = False - self.__update_config() - return - - if not self._cf_ip: - self.error("CloudflareSpeedTest加载成功，首次运行，需要配置优选ip") - self._onlyonce = False - self.__update_config() - return - - # ipv4和ipv6必须其一 - if not self._ipv4 and not self._ipv6: - self._ipv4 = True - self.__update_config() - self.warn(f"Cloudflare CDN优选未指定ip类型，默认ipv4") - + if self.get_state() or self._onlyonce: self._scheduler = BackgroundScheduler(timezone=Config().get_timezone()) if self._cron: + self.info(f"Cloudflare CDN优选服务启动，周期：{self._cron}") self._scheduler.add_job(self.__cloudflareSpeedTest, CronTrigger.from_crontab(self._cron)) - if self._onlyonce: - self._scheduler.add_job(self.__cloudflareSpeedTest, 'date', - run_date=datetime.now(tz=pytz.timezone(Config().get_timezone()))) - self._scheduler.print_jobs() - self._scheduler.start() if self._onlyonce: self.info(f"Cloudflare CDN优选服务启动，立即运行一次") - if self._cron: - self.info(f"Cloudflare CDN优选服务启动，周期：{self._cron}") + self._scheduler.add_job(self.__cloudflareSpeedTest, 'date', + run_date=datetime.now(tz=pytz.timezone(Config().get_timezone()))) + # 关闭一次性开关 + self._onlyonce = False + self.__update_config() - # 关闭一次性开关 - self._onlyonce = False - self.__update_config() + if self._cron or self._onlyonce: + # 启动服务 + self._scheduler.print_jobs() + self._scheduler.start() def __cloudflareSpeedTest(self): """ CloudflareSpeedTest优选 """ + # 获取自定义Hosts插件，若无设置则停止 customHosts = self.get_config("CustomHosts") + self._customhosts = customHosts and customHosts.get("enable") + if self._cf_ip and not customHosts or not customHosts.get("hosts"): + self.error(f"Cloudflare CDN优选依赖于自定义Hosts，请先维护hosts") + return + + if not self._cf_ip: + self.error("CloudflareSpeedTest加载成功，首次运行，需要配置优选ip") + return + + # ipv4和ipv6必须其一 + if not self._ipv4 and not self._ipv6: + self._ipv4 = True + self.__update_config() + self.warn(f"Cloudflare CDN优选未指定ip类型，默认ipv4") + err_flag, release_version = self.__check_envirment() if err_flag and release_version: # 更新版本 @@ -274,9 +279,10 @@ def __cloudflareSpeedTest(self): }, "CustomHosts") # 更新优选ip + old_ip = self._cf_ip self._cf_ip = best_ip self.__update_config() - self.info(f"CLoudflare CDN优选ip [{best_ip}] 已替换自定义Hosts插件") + self.info(f"Cloudflare CDN优选ip [{best_ip}] 已替换自定义Hosts插件") # 解发自定义hosts插件重载 self.info("通知CustomHosts插件重载 ...") @@ -284,6 +290,12 @@ def __cloudflareSpeedTest(self): { "plugin_id": "CustomHosts" }) + if self._notify: + self.message.send_custom_message( + title="【Cloudflare优选任务完成】", + text=f"原ip：{old_ip}\n" + f"新ip：{best_ip}" + ) else: self.error("获取到最优ip格式错误，请重试") self._onlyonce = False @@ -419,7 +431,8 @@ def __update_config(self): "ipv4": self._ipv4, "ipv6": self._ipv6, "re_install": self._re_install, - "additional_args": self._additional_args + "additional_args": self._additional_args, + "notify": self._notify }) @staticmethod @@ -440,7 +453,7 @@ def __get_release_version(): return None def get_state(self): - return self._customhosts and self._cf_ip and True if self._cron else False + return self._cf_ip and True if self._cron else False def stop_service(self): """ diff --git a/app/plugins/modules/cookiecloud.py b/app/plugins/modules/cookiecloud.py new file mode 100644 index 0000000000..d343bd9fa7 --- /dev/null +++ b/app/plugins/modules/cookiecloud.py @@ -0,0 +1,336 @@ +from datetime import datetime +from threading import Event +from collections import defaultdict + +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.cron import CronTrigger + +import pytz + +import log +from app.helper import DbHelper, IndexerHelper +from app.message import Message +from app.plugins.modules._base import _IPluginModule +from app.sites import Sites +from app.utils import RequestUtils +from config import Config + + +class CookieCloud(_IPluginModule): + # 插件名称 + module_name = "CookieCloud同步" + # 插件描述 + module_desc = "从CookieCloud云端同步数据，自动新增站点或更新已有站点Cookie。" + # 插件图标 + module_icon = "cloud.png" + # 主题色 + module_color = "#77B3D4" + # 插件版本 + module_version = "1.0" + # 插件作者 + module_author = "jxxghp" + # 作者主页 + author_url = "https://github.com/jxxghp" + # 插件配置项ID前缀 + module_config_prefix = "cookiecloud_" + # 加载顺序 + module_order = 21 + # 可使用的用户级别 + auth_level = 2 + + # 私有属性 + _scheduler = None + _site = None + _dbhelper = None + _message = None + _index_helper = None + # 设置开关 + _req = None + _server = None + _key = None + _password = None + _enabled = False + # 任务执行间隔 + _cron = None + _onlyonce = False + # 通知 + _notify = False + # 退出事件 + _event = Event() + + @staticmethod + def get_fields(): + return [ + # 同一板块 + { + 'type': 'div', + 'content': [ + # 同一行 + [ + { + 'title': '服务器地址', + 'required': "required", + 'tooltip': '参考https://github.com/easychen/CookieCloud搭建私有CookieCloud服务器；也可使用默认的公共服务器，公共服务器不会存储任何非加密用户数据，也不会存储用户KEY、端对端加密密码，但要注意千万不要对外泄露加密信息，否则Cookie数据也会被泄露！', + 'type': 'text', + 'content': [ + { + 'id': 'server', + 'placeholder': 'http://nastool.cn:8088' + } + ] + + }, + { + 'title': '执行周期', + 'required': "", + 'tooltip': '设置自动同步时间周期，支持5位cron表达式', + 'type': 'text', + 'content': [ + { + 'id': 'cron', + 'placeholder': '0 0 0 ? *', + } + ] + }, + ] + ] + }, + { + 'type': 'div', + 'content': [ + # 同一行 + [ + { + 'title': '用户KEY', + 'required': 'required', + 'tooltip': '浏览器CookieCloud插件中获取，使用公共服务器时注意不要泄露该信息', + 'type': 'text', + 'content': [ + { + 'id': 'key', + 'placeholder': '', + } + ] + }, + { + 'title': '端对端加密密码', + 'required': "", + 'tooltip': '浏览器CookieCloud插件中获取，使用公共服务器时注意不要泄露该信息', + 'type': 'text', + 'content': [ + { + 'id': 'password', + 'placeholder': '' + } + ] + } + ] + ] + }, + { + 'type': 'div', + 'content': [ + # 同一行 + [ + { + 'title': '运行时通知', + 'required': "", + 'tooltip': '运行任务后会发送通知（需要打开自定义消息通知）', + 'type': 'switch', + 'id': 'notify', + }, + { + 'title': '立即运行一次', + 'required': "", + 'tooltip': '打开后立即运行一次（点击此对话框的确定按钮后即会运行，周期未设置也会运行），关闭后将仅按照定时周期运行（同时上次触发运行的任务如果在运行中也会停止）', + 'type': 'switch', + 'id': 'onlyonce', + } + ] + ] + } + ] + + def init_config(self, config=None): + self._dbhelper = DbHelper() + self._site = Sites() + self._message = Message() + self._index_helper = IndexerHelper() + + # 读取配置 + if config: + self._server = config.get("server") + self._cron = config.get("cron") + self._key = config.get("key") + self._password = config.get("password") + self._notify = config.get("notify") + self._onlyonce = config.get("onlyonce") + self._req = RequestUtils(content_type="application/json") + if self._server: + if not self._server.startswith("http"): + self._server = "http://%s" % self._server + if self._server.endswith("/"): + self._server = self._server[:-1] + + # 测试 + _, msg, flag = self.__download_data() + if flag: + self._enabled = True + else: + self._enabled = False + self.info(msg) + + # 停止现有任务 + self.stop_service() + + # 启动服务 + if self._enabled: + self._scheduler = BackgroundScheduler(timezone=Config().get_timezone()) + + # 运行一次 + if self._onlyonce: + self.info(f"同步服务启动，立即运行一次") + self._scheduler.add_job(self.__cookie_sync, 'date', + run_date=datetime.now(tz=pytz.timezone(Config().get_timezone()))) + # 关闭一次性开关 + self._onlyonce = False + self.update_config({ + "server": self._server, + "cron": self._cron, + "key": self._key, + "password": self._password, + "notify": self._notify, + "onlyonce": self._onlyonce, + }) + + # 周期运行 + if self._cron: + self.info(f"同步服务启动，周期：{self._cron}") + self._scheduler.add_job(self.__cookie_sync, + CronTrigger.from_crontab(self._cron)) + + # 启动任务 + if self._scheduler.get_jobs(): + self._scheduler.print_jobs() + self._scheduler.start() + + def get_state(self): + return self._enabled and self._cron + + def __download_data(self) -> [dict, str, bool]: + """ + 从CookieCloud下载数据 + """ + if not self._server or not self._key or not self._password: + return {}, "CookieCloud参数不正确", False + req_url = "%s/get/%s" % (self._server, self._key) + ret = self._req.post_res(url=req_url, json={"password": self._password}) + if ret and ret.status_code == 200: + result = ret.json() + if not result: + return {}, "", True + if result.get("cookie_data"): + return result.get("cookie_data"), "", True + return result, "", True + elif ret: + return {}, "同步CookieCloud失败，错误码：%s" % ret.status_code, False + else: + return {}, "CookieCloud请求失败，请检查服务器地址、用户KEY及加密密码是否正确", False + + def __cookie_sync(self): + """ + 同步站点Cookie + """ + # 同步数据 + self.info(f"同步服务开始 ...") + contents, msg, _ = self.__download_data() + if not contents: + log.error(msg) + self.__send_message(msg) + return + # 整理数据,使用domain域名的最后两级作为分组依据 + domain_groups = defaultdict(list) + for site, cookies in contents.items(): + for cookie in cookies: + domain_parts = cookie["domain"].split(".")[-2:] + domain_key = tuple(domain_parts) + domain_groups[domain_key].append(cookie) + # 计数 + update_count = 0 + add_count = 0 + # 索引器 + for domain, content_list in domain_groups.items(): + if self._event.is_set(): + self.info(f"同步服务停止") + return + if not content_list: + continue + # 域名 + domain_url = ".".join(domain) + # 只有cf的cookie过滤掉 + cloudflare_cookie = True + for content in content_list: + if content["name"] != "cf_clearance": + cloudflare_cookie = False + break + if cloudflare_cookie: + continue + # Cookie + cookie_str = ";".join( + [f"{content['name']}={content['value']}" for content in content_list] + ) + # 查询站点 + site_info = self._site.get_sites_by_suffix(domain_url) + if site_info: + # 已存在的站点更新Cookie + self._dbhelper.update_site_cookie_ua(tid=site_info.get("id"), cookie=cookie_str) + update_count += 1 + else: + # 查询是否在索引器范围 + indexer_info = self._index_helper.get_indexer_info(domain_url) + if indexer_info: + # 支持则新增站点 + site_pri = self._site.get_max_site_pri() + 1 + self._dbhelper.insert_config_site( + name=indexer_info.get("name"), + site_pri=site_pri, + signurl=indexer_info.get("domain"), + cookie=cookie_str, + rss_uses='QT' + ) + add_count += 1 + # 发送消息 + if update_count or add_count: + # 重载站点信息 + self._site.init_config() + msg = f"更新了 {update_count} 个站点的Cookie数据，新增了 {add_count} 个站点" + else: + msg = f"同步完成，但未更新任何站点数据！" + self.info(msg) + # 发送消息 + if self._notify: + self.__send_message(msg) + + def __send_message(self, msg): + """ + 发送通知 + """ + self._message.send_custom_message( + title="【CookieCloud同步任务执行完成】", + text=f"{msg}" + ) + + def stop_service(self): + """ + 退出插件 + """ + try: + if self._scheduler: + self._scheduler.remove_all_jobs() + if self._scheduler.running: + self._event.set() + self._scheduler.shutdown() + self._event.clear() + self._scheduler = None + except Exception as e: + print(str(e)) diff --git a/app/plugins/modules/doubanrank.py b/app/plugins/modules/doubanrank.py index 45bb6fe944..e68a415410 100644 --- a/app/plugins/modules/doubanrank.py +++ b/app/plugins/modules/doubanrank.py @@ -59,6 +59,7 @@ class DoubanRank(_IPluginModule): _cron = "" _rss_addrs = [] _ranks = [] + _vote = 0 _scheduler = None def init_config(self, config: dict = None): @@ -69,6 +70,7 @@ def init_config(self, config: dict = None): self._enable = config.get("enable") self._onlyonce = config.get("onlyonce") self._cron = config.get("cron") + self._vote = int(config.get("vote")) if config.get("vote") else 0 rss_addrs = config.get("rss_addrs") if rss_addrs: if isinstance(rss_addrs, str): @@ -86,14 +88,13 @@ def init_config(self, config: dict = None): if self.get_state() or self._onlyonce: self._scheduler = BackgroundScheduler(timezone=Config().get_timezone()) if self._cron: - self._scheduler.add_job(self.__refresh_rss, CronTrigger.from_crontab(self._cron)) + self.info(f"订阅服务启动，周期：{self._cron}") + self._scheduler.add_job(self.__refresh_rss, + CronTrigger.from_crontab(self._cron)) if self._onlyonce: + self.info(f"订阅服务启动，立即运行一次") self._scheduler.add_job(self.__refresh_rss, 'date', run_date=datetime.now(tz=pytz.timezone(Config().get_timezone()))) - self._scheduler.print_jobs() - self._scheduler.start() - if self._onlyonce: - self.info(f"订阅服务启动，立即运行一次") # 关闭一次性开关 self._onlyonce = False self.update_config({ @@ -101,10 +102,13 @@ def init_config(self, config: dict = None): "enable": self._enable, "cron": self._cron, "ranks": self._ranks, + "vote": self._vote, "rss_addrs": "\n".join(self._rss_addrs) }) - if self._cron: - self.info(f"订阅服务启动，周期：{self._cron}") + if self._scheduler.get_jobs(): + # 启动服务 + self._scheduler.print_jobs() + self._scheduler.start() def get_state(self): return self._enable and self._cron and (self._ranks or self._rss_addrs) @@ -147,6 +151,18 @@ def get_fields(): 'placeholder': '0 0 0 ? *', } ] + }, + { + 'title': '评分', + 'required': "", + 'tooltip': '大于该评分的才会被订阅，不填则不限制', + 'type': 'text', + 'content': [ + { + 'id': 'vote', + 'placeholder': '0', + } + ] } ], [ @@ -254,9 +270,14 @@ def __refresh_rss(self): return # 识别媒体信息 media_info = WebUtils.get_mediainfo_from_id(mtype=rss_info.get("type"), - mediaid=f"DB:{rss_info.get('doubanid')}") + mediaid=f"DB:{rss_info.get('doubanid')}", + wait=True) if not media_info: - self.warn(f"未查询到TMDB媒体信息：{rss_info.get('doubanid')} - {rss_info.get('title')}") + self.warn(f"未查询到媒体信息：{rss_info.get('doubanid')} - {rss_info.get('title')}") + continue + if self._vote and media_info.vote_average \ + and media_info.vote_average < self._vote: + self.info(f"评分低于限制：{media_info.get_title_string()}，跳过 ．．．") continue # 检查媒体服务器是否存在 item_id = self.mediaserver.check_item_exists(mtype=media_info.type, diff --git a/app/plugins/modules/doubansync.py b/app/plugins/modules/doubansync.py index 4d11c752a7..44664fa3fa 100644 --- a/app/plugins/modules/doubansync.py +++ b/app/plugins/modules/doubansync.py @@ -103,18 +103,18 @@ def init_config(self, config: dict = None): if self.get_state() or self._onlyonce: self._scheduler = BackgroundScheduler(timezone=Config().get_timezone()) if self._interval: - self._scheduler.add_job(self.sync, 'interval', hours=self._interval) + self.info(f"订阅服务启动，周期：{self._interval} 小时，类型：{self._types}，用户：{self._users}") + self._scheduler.add_job(self.sync, 'interval', + hours=self._interval) + if self._onlyonce: + self.info(f"同步服务启动，立即运行一次") self._scheduler.add_job(self.sync, 'date', run_date=datetime.now(tz=pytz.timezone(Config().get_timezone()))) - self._scheduler.print_jobs() - self._scheduler.start() - if self._onlyonce: - self.info(f"同步服务启动，立即运行一次") # 关闭一次性开关 self._onlyonce = False self.update_config({ - "onlyonce": False, + "onlyonce": self._onlyonce, "enable": self._enable, "interval": self._interval, "auto_search": self._auto_search, @@ -124,8 +124,10 @@ def init_config(self, config: dict = None): "days": self._days, "types": self._types }) - if self._interval: - self.info(f"订阅服务启动，周期：{self._interval} 小时，类型：{self._types}，用户：{self._users}") + if self._scheduler.get_jobs(): + # 启动服务 + self._scheduler.print_jobs() + self._scheduler.start() def get_state(self): return self._enable \ diff --git a/app/plugins/modules/iyuuautoseed.py b/app/plugins/modules/iyuuautoseed.py new file mode 100644 index 0000000000..30b56a62d4 --- /dev/null +++ b/app/plugins/modules/iyuuautoseed.py @@ -0,0 +1,663 @@ +import re +from copy import deepcopy +from datetime import datetime +from threading import Event + +import pytz +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.cron import CronTrigger +from lxml import etree + +from app.downloader import Downloader +from app.helper import IyuuHelper +from app.media.meta import MetaInfo +from app.message import Message +from app.plugins.modules._base import _IPluginModule +from app.sites import Sites +from app.utils import RequestUtils +from app.utils.types import DownloaderType +from config import Config + + +class IYUUAutoSeed(_IPluginModule): + # 插件名称 + module_name = "IYUU自动辅种" + # 插件描述 + module_desc = "基于IYUU官方Api实现自动辅种。" + # 插件图标 + module_icon = "iyuu.png" + # 主题色 + module_color = "#F3B70B" + # 插件版本 + module_version = "1.0" + # 插件作者 + module_author = "jxxghp" + # 作者主页 + author_url = "https://github.com/jxxghp" + # 插件配置项ID前缀 + module_config_prefix = "iyuuautoseed_" + # 加载顺序 + module_order = 20 + # 可使用的用户级别 + user_level = 1 + + # 私有属性 + _scheduler = None + downloader = None + iyuuhelper = None + sites = None + message = None + # 限速开关 + _enable = False + _cron = None + _onlyonce = False + _token = None + _downloaders = [] + _sites = [] + _notify = False + _nolabels = None + _clearcache = False + # 退出事件 + _event = Event() + # 种子链接xpaths + _torrent_xpaths = [ + "//form[contains(@action, 'download.php?id=')]/@action", + "//a[contains(@href, 'download.php?hash=')]/@href", + "//a[contains(@href, 'download.php?id=')]/@href", + "//a[@class='index'][contains(@href, '/dl/')]/@href", + ] + _torrent_tags = ["已整理", "辅种"] + # 待校全种子hash清单 + _recheck_torrents = {} + _is_recheck_running = False + # 辅助缓存，出错的种子不再重复辅种 + _error_caches = [] + # 辅种计数 + total = 0 + realtotal = 0 + success = 0 + exist = 0 + fail = 0 + cached = 0 + + @staticmethod + def get_fields(): + downloaders = {k: v for k, v in Downloader().get_downloader_conf_simple().items() + if v.get("type") in ["qbittorrent", "transmission"] and v.get("enabled")} + sites = {site.get("id"): site for site in Sites().get_site_dict()} + return [ + # 同一板块 + { + 'type': 'div', + 'content': [ + # 同一行 + [ + { + 'title': '开启自动辅种', + 'required': "", + 'tooltip': '开启后，自动监控下载器，对下载完成的任务根据执行周期自动辅种，辅种任务会自动暂停，校验通过且完整后才开始做种。', + 'type': 'switch', + 'id': 'enable', + } + ], + [ + { + 'title': 'IYUU Token', + 'required': "required", + 'tooltip': '登录IYUU使用的Token，用于调用IYUU官方Api', + 'type': 'text', + 'content': [ + { + 'id': 'token', + 'placeholder': 'IYUUxxx', + } + ] + }, + { + 'title': '执行周期', + 'required': "required", + 'tooltip': '辅种任务执行的时间周期，支持5位cron表达式；应避免任务执行过于频繁', + 'type': 'text', + 'content': [ + { + 'id': 'cron', + 'placeholder': '0 0 0 ? *', + } + ] + } + ], + [ + { + 'title': '不辅种标签', + 'required': "", + 'tooltip': '下载器中的种子有以下标签时不进行辅种，多个标签使用英文,分隔', + 'type': 'text', + 'content': [ + { + 'id': 'nolabels', + 'placeholder': '使用,分隔多个标签', + } + ] + } + ] + ] + }, + { + 'type': 'details', + 'summary': '辅种下载器', + 'tooltip': '只有选中的下载器才会执行辅种任务', + 'content': [ + # 同一行 + [ + { + 'id': 'downloaders', + 'type': 'form-selectgroup', + 'content': downloaders + }, + ] + ] + }, + { + 'type': 'details', + 'summary': '辅种站点', + 'tooltip': '只有选中的站点才会执行辅种任务，不选则默认为全选', + 'content': [ + # 同一行 + [ + { + 'id': 'sites', + 'type': 'form-selectgroup', + 'content': sites + }, + ] + ] + }, + { + 'type': 'div', + 'content': [ + # 同一行 + [ + { + 'title': '运行时通知', + 'required': "", + 'tooltip': '运行辅助任务后会发送通知（需要打开自定义消息通知）', + 'type': 'switch', + 'id': 'notify', + }, + { + 'title': '立即运行一次', + 'required': "", + 'tooltip': '打开后立即运行一次（点击此对话框的确定按钮后即会运行，周期未设置也会运行），关闭后将仅按照刮削周期运行（同时上次触发运行的任务如果在运行中也会停止）', + 'type': 'switch', + 'id': 'onlyonce', + }, + { + 'title': '下一次运行时清除缓存', + 'required': "", + 'tooltip': '打开后下一次运行前会先清除辅种缓存，辅种出错的种子会重新尝试辅种，此开关仅生效一次', + 'type': 'switch', + 'id': 'clearcache', + } + ] + ] + } + ] + + def init_config(self, config=None): + self.downloader = Downloader() + self.sites = Sites() + self.message = Message() + # 读取配置 + if config: + self._enable = config.get("enable") + self._onlyonce = config.get("onlyonce") + self._cron = config.get("cron") + self._token = config.get("token") + self._downloaders = config.get("downloaders") + self._sites = config.get("sites") + self._notify = config.get("notify") + self._nolabels = config.get("nolabels") + self._clearcache = config.get("clearcache") + self._error_caches = [] if self._clearcache else config.get("error_caches") or [] + # 停止现有任务 + self.stop_service() + + # 启动定时任务 & 立即运行一次 + if self.get_state() or self._onlyonce: + self.iyuuhelper = IyuuHelper(token=self._token) + self._scheduler = BackgroundScheduler(timezone=Config().get_timezone()) + if self._cron: + self.info(f"辅种服务启动，周期：{self._cron}") + self._scheduler.add_job(self.auto_seed, + CronTrigger.from_crontab(self._cron)) + if self._onlyonce: + self.info(f"辅种服务启动，立即运行一次") + self._scheduler.add_job(self.auto_seed, 'date', + run_date=datetime.now(tz=pytz.timezone(Config().get_timezone()))) + # 关闭一次性开关 + self._onlyonce = False + if self._clearcache: + # 关闭清除缓存开关 + self._clearcache = False + + if self._clearcache or self._onlyonce: + # 保存配置 + self.__update_config() + + if self._scheduler.get_jobs(): + # 追加种子校验服务 + self._scheduler.add_job(self.check_recheck, 'interval', minutes=3) + # 启动服务 + self._scheduler.print_jobs() + self._scheduler.start() + + def get_state(self): + return True if self._enable and self._cron and self._token and self._downloaders else False + + def __update_config(self): + self.update_config({ + "enable": self._enable, + "onlyonce": self._onlyonce, + "clearcache": self._clearcache, + "cron": self._cron, + "token": self._token, + "downloaders": self._downloaders, + "sites": self._sites, + "notify": self._notify, + "nolabels": self._nolabels, + "error_caches": self._error_caches + }) + + def auto_seed(self): + """ + 开始辅种 + """ + if not self._enable or not self._token or not self._downloaders: + self.warn("辅种服务未启用或未配置") + return + if not self.iyuuhelper: + return + self.info("开始辅种任务 ...") + # 扫描下载器辅种 + for downloader in self._downloaders: + self.info(f"开始扫描下载器 {downloader} ...") + # 下载器类型 + downloader_type = self.downloader.get_downloader_type(downloader_id=downloader) + # 获取下载器中已完成的种子 + torrents = self.downloader.get_completed_torrents(downloader_id=downloader) + if torrents: + self.info(f"下载器 {downloader} 已完成种子数：{len(torrents)}") + else: + self.info(f"下载器 {downloader} 没有已完成种子") + continue + hash_strs = [] + for torrent in torrents: + if self._event.is_set(): + self.info(f"辅种服务停止") + return + # 获取种子hash + hash_str = self.__get_hash(torrent, downloader_type) + if hash_str in self._error_caches: + self.info(f"种子 {hash_str} 辅种失败且已缓存，跳过 ...") + continue + save_path = self.__get_save_path(torrent, downloader_type) + # 获取种子标签 + torrent_labels = self.__get_label(torrent, downloader_type) + if self._nolabels \ + and torrent_labels \ + and set(self._nolabels.split(',')).intersection(set(torrent_labels)): + self.info(f"种子 {hash_str} 含有不辅种标签，跳过 ...") + continue + hash_strs.append({ + "hash": hash_str, + "save_path": save_path + }) + if hash_strs: + self.info(f"总共需要辅种的种子数：{len(hash_strs)}") + # 分组处理，减少IYUU Api请求次数 + chunk_size = 200 + for i in range(0, len(hash_strs), chunk_size): + # 切片操作 + chunk = hash_strs[i:i + chunk_size] + # 处理分组 + self.__seed_torrents(hash_strs=chunk, + downloader=downloader) + # 触发校验检查 + self.check_recheck() + else: + self.info(f"没有需要辅种的种子") + # 保存缓存 + self.__update_config() + # 发送消息 + if self._notify: + self.message.send_custom_message( + title="【IYUU自动辅种任务完成】", + text=f"服务器返回可辅种总数：{self.total}\n" + f"实际可辅种数：{self.realtotal}\n" + f"已存在：{self.exist}\n" + f"成功：{self.success}\n" + f"失败：{self.fail}\n" + f"{self.cached} 条失败记录已加入缓存" + ) + self.info("辅种任务执行完成") + + def check_recheck(self): + """ + 定时检查下载器中种子是否校验完成，校验完成且完整的自动开始辅种 + """ + if not self._recheck_torrents: + return + if self._is_recheck_running: + return + self._is_recheck_running = True + for downloader in self._downloaders: + # 需要检查的种子 + recheck_torrents = self._recheck_torrents.get(downloader) or [] + if not recheck_torrents: + continue + self.info(f"开始检查下载器 {downloader} 的校验任务 ...") + # 下载器类型 + downloader_type = self.downloader.get_downloader_type(downloader_id=downloader) + # 获取下载器中的种子 + torrents = self.downloader.get_torrents(downloader_id=downloader, + ids=recheck_torrents) + if torrents: + can_seeding_torrents = [] + for torrent in torrents: + # 获取种子hash + hash_str = self.__get_hash(torrent, downloader_type) + if self.__can_seeding(torrent, downloader_type): + can_seeding_torrents.append(hash_str) + if can_seeding_torrents: + self.info(f"共 {len(can_seeding_torrents)} 个任务校验完成，开始辅种 ...") + self.downloader.start_torrents(downloader_id=downloader, ids=can_seeding_torrents) + # 去除已经处理过的种子 + self._recheck_torrents[downloader] = list( + set(recheck_torrents).difference(set(can_seeding_torrents))) + elif torrents is None: + self.info(f"下载器 {downloader} 查询校验任务失败，将在下次继续查询 ...") + continue + else: + self.info(f"下载器 {downloader} 中没有需要检查的校验任务，清空待处理列表 ...") + self._recheck_torrents[downloader] = [] + self._is_recheck_running = False + + def __seed_torrents(self, hash_strs: list, downloader): + """ + 执行一批种子的辅种 + """ + if not hash_strs: + return + self.info(f"下载器 {downloader} 开始查询辅种，数量：{len(hash_strs)} ...") + # 下载器中的Hashs + hashs = [item.get("hash") for item in hash_strs] + # 每个Hash的保存目录 + save_paths = {} + for item in hash_strs: + save_paths[item.get("hash")] = item.get("save_path") + # 查询可辅种数据 + seed_list, msg = self.iyuuhelper.get_seed_info(hashs) + if not isinstance(seed_list, dict): + self.warn(f"当前种子列表没有可辅种的站点：{msg}") + return + else: + self.info(f"IYUU返回可辅种数：{len(seed_list)}") + # 遍历 + for current_hash, seed_info in seed_list.items(): + if not seed_info: + continue + seed_torrents = seed_info.get("torrent") + if not isinstance(seed_torrents, list): + seed_torrents = [seed_torrents] + for seed in seed_torrents: + if not seed: + continue + if not isinstance(seed, dict): + continue + if not seed.get("sid") or not seed.get("info_hash"): + continue + if seed.get("info_hash") in hashs: + self.info(f"{seed.get('info_hash')} 已在下载器中，跳过 ...") + continue + if seed.get("info_hash") in self._error_caches: + self.info(f"种子 {seed.get('info_hash')} 辅种失败且已缓存，跳过 ...") + continue + # 添加任务 + self.__download_torrent(seed=seed, + downloader=downloader, + save_path=save_paths.get(current_hash)) + self.info(f"下载器 {downloader} 辅种完成") + + def __download_torrent(self, seed, downloader, save_path): + """ + 下载种子 + torrent: { + "sid": 3, + "torrent_id": 377467, + "info_hash": "a444850638e7a6f6220e2efdde94099c53358159" + } + """ + self.total += 1 + # 获取种子站点及下载地址模板 + site_url, download_page = self.iyuuhelper.get_torrent_url(seed.get("sid")) + if not site_url or not download_page: + # 加入缓存 + self._error_caches.append(seed.get("info_hash")) + self.fail += 1 + self.cached += 1 + return + # 查询站点 + site_info = self.sites.get_sites(siteurl=site_url) + if not site_info: + self.debug(f"没有维护种子对应的站点：{site_url}") + return + if self._sites and str(site_info.get("id")) not in self._sites: + self.info("当前站点不在选择的辅助站点范围，跳过 ...") + return + self.realtotal += 1 + # 查询hash值是否已经在下载器中 + torrent_info = self.downloader.get_torrents(downloader_id=downloader, + ids=[seed.get("info_hash")]) + if torrent_info: + self.debug(f"{seed.get('info_hash')} 已在下载器中，跳过 ...") + self.exist += 1 + return + # 站点流控 + if self.sites.check_ratelimit(site_info.get("id")): + self.fail += 1 + return + # 下载种子 + torrent_url = self.__get_download_url(seed=seed, + site=site_info, + base_url=download_page) + if not torrent_url: + # 加入失败缓存 + self._error_caches.append(seed.get("info_hash")) + self.fail += 1 + self.cached += 1 + return + meta_info = MetaInfo(title="IYUU自动辅种") + meta_info.set_torrent_info(site=site_info.get("name"), + enclosure=torrent_url) + # 辅种任务默认暂停，关闭自动管理模式 + _, download_id, retmsg = self.downloader.download( + media_info=meta_info, + is_paused=True, + tag=deepcopy(self._torrent_tags), + downloader_id=downloader, + download_dir=save_path, + download_setting="-2", + is_auto=False + ) + if not download_id: + # 下载失败 + self.warn(f"添加下载任务出错，" + f"错误原因：{retmsg or '下载器添加任务失败'}，" + f"种子链接：{torrent_url}") + self.fail += 1 + return + else: + self.success += 1 + # 追加校验任务 + self.info(f"添加校验检查任务：{download_id} ...") + if not self._recheck_torrents.get(downloader): + self._recheck_torrents[downloader] = [] + self._recheck_torrents[downloader].append(download_id) + # 下载成功 + self.info(f"成功添加辅种下载，站点：{site_info.get('name')}，种子链接：{torrent_url}") + # TR会自动校验 + downloader_type = self.downloader.get_downloader_type(downloader_id=downloader) + if downloader_type == DownloaderType.QB: + # 开始校验种子 + self.downloader.recheck_torrents(downloader_id=downloader, ids=[download_id]) + + @staticmethod + def __get_hash(torrent, dl_type): + """ + 获取种子hash + """ + try: + return torrent.get("hash") if dl_type == DownloaderType.QB else torrent.hashString + except Exception as e: + print(str(e)) + return "" + + @staticmethod + def __get_label(torrent, dl_type): + """ + 获取种子标签 + """ + try: + return torrent.get("tags") or [] if dl_type == DownloaderType.QB else torrent.labels or [] + except Exception as e: + print(str(e)) + return [] + + @staticmethod + def __can_seeding(torrent, dl_type): + """ + 判断种子是否可以做种并处于暂停状态 + """ + try: + return torrent.get("state") == "pausedUP" if dl_type == DownloaderType.QB \ + else (torrent.status.stopped and torrent.percent_done == 1) + except Exception as e: + print(str(e)) + return False + + @staticmethod + def __get_save_path(torrent, dl_type): + """ + 获取种子保存路径 + """ + try: + return torrent.get("save_path") if dl_type == DownloaderType.QB else torrent.download_dir + except Exception as e: + print(str(e)) + return "" + + def __get_download_url(self, seed, site, base_url): + """ + 拼装种子下载链接 + """ + + def __is_special_site(url): + """ + 判断是否为特殊站点 + """ + if "hdchina.org" in url: + return True + if "hdsky.me" in url: + return True + if "hdcity.in" in url: + return True + if "totheglory.im" in url: + return True + return False + + try: + if __is_special_site(site.get('strict_url')): + # 从详情页面获取下载链接 + return self.__get_torrent_url_from_page(seed=seed, site=site) + else: + download_url = base_url.replace( + "id={}", + "id={id}" + ).replace( + "/{}", + "/{id}" + ).format( + **{ + "id": seed.get("torrent_id"), + "passkey": site.get("passkey") or '', + "uid": site.get("uid") or '' + } + ) + if download_url.count("{"): + self.warn(f"当前不支持该站点的辅助任务，Url转换失败：{seed}") + return None + download_url = re.sub(r"[&?]passkey=", "", + re.sub(r"[&?]uid=", "", + download_url, + flags=re.IGNORECASE), + flags=re.IGNORECASE) + return f"{site.get('strict_url')}/{download_url}" + except Exception as e: + self.warn(f"当前不支持该站点的辅助任务，Url转换失败：{str(e)}") + return None + + def __get_torrent_url_from_page(self, seed, site): + """ + 从详情页面获取下载链接 + """ + try: + page_url = f"{site.get('strict_url')}/details.php?id={seed.get('torrent_id')}&hit=1" + self.info(f"正在获取种子下载链接：{page_url} ...") + res = RequestUtils( + cookies=site.get("cookie"), + headers=site.get("ua"), + proxies=Config().get_proxies() if site.get("proxy") else None + ).get_res(url=page_url) + if res is not None and res.status_code in (200, 500): + if "charset=utf-8" in res.text or "charset=UTF-8" in res.text: + res.encoding = "UTF-8" + else: + res.encoding = res.apparent_encoding + if not res.text: + self.warn(f"获取种子下载链接失败，页面内容为空：{page_url}") + return None + # 使用xpath从页面中获取下载链接 + html = etree.HTML(res.text) + for xpath in self._torrent_xpaths: + download_url = html.xpath(xpath) + if download_url: + download_url = download_url[0] + self.info(f"获取种子下载链接成功：{download_url}") + if not download_url.startswith("http"): + if download_url.startswith("/"): + download_url = f"{site.get('strict_url')}{download_url}" + else: + download_url = f"{site.get('strict_url')}/{download_url}" + return download_url + self.warn(f"获取种子下载链接失败，未找到下载链接：{page_url}") + return None + else: + return None + except Exception as e: + self.warn(f"获取种子下载链接失败：{str(e)}") + return None + + def stop_service(self): + """ + 退出插件 + """ + try: + if self._scheduler: + self._scheduler.remove_all_jobs() + if self._scheduler.running: + self._event.set() + self._scheduler.shutdown() + self._event.clear() + self._scheduler = None + except Exception as e: + print(str(e)) diff --git a/app/plugins/modules/libraryrefresh.py b/app/plugins/modules/libraryrefresh.py index 4e5f63ee31..e6884e3e72 100644 --- a/app/plugins/modules/libraryrefresh.py +++ b/app/plugins/modules/libraryrefresh.py @@ -2,13 +2,18 @@ from app.plugins import EventHandler from app.plugins.modules._base import _IPluginModule from app.utils.types import EventType +from datetime import datetime, timedelta +from app.utils import ExceptionUtils +from apscheduler.schedulers.background import BackgroundScheduler + +from config import Config class LibraryRefresh(_IPluginModule): # 插件名称 - module_name = "实时刷新媒体库" + module_name = "刷新媒体库" # 插件描述 - module_desc = "入库完成后实时刷新媒体库服务器海报墙。" + module_desc = "入库完成后刷新媒体库服务器海报墙。" # 插件图标 module_icon = "refresh.png" # 主题色 @@ -28,6 +33,8 @@ class LibraryRefresh(_IPluginModule): # 私有属性 _enable = False + _scheduler = None + _refresh_delay = 0 mediaserver = None @@ -35,6 +42,26 @@ def init_config(self, config: dict = None): self.mediaserver = MediaServer() if config: self._enable = config.get("enable") + try: + # 延迟时间 + delay = int(float(config.get("delay") or 0)) + if delay < 0: + delay = 0 + self._refresh_delay = delay + except Exception as e: + ExceptionUtils.exception_traceback(e) + self._refresh_delay = 0 + + self.stop_service() + + if not self._enable: + return + + if self._refresh_delay > 0: + self.info(f"媒体库延迟刷新服务启动，延迟 {self._refresh_delay} 秒刷新媒体库") + self._scheduler = BackgroundScheduler(timezone=Config().get_timezone()) + else: + self.info("媒体库实时刷新服务启动") def get_state(self): return self._enable @@ -49,38 +76,87 @@ def get_fields(): # 同一行 [ { - 'title': '开启媒体库实时刷新', + 'title': '开启媒体库刷新', 'required': "", 'tooltip': 'Emby已有电视剧新增剧集时只会刷新对应电视剧，其它场景下如开启了二级分类则只刷新二级分类对应媒体库，否则刷新整库；Jellyfin/Plex只支持刷新整库', 'type': 'switch', 'id': 'enable', } ], + [ + { + 'title': '延迟刷新时间', + 'required': "", + 'tooltip': '延迟刷新时间，单位秒，0或留空则不延迟', + 'type': 'text', + 'content': [ + { + 'id': 'delay', + 'placeholder': '0', + } + ] + } + ] ] } ] def stop_service(self): - pass + """ + 退出插件 + """ + try: + if self._scheduler: + self._scheduler.remove_all_jobs() + if self._scheduler.running: + self._scheduler.shutdown() + self._scheduler = None + except Exception as e: + print(str(e)) + + def __refresh_library(self, event_data): + mediaserver_type = self.mediaserver.get_type().value + media_info = event_data.get("media_info") + if media_info: + title = media_info.get("title") + year = media_info.get("year") + media_name = f"{title} ({year})" if year else title + self.info(f"媒体服务器 {mediaserver_type} 刷新媒体 {media_name} ...") + self.mediaserver.refresh_library_by_items([{ + "title": title, + "year": year, + "type": media_info.get("type"), + "category": media_info.get("category"), + "target_path": event_data.get("dest") + }]) + else: + self.info(f"媒体服务器 {mediaserver_type} 刷新整库 ...") + self.mediaserver.refresh_root_library() - @EventHandler.register(EventType.TransferFinished) + @EventHandler.register([ + EventType.TransferFinished, + EventType.RefreshMediaServer + ]) def refresh(self, event): """ 监听入库完成事件 """ if not self._enable: return - event_data = event.event_data - media_info = event_data.get("media_info") - title = media_info.get("title") - year = media_info.get("year") - media_name = f"{title} ({year})" if year else title - mediaserver_type = self.mediaserver.get_type().value - self.info(f"媒体服务器 {mediaserver_type} 刷新媒体 {media_name}") - self.mediaserver.refresh_library_by_items([{ - "title": title, - "year": year, - "type": media_info.get("type"), - "category": media_info.get("category"), - "target_path": event_data.get("dest") - }]) + + if self._refresh_delay > 0: + # 计算延迟时间 + run_date = datetime.now() + timedelta(seconds=self._refresh_delay) + + # 使用 date 触发器添加任务到调度器 + formatted_run_date = run_date.strftime("%Y-%m-%d %H:%M:%S") + self.info(f"新增延迟刷新任务，将在 {formatted_run_date} 刷新媒体库") + self._scheduler.add_job(func=self.__refresh_library, args=[event.event_data], trigger='date', + run_date=run_date) + + # 启动调度器（懒启动） + if not self._scheduler.running: + self._scheduler.start() + else: + # 不延迟刷新 + self.__refresh_library(event.event_data) diff --git a/app/plugins/modules/libraryscraper.py b/app/plugins/modules/libraryscraper.py index c3a3098410..c46ecb7549 100644 --- a/app/plugins/modules/libraryscraper.py +++ b/app/plugins/modules/libraryscraper.py @@ -1,4 +1,3 @@ -import os from datetime import datetime from threading import Event @@ -6,13 +5,11 @@ from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.cron import CronTrigger -from app.media import Media, Scraper -from app.media.meta import MetaInfo +from app.media import Scraper from app.plugins import EventHandler from app.plugins.modules._base import _IPluginModule -from app.utils import NfoReader -from app.utils.types import MediaType, EventType -from config import Config, RMT_MEDIAEXT +from app.utils.types import EventType +from config import Config class LibraryScraper(_IPluginModule): @@ -39,7 +36,6 @@ class LibraryScraper(_IPluginModule): # 私有属性 _scheduler = None - _media = None _scraper = None # 限速开关 _cron = None @@ -92,15 +88,6 @@ def get_fields(): ] } ], - [ - { - 'title': '立即运行一次', - 'required': "", - 'tooltip': '打开后立即运行一次（点击此对话框的确定按钮后即会运行，周期未设置也会运行），关闭后将仅按照刮削周期运行（同时上次触发运行的任务如果在运行中也会停止）', - 'type': 'switch', - 'id': 'onlyonce', - } - ], ] }, { @@ -130,17 +117,30 @@ def get_fields(): 'content': [ { 'id': 'exclude_path', - 'placeholder': '' + 'placeholder': '多个路径用,分割' } ] } ] ] + }, + { + 'type': 'div', + 'content': [ + [ + { + 'title': '立即运行一次', + 'required': "", + 'tooltip': '打开后立即运行一次（点击此对话框的确定按钮后即会运行，周期未设置也会运行），关闭后将仅按照刮削周期运行（同时上次触发运行的任务如果在运行中也会停止）', + 'type': 'switch', + 'id': 'onlyonce', + } + ], + ] } ] def init_config(self, config=None): - self._media = Media() self._scraper = Scraper() # 读取配置 @@ -155,30 +155,29 @@ def init_config(self, config=None): self.stop_service() # 启动定时任务 & 立即运行一次 - if self._cron or self._onlyonce: + if self.get_state() or self._onlyonce: self._scheduler = BackgroundScheduler(timezone=Config().get_timezone()) if self._cron: - self._scheduler.add_job(self.__libraryscraper, CronTrigger.from_crontab(self._cron)) + self.info(f"刮削服务启动，周期：{self._cron}") + self._scheduler.add_job(self.__libraryscraper, + CronTrigger.from_crontab(self._cron)) if self._onlyonce: + self.info(f"刮削服务启动，立即运行一次") self._scheduler.add_job(self.__libraryscraper, 'date', run_date=datetime.now(tz=pytz.timezone(Config().get_timezone()))) - self._scheduler.print_jobs() - self._scheduler.start() - - if self._onlyonce: - self.info(f"刮削服务启动，立即运行一次") - if self._cron: - self.info(f"刮削服务启动，周期：{self._cron}") - - # 关闭一次性开关 - self._onlyonce = False - self.update_config({ - "onlyonce": False, - "cron": self._cron, - "mode": self._mode, - "scraper_path": self._scraper_path, - "exclude_path": self._exclude_path - }) + # 关闭一次性开关 + self._onlyonce = False + self.update_config({ + "onlyonce": self._onlyonce, + "cron": self._cron, + "mode": self._mode, + "scraper_path": self._scraper_path, + "exclude_path": self._exclude_path + }) + if self._scheduler.get_jobs(): + # 启动服务 + self._scheduler.print_jobs() + self._scheduler.start() def get_state(self): return True if self._cron else False @@ -195,71 +194,11 @@ def start_scrap(self, event): return path = event_info.get("path") force = event_info.get("force") - self.__folder_scraper(path, force=force) - - def __folder_scraper(self, path, exclude_path=None, force=None): - """ - 刮削指定文件夹或文件 - :param path: 文件夹或文件路径 - :param force: 是否强制刮削 - :return: - """ - # 模式 - if force is not None: - force_nfo = force_pic = force + if force: + mode = 'force_all' else: - force_nfo = True if self._mode in ["force_nfo", "force_all"] else False - force_pic = True if self._mode in ["force_all"] else False - # 每个媒体库下的所有文件 - for file in self.__get_library_files(path, exclude_path): - if self._event.is_set(): - self.info(f"媒体库刮削服务停止") - return - if not file: - continue - self.info(f"开始刮削媒体库文件：{file} ...") - # 识别媒体文件 - meta_info = MetaInfo(os.path.basename(file)) - # 优先读取本地文件 - tmdbid = None - if meta_info.type == MediaType.MOVIE: - # 电影 - movie_nfo = os.path.join(os.path.dirname(file), "movie.nfo") - if os.path.exists(movie_nfo): - tmdbid = self.__get_tmdbid_from_nfo(movie_nfo) - file_nfo = os.path.join(os.path.splitext(file)[0] + ".nfo") - if not tmdbid and os.path.exists(file_nfo): - tmdbid = self.__get_tmdbid_from_nfo(file_nfo) - else: - # 电视剧 - tv_nfo = os.path.join(os.path.dirname(os.path.dirname(file)), "tvshow.nfo") - if os.path.exists(tv_nfo): - tmdbid = self.__get_tmdbid_from_nfo(tv_nfo) - if tmdbid and not force_nfo: - self.info(f"读取到本地nfo文件的tmdbid：{tmdbid}") - meta_info.set_tmdb_info(self._media.get_tmdb_info(mtype=meta_info.type, - tmdbid=tmdbid, - append_to_response='all')) - media_info = meta_info - else: - medias = self._media.get_media_info_on_files(file_list=[file], - append_to_response="all") - if not medias: - continue - media_info = None - for _, media in medias.items(): - media_info = media - break - if not media_info or not media_info.tmdb_info: - continue - self._scraper.gen_scraper_files(media=media_info, - dir_path=os.path.dirname(file), - file_name=os.path.splitext(os.path.basename(file))[0], - file_ext=os.path.splitext(file)[-1], - force=True, - force_nfo=force_nfo, - force_pic=force_pic) - self.info(f"{file} 刮削完成") + mode = 'no_force' + self._scraper.folder_scraper(path, mode=mode) def __libraryscraper(self): """ @@ -270,55 +209,15 @@ def __libraryscraper(self): for path in self._scraper_path: if not path: continue + if self._event.is_set(): + self.info(f"媒体库刮削服务停止") + return # 刮削目录 - self.__folder_scraper(path, self._exclude_path) + self._scraper.folder_scraper(path=path, + exclude_path=self._exclude_path, + mode=self._mode) self.info(f"媒体库刮削完成") - @staticmethod - def __get_library_files(in_path, exclude_path=None): - """ - 获取媒体库文件列表 - """ - if not os.path.isdir(in_path): - yield in_path - return - - for root, dirs, files in os.walk(in_path): - if exclude_path and any(os.path.abspath(root).startswith(os.path.abspath(path)) - for path in exclude_path.split(",")): - continue - - for file in files: - cur_path = os.path.join(root, file) - # 检查后缀 - if os.path.splitext(file)[-1].lower() in RMT_MEDIAEXT: - yield cur_path - - @staticmethod - def __get_tmdbid_from_nfo(file_path): - """ - 从nfo文件中获取信息 - :param file_path: - :return: tmdbid - """ - if not file_path: - return None - xpaths = [ - "uniqueid[@type='Tmdb']", - "uniqueid[@type='tmdb']", - "uniqueid[@type='TMDB']", - "tmdbid" - ] - reader = NfoReader(file_path) - for xpath in xpaths: - try: - tmdbid = reader.get_element_value(xpath) - if tmdbid: - return tmdbid - except Exception as err: - print(str(err)) - return None - def stop_service(self): """ 退出插件 diff --git a/app/plugins/modules/mediasyncdel.py b/app/plugins/modules/mediasyncdel.py index 4e79eb9242..438716b800 100644 --- a/app/plugins/modules/mediasyncdel.py +++ b/app/plugins/modules/mediasyncdel.py @@ -113,6 +113,22 @@ def sync_del(self, event): if not event_type or str(event_type) != 'media_del': return + # 是否虚拟标识 + item_isvirtual = event_data.get("item_isvirtual") + if not item_isvirtual: + self.error("item_isvirtual参数未配置，为防止误删除，暂停插件运行") + self.update_config({ + "enable": False, + "del_source": self._del_source, + "exclude_path": self._exclude_path, + "send_notify": self._send_notify + }) + return + + # 如果是虚拟item，则直接return，不进行删除 + if item_isvirtual == 'True': + return + # 媒体类型 media_type = event_data.get("media_type") # 媒体名称 @@ -184,7 +200,17 @@ def sync_del(self, event): return # 开始删除 - logids = [history.ID for history in transfer_history] + if media_type == "Episode" or media_type == "Movie": + # 如果有剧集或者电影有多个版本的话，需要根据名称筛选下要删除的版本 + logids = [history.ID for history in transfer_history if + history.DEST_FILENAME == os.path.basename(media_path)] + else: + logids = [history.ID for history in transfer_history] + + if len(logids) == 0: + self.warn(f"{media_type} {media_name} 未获取到可删除数据") + return + self.info(f"获取到删除媒体数量 {len(logids)}") WebAction().delete_history({ "logids": logids, diff --git a/app/plugins/modules/movielike.py b/app/plugins/modules/movielike.py index 3a448a5327..d1cbcb8ac6 100644 --- a/app/plugins/modules/movielike.py +++ b/app/plugins/modules/movielike.py @@ -35,6 +35,12 @@ class MovieLike(_IPluginModule): # 私有属性 _enable = False _dir_name = RMT_FAVTYPE + _remote_path = None + _local_path = None + _remote_path2 = None + _local_path2 = None + _remote_path3 = None + _local_path3 = None mediaserver = None filetransfer = None @@ -49,6 +55,12 @@ def init_config(self, config: dict = None): self._dir_name = config.get("dir_name") if self._dir_name: Config().update_favtype(self._dir_name) + self._local_path = config.get("local_path") + self._remote_path = config.get("remote_path") + self._local_path2 = config.get("local_path2") + self._remote_path2 = config.get("remote_path2") + self._local_path3 = config.get("local_path3") + self._remote_path3 = config.get("remote_path3") def get_state(self): return self._enable @@ -61,6 +73,15 @@ def get_fields(): 'type': 'div', 'content': [ # 同一行 + [ + { + 'title': '开启电影精选', + 'required': "", + 'tooltip': '目前仅支持Emby，NAStool挂载目录如与Emby媒体库目录不一致则需要配置路径映射。在Emby的Webhooks中勾选 用户->添加到最爱 事件，如需控制仅部分用户生效，可在媒体服务器单独建立Webhook并设置对应用户范围', + 'type': 'switch', + 'id': 'enable', + } + ], [ { 'title': '分类目录名称', @@ -76,13 +97,56 @@ def get_fields(): ] } ], + ] + }, + { + 'type': 'details', + 'summary': '路径映射', + 'tooltip': '当NAStool与媒体服务器的媒体库路程不一致时，需要映射转换，最多可设置三组,留空时不启用', + 'content': [ + # 同一行 [ { - 'title': '开启电影精选', - 'required': "", - 'tooltip': '目前仅支持Emby，NAStool挂载目录需与Emby媒体库目录一致。在Emby的Webhooks中勾选 用户->添加到最爱 事件，如需控制仅部分用户生效，可在媒体服务器单独建立Webhook并设置对应用户范围', - 'type': 'switch', - 'id': 'enable', + 'title': '路径1', + 'type': 'text', + 'content': [ + { + 'id': 'local_path', + 'placeholder': '本地路径' + }, + { + 'id': 'remote_path', + 'placeholder': '远程路径' + } + ] + }, + { + 'title': '路径2', + 'type': 'text', + 'content': [ + { + 'id': 'local_path2', + 'placeholder': '本地路径' + }, + { + 'id': 'remote_path2', + 'placeholder': '远程路径' + } + ] + }, + { + 'title': '路径3', + 'type': 'text', + 'content': [ + { + 'id': 'local_path3', + 'placeholder': '本地路径' + }, + { + 'id': 'remote_path3', + 'placeholder': '远程路径' + } + ] } ], ] @@ -115,9 +179,20 @@ def favtransfer(self, event): item_path = event_info.get('Item', {}).get('Path') if not item_path: return + # 路径替换 + if self._local_path and self._remote_path and item_path.startswith(self._remote_path): + item_path = item_path.replace(self._remote_path, self._local_path).replace('\\', '/') + + if self._local_path2 and self._remote_path2 and item_path.startswith(self._remote_path2): + item_path = item_path.replace(self._remote_path2, self._local_path2).replace('\\', '/') + + if self._local_path3 and self._remote_path3 and item_path.startswith(self._remote_path3): + item_path = item_path.replace(self._remote_path3, self._local_path3).replace('\\', '/') + # 路径不存在不处理 if not os.path.exists(item_path): self.warn(f"{item_path} 文件不存在") return + # 文件转为目录 if os.path.isdir(item_path): movie_dir = item_path @@ -146,5 +221,11 @@ def favtransfer(self, event): ret, retmsg = SystemUtils.move(org_path, new_path) if ret != 0: self.error(f"{retmsg}") + else: + # 发送刷新媒体库事件 + EventHandler.send_event(EventType.RefreshMediaServer, { + "dest": new_path, + "media_info": {} + }) else: self.warn(f"{org_path} 目录不存在") diff --git a/app/plugins/modules/movierandom.py b/app/plugins/modules/movierandom.py new file mode 100644 index 0000000000..61ad3e1125 --- /dev/null +++ b/app/plugins/modules/movierandom.py @@ -0,0 +1,330 @@ +import random +from datetime import datetime +from threading import Event + +import pytz +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.cron import CronTrigger + +import log +from app.conf import ModuleConf +from app.helper import DbHelper +from app.media import Media +from app.mediaserver import MediaServer +from app.plugins.modules._base import _IPluginModule +from app.subscribe import Subscribe +from app.utils.types import SearchType, RssType, MediaType +from config import Config + + +class MovieRandom(_IPluginModule): + # 插件名称 + module_name = "电影随机订阅" + # 插件描述 + module_desc = "随机获取一部未入库的电影，自动添加订阅。" + # 插件图标 + module_icon = "random.png" + # 主题色 + module_color = "#0000FF" + # 插件版本 + module_version = "1.0" + # 插件作者 + module_author = "thsrite" + # 作者主页 + author_url = "https://github.com/thsrite" + # 插件配置项ID前缀 + module_config_prefix = "movierandom_" + # 加载顺序 + module_order = 18 + # 可使用的用户级别 + auth_level = 2 + + # 退出事件 + _event = Event() + # 私有属性 + mediaserver = None + dbhelper = None + subscribe = None + _scheduler = None + _enable = False + _onlyonce = False + _cron = None + _language = None + _genres = None + _vote = None + _date = None + + @staticmethod + def get_fields(): + language_options = ModuleConf.DISCOVER_FILTER_CONF.get("tmdb_movie").get("with_original_language").get( + "options") + genres_options = ModuleConf.DISCOVER_FILTER_CONF.get("tmdb_movie").get("with_genres").get("options") + # tmdb电影类型 + genres = {m.get('name'): m.get('name') for m in genres_options} + # tmdb电影语言 + language = {m.get('name'): m.get('name') for m in language_options} + return [ + # 同一板块 + { + 'type': 'div', + 'content': [ + # 同一行 + [ + { + 'title': '开启电影随机订阅', + 'required': "", + 'tooltip': '开启后，定时随机订阅一部电影。', + 'type': 'switch', + 'id': 'enable', + }, + { + 'title': '立即运行一次', + 'required': "", + 'tooltip': '打开后立即运行一次（点击此对话框的确定按钮后即会运行，周期未设置也会运行），关闭后将仅按照随机周期运行（同时上次触发运行的任务如果在运行中也会停止）', + 'type': 'switch', + 'id': 'onlyonce', + }, + ], + [ + { + 'title': '随机周期', + 'required': "required", + 'tooltip': '电影随机订阅的时间周期，支持5位cron表达式。', + 'type': 'text', + 'content': [ + { + 'id': 'cron', + 'placeholder': '0 0 0 ? *', + } + ] + }, + { + 'title': '上映时间', + 'required': "", + 'tooltip': '电影上映时间，大于该时间的会被订阅', + 'type': 'text', + 'content': [ + { + 'id': 'date', + 'placeholder': '2022', + } + ] + }, + { + 'title': '电影评分', + 'required': "", + 'tooltip': '最低评分，大于等于该评分的会被订阅（最大10）', + 'type': 'text', + 'content': [ + { + 'id': 'vote', + 'placeholder': '8', + } + ] + }, + ], + [ + { + 'title': '电影类型', + 'required': "", + 'type': 'select', + 'content': [ + { + 'id': 'genres', + 'options': genres, + 'default': '全部' + }, + ] + }, + { + 'title': '电影语言', + 'required': "", + 'type': 'select', + 'content': [ + { + 'id': 'language', + 'options': language, + 'default': '全部' + }, + ] + }, + ] + ] + } + ] + + def init_config(self, config: dict = None): + self.mediaserver = MediaServer() + self.dbhelper = DbHelper() + self.subscribe = Subscribe() + if config: + self._enable = config.get("enable") + self._onlyonce = config.get("onlyonce") + self._cron = config.get("cron") + self._language = config.get("language") + self._genres = config.get("genres") + self._vote = config.get("vote") + self._date = config.get("date") + + # 停止现有任务 + self.stop_service() + + # 启动定时任务 & 立即运行一次 + if self.get_state() or self._onlyonce: + self._scheduler = BackgroundScheduler(timezone=Config().get_timezone()) + if self._cron: + self.info(f"电影随机服务启动，周期：{self._cron}") + self._scheduler.add_job(self.__random, + CronTrigger.from_crontab(self._cron)) + if self._onlyonce: + self.info(f"电影随机服务启动，立即运行一次") + self._scheduler.add_job(self.__random, 'date', + run_date=datetime.now(tz=pytz.timezone(Config().get_timezone()))) + # 关闭一次性开关 + self._onlyonce = False + self.update_config({ + "enable": self._enable, + "onlyonce": self._onlyonce, + "cron": self._cron, + "language": self._language, + "genres": self._genres, + "vote": self._vote, + "date": self._date, + }) + if self._scheduler.get_jobs(): + # 启动服务 + self._scheduler.print_jobs() + self._scheduler.start() + + def __random(self): + """ + 随机获取一部tmdb电影下载 + """ + params = {} + if self._date: + params['release_date.gte'] = self._date + if self._vote: + params['vote_average.gte'] = self._vote + if self._language: + language_options = ModuleConf.DISCOVER_FILTER_CONF.get("tmdb_movie").get("with_original_language").get( + "options") + for m in language_options: + if m.get('name') == self._language: + params['with_original_language'] = m.get('value') + break + if self._genres: + genres_options = ModuleConf.DISCOVER_FILTER_CONF.get("tmdb_movie").get("with_genres").get("options") + for m in genres_options: + if m.get('name') == self._genres: + params['with_genres'] = m.get('value') + break + + # 查询选择条件下所有页数 + random_max_page = Media().get_tmdb_discover_movies_pages(params=params) + if random_max_page == 0: + log.error("当前所选条件下未获取到电影数据，停止随机订阅") + return + + log.info(f"当前所选条件下获取到电影数据 {random_max_page} 页，开始随机订阅") + + movie_list = [] + retry_time = 0 + try_page = [] + while not movie_list and retry_time < 5: + page = random.randint(0, random_max_page - 1) + + # 已经试过的页数重新random + while page in try_page: + page = random.randint(0, random_max_page - 1) + + # 根据请求参数随机获取一页电影 + movie_list = self.__get_discover(page=page, + params=params) + self.info( + f"正在尝试第 {retry_time + 1} 次获取，获取到随机页数 {page} 电影数据 {len(movie_list)} 条，最多尝试5次") + retry_time = retry_time + 1 + try_page.append(page) + + if not movie_list: + self.error("已达最大尝试次数，当前条件下未随机到电影") + return + + # 随机出媒体库不存在的视频 + media_info = self.__random_check(movie_list) + if not media_info: + self.warn("本次未随机出满足条件的电影") + return + + log.info( + f"电影 {media_info.get('title')} {media_info.get('year')} tmdbid:{media_info.get('id')}未入库，开始订阅") + + # 检查是否已订阅过 + if self.dbhelper.check_rss_history( + type_str="MOV", + name=media_info.get('title'), + year=media_info.get('year'), + season=None): + self.info( + f"{media_info.get('title')} 已订阅过") + return + # 添加订阅 + code, msg, rss_media = self.subscribe.add_rss_subscribe( + mtype=MediaType.MOVIE, + name=media_info.get('title'), + year=media_info.get('year'), + season=None, + channel=RssType.Auto, + in_from=SearchType.PLUGIN + ) + if not rss_media or code != 0: + self.warn("%s 添加订阅失败：%s" % (media_info.get('title'), msg)) + else: + self.info("%s 添加订阅成功" % media_info.get('title')) + + def __random_check(self, movie_list): + """ + 随机一个电影 + 检查媒体服务器是否存在 + """ + # 随机一个电影 + media_info = random.choice(movie_list) + log.info(f"随机出电影 {media_info.get('title')} {media_info.get('year')} tmdbid:{media_info.get('id')}") + # 删除该电影，防止再次random到 + movie_list.remove(media_info) + + # 检查媒体服务器是否存在 + item_id = self.mediaserver.check_item_exists(mtype=MediaType.MOVIE, + title=media_info.get('title'), + year=media_info.get('year'), + tmdbid=media_info.get('id')) + if item_id: + self.info(f"媒体服务器已存在：{media_info.get('title')}") + if len(movie_list) == 0: + return None + self.__random_check(movie_list) + return media_info + + @staticmethod + def __get_discover(page, params): + return Media().get_tmdb_discover(mtype=MediaType.MOVIE, + page=page, + params=params) + + def get_state(self): + return self._enable \ + and self._cron + + def stop_service(self): + """ + 停止服务 + """ + try: + if self._scheduler: + self._scheduler.remove_all_jobs() + if self._scheduler.running: + self._event.set() + self._scheduler.shutdown() + self._event.clear() + self._scheduler = None + except Exception as e: + print(str(e)) diff --git a/app/plugins/modules/opensubtitles.py b/app/plugins/modules/opensubtitles.py index 0ac8633a01..8df2846ffc 100644 --- a/app/plugins/modules/opensubtitles.py +++ b/app/plugins/modules/opensubtitles.py @@ -48,7 +48,7 @@ class OpenSubtitles(_IPluginModule): def __init__(self): self._ua = Config().get_ua() - def init_config(self, config: dict): + def init_config(self, config: dict = None): self.sitehelper = SiteHelper() self._save_tmp_path = Config().get_temp_path() if not os.path.exists(self._save_tmp_path): diff --git a/app/plugins/modules/torrentremover.py b/app/plugins/modules/torrentremover.py index bf474daf6b..a0648e452d 100644 --- a/app/plugins/modules/torrentremover.py +++ b/app/plugins/modules/torrentremover.py @@ -10,7 +10,7 @@ class TorrentRemover(_IPluginModule): # 插件名称 - module_name = "下载任务清理" + module_name = "删除下载任务" # 插件描述 module_desc = "历史记录中源文件被删除时，同步删除下载器中的下载任务。" # 插件图标 @@ -37,7 +37,7 @@ class TorrentRemover(_IPluginModule): def __init__(self): self._ua = Config().get_ua() - def init_config(self, config: dict): + def init_config(self, config: dict = None): self.dbhelper = DbHelper() if config: self._enable = config.get("enable") diff --git a/app/plugins/modules/torrenttransfer.py b/app/plugins/modules/torrenttransfer.py new file mode 100644 index 0000000000..0f8b9c17d0 --- /dev/null +++ b/app/plugins/modules/torrenttransfer.py @@ -0,0 +1,562 @@ +import os.path +from copy import deepcopy +from datetime import datetime +from threading import Event + +import pytz +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.cron import CronTrigger + +from app.downloader import Downloader +from app.media.meta import MetaInfo +from app.message import Message +from app.plugins.modules._base import _IPluginModule +from app.utils.types import DownloaderType +from config import Config + + +class TorrentTransfer(_IPluginModule): + # 插件名称 + module_name = "自动转移做种" + # 插件描述 + module_desc = "定期转移下载器中的做种任务到另一个下载器。" + # 插件图标 + module_icon = "torrenttransfer.jpg" + # 主题色 + module_color = "#272636" + # 插件版本 + module_version = "1.0" + # 插件作者 + module_author = "jxxghp" + # 作者主页 + author_url = "https://github.com/jxxghp" + # 插件配置项ID前缀 + module_config_prefix = "torrenttransfer_" + # 加载顺序 + module_order = 20 + # 可使用的用户级别 + user_level = 1 + + # 私有属性 + _scheduler = None + downloader = None + sites = None + message = None + # 限速开关 + _enable = False + _cron = None + _onlyonce = False + _fromdownloader = None + _todownloader = None + _frompath = None + _topath = None + _notify = False + _nolabels = None + _nopaths = None + _deletesource = False + _fromtorrentpath = None + # 退出事件 + _event = Event() + # 待检查种子清单 + _recheck_torrents = {} + _is_recheck_running = False + # 任务标签 + _torrent_tags = ["已整理", "转移做种"] + + @staticmethod + def get_fields(): + downloaders = {k: v for k, v in Downloader().get_downloader_conf_simple().items() + if v.get("type") in ["qbittorrent", "transmission"] and v.get("enabled")} + return [ + # 同一板块 + { + 'type': 'div', + 'content': [ + # 同一行 + [ + { + 'title': '开启自动转移做种', + 'required': "", + 'tooltip': '开启后，定期将源下载器中已完成的种子任务迁移至目的下载器，任务转移后会自动暂停，校验通过且完整后才开始做种。', + 'type': 'switch', + 'id': 'enable', + } + ], + [ + { + 'title': '执行周期', + 'required': "required", + 'tooltip': '设置移转做种任务执行的时间周期，支持5位cron表达式；应避免任务执行过于频繁', + 'type': 'text', + 'content': [ + { + 'id': 'cron', + 'placeholder': '0 0 0 ? *', + } + ] + }, + { + 'title': '不转移种子标签', + 'required': "", + 'tooltip': '下载器中的种子有以下标签时不进行移转做种，多个标签使用英文,分隔', + 'type': 'text', + 'content': [ + { + 'id': 'nolabels', + 'placeholder': '使用,分隔多个标签', + } + ] + } + ] + ] + }, + { + 'type': 'details', + 'summary': '源下载器', + 'tooltip': '只有选中的下载器才会执行转移任务，只能选择一个', + 'content': [ + # 同一行 + [ + { + 'id': 'fromdownloader', + 'type': 'form-selectgroup', + 'radio': True, + 'content': downloaders + }, + ], + [ + { + 'title': '种子文件路径', + 'required': "required", + 'tooltip': '源下载器保存种子文件的路径，需要是NAStool可访问的路径，QB一般为BT_backup，TR一般为torrents', + 'type': 'text', + 'content': [ + { + 'id': 'fromtorrentpath', + 'placeholder': 'xxx/BT_backup、xxx/torrents', + } + ] + }, + { + 'title': '数据文件根路径', + 'required': "required", + 'tooltip': '源下载器中的种子数据文件保存根目录路径，必须是下载器能访问的路径，用于转移时转换种子数据文件路径使用；留空不进行路径转换，使用种子的数据文件保存目录', + 'type': 'text', + 'content': [ + { + 'id': 'frompath', + 'placeholder': '根路径，留空不进行路径转换', + } + ] + } + ] + ] + }, + { + 'type': 'details', + 'summary': '目的下载器', + 'tooltip': '将做种任务转移到这个下载器，只能选择一个', + 'content': [ + # 同一行 + [ + { + 'id': 'todownloader', + 'type': 'form-selectgroup', + 'radio': True, + 'content': downloaders + }, + ], + [ + { + 'title': '数据文件根路径', + 'required': "required", + 'tooltip': '目的下载器的种子数据文件保存目录根路径，必须是下载器能访问的路径，将会使用该路径替换源下载器中种子数据文件保存路径中的源目录根路径，替换后的新路径做为目的下载器种子数据文件的保存路径，需要准确填写，否则可能导致移转做种后找不到数据文件无法做种；留空不进行路径转换，使用种子的数据文件保存路径', + 'type': 'text', + 'content': [ + { + 'id': 'topath', + 'placeholder': '根路径，留空不进行路径转换', + } + ] + } + ] + ] + }, + { + 'type': 'div', + 'content': [ + [ + { + 'title': '不转移数据文件目录', + 'required': "", + 'tooltip': '以下数据文件目录的任务不进行转移，指下载器可访问的目录，每一行一个目录', + 'type': 'textarea', + 'content': { + 'id': 'nopaths', + 'placeholder': '每一行一个目录', + 'rows': 3 + } + } + ] + ] + }, + { + 'type': 'div', + 'content': [ + # 同一行 + [ + { + 'title': '删除源种子', + 'required': "", + 'tooltip': '转移成功后删除源下载器中的种子，首次运行请不要打开，避免种子丢失', + 'type': 'switch', + 'id': 'deletesource', + }, + { + 'title': '运行时通知', + 'required': "", + 'tooltip': '运行任务后会发送通知（需要打开自定义消息通知）', + 'type': 'switch', + 'id': 'notify', + }, + { + 'title': '立即运行一次', + 'required': "", + 'tooltip': '打开后立即运行一次（点击此对话框的确定按钮后即会运行，周期未设置也会运行），关闭后将仅按照周期运行（同时上次触发运行的任务如果在运行中也会停止）', + 'type': 'switch', + 'id': 'onlyonce', + } + ] + ] + } + ] + + def init_config(self, config=None): + self.downloader = Downloader() + self.message = Message() + # 读取配置 + if config: + self._enable = config.get("enable") + self._onlyonce = config.get("onlyonce") + self._cron = config.get("cron") + self._notify = config.get("notify") + self._nolabels = config.get("nolabels") + self._frompath = config.get("frompath") + self._topath = config.get("topath") + self._fromdownloader = config.get("fromdownloader") + self._todownloader = config.get("todownloader") + self._deletesource = config.get("deletesource") + self._fromtorrentpath = config.get("fromtorrentpath") + self._nopaths = config.get("nopaths") + + # 停止现有任务 + self.stop_service() + + # 启动定时任务 & 立即运行一次 + if self.get_state() or self._onlyonce: + # 检查配置 + if self._fromtorrentpath and not os.path.exists(self._fromtorrentpath): + self.error(f"源下载器种子文件保存路径不存在：{self._fromtorrentpath}") + return + if isinstance(self._fromdownloader, list) and len(self._fromdownloader) > 1: + self.error(f"源下载器只能选择一个") + return + if isinstance(self._todownloader, list) and len(self._todownloader) > 1: + self.error(f"目的下载器只能选择一个") + return + self._scheduler = BackgroundScheduler(timezone=Config().get_timezone()) + if self._cron: + self.info(f"移转做种服务启动，周期：{self._cron}") + self._scheduler.add_job(self.transfer, + CronTrigger.from_crontab(self._cron)) + if self._onlyonce: + self.info(f"移转做种服务启动，立即运行一次") + self._scheduler.add_job(self.transfer, 'date', + run_date=datetime.now(tz=pytz.timezone(Config().get_timezone()))) + # 关闭一次性开关 + self._onlyonce = False + self.update_config({ + "enable": self._enable, + "onlyonce": self._onlyonce, + "cron": self._cron, + "notify": self._notify, + "nolabels": self._nolabels, + "frompath": self._frompath, + "topath": self._topath, + "fromdownloader": self._fromdownloader, + "todownloader": self._todownloader, + "deletesource": self._deletesource, + "fromtorrentpath": self._fromtorrentpath, + "nopaths": self._nopaths + }) + if self._scheduler.get_jobs(): + # 追加种子校验服务 + self._scheduler.add_job(self.check_recheck, 'interval', minutes=3) + # 启动服务 + self._scheduler.print_jobs() + self._scheduler.start() + + def get_state(self): + return True if self._enable \ + and self._cron \ + and self._fromdownloader \ + and self._todownloader \ + and self._fromtorrentpath else False + + def transfer(self): + """ + 开始移转做种 + """ + if not self._enable \ + or not self._fromdownloader \ + or not self._todownloader \ + or not self._fromtorrentpath: + self.warn("移转做种服务未启用或未配置") + return + self.info("开始移转做种任务 ...") + # 源下载器 + downloader = self._fromdownloader[0] + # 目的下载器 + todownloader = self._todownloader[0] + # 下载器类型 + downloader_type = self.downloader.get_downloader_type(downloader_id=downloader) + # 获取下载器中已完成的种子 + torrents = self.downloader.get_completed_torrents(downloader_id=downloader) + if torrents: + self.info(f"下载器 {downloader} 已完成种子数：{len(torrents)}") + else: + self.info(f"下载器 {downloader} 没有已完成种子") + return + # 过滤种子，记录保存目录 + hash_strs = [] + for torrent in torrents: + if self._event.is_set(): + self.info(f"移转服务停止") + return + # 获取种子hash + hash_str = self.__get_hash(torrent, downloader_type) + # 获取保存路径 + save_path = self.__get_save_path(torrent, downloader_type) + if self._nopaths and save_path: + # 过滤不需要移转的路径 + nopath_skip = False + for nopath in self._nopaths.split('\n'): + if os.path.normpath(save_path).startswith(os.path.normpath(nopath)): + self.info(f"种子 {hash_str} 保存路径 {save_path} 不需要移转，跳过 ...") + nopath_skip = True + break + if nopath_skip: + continue + # 获取种子标签 + torrent_labels = self.__get_label(torrent, downloader_type) + if self._nolabels \ + and torrent_labels \ + and set(self._nolabels.split(',')).intersection(set(torrent_labels)): + self.info(f"种子 {hash_str} 含有不转移标签，跳过 ...") + continue + hash_strs.append({ + "hash": hash_str, + "save_path": save_path + }) + # 开始转移任务 + if hash_strs: + self.info(f"需要移转的种子数：{len(hash_strs)}") + # 记数 + total = len(hash_strs) + success = 0 + fail = 0 + for hash_item in hash_strs: + # 检查种子文件是否存在 + torrent_file = os.path.join(self._fromtorrentpath, f"{hash_item.get('hash')}.torrent") + if not os.path.exists(torrent_file): + self.error(f"种子文件不存在：{torrent_file}") + fail += 1 + continue + # 查询hash值是否已经在目的下载器中 + torrent_info = self.downloader.get_torrents(downloader_id=todownloader, + ids=[hash_item.get('hash')]) + if torrent_info: + self.debug(f"{hash_item.get('hash')} 已在目的下载器中，跳过 ...") + continue + # 转换保存路径 + download_dir = self.__convert_save_path(hash_item.get('save_path'), + self._frompath, + self._topath) + if not download_dir: + self.error(f"转换保存路径失败：{hash_item.get('save_path')}") + fail += 1 + continue + # 发送到另一个下载器中下载：默认暂停、传输下载路径、关闭自动管理模式 + _, download_id, retmsg = self.downloader.download( + media_info=MetaInfo("自动转移做种"), + torrent_file=torrent_file, + is_paused=True, + tag=deepcopy(self._torrent_tags), + downloader_id=todownloader, + download_dir=download_dir, + download_setting="-2", + is_auto=False + ) + if not download_id: + # 下载失败 + self.warn(f"添加转移任务出错，" + f"错误原因：{retmsg or '下载器添加任务失败'}，" + f"种子文件：{torrent_file}") + fail += 1 + continue + else: + # 追加校验任务 + self.info(f"添加校验检查任务：{download_id} ...") + if not self._recheck_torrents.get(todownloader): + self._recheck_torrents[todownloader] = [] + self._recheck_torrents[todownloader].append(download_id) + # 下载成功 + self.info(f"成功添加转移做种任务，种子文件：{torrent_file}") + # TR会自动校验 + downloader_type = self.downloader.get_downloader_type(downloader_id=todownloader) + if downloader_type == DownloaderType.QB: + # 开始校验种子 + self.downloader.recheck_torrents(downloader_id=todownloader, ids=[download_id]) + # 删除源种子，不能删除文件！ + if self._deletesource: + self.downloader.delete_torrents(downloader_id=downloader, + ids=[download_id], + delete_file=False) + success += 1 + # 触发校验任务 + if success > 0: + self.check_recheck() + # 发送通知 + if self._notify: + self.message.send_custom_message( + title="【移转做种任务执行完成】", + text=f"总数：{total}，成功：{success}，失败：{fail}" + ) + else: + self.info(f"没有需要移转的种子") + self.info("移转做种任务执行完成") + + def check_recheck(self): + """ + 定时检查下载器中种子是否校验完成，校验完成且完整的自动开始辅种 + """ + if not self._recheck_torrents: + return + if not self._todownloader: + return + if self._is_recheck_running: + return + downloader = self._todownloader[0] + # 需要检查的种子 + recheck_torrents = self._recheck_torrents.get(downloader, []) + if not recheck_torrents: + return + self.info(f"开始检查下载器 {downloader} 的校验任务 ...") + self._is_recheck_running = True + # 下载器类型 + downloader_type = self.downloader.get_downloader_type(downloader_id=downloader) + # 获取下载器中的种子 + torrents = self.downloader.get_torrents(downloader_id=downloader, + ids=recheck_torrents) + if torrents: + can_seeding_torrents = [] + for torrent in torrents: + # 获取种子hash + hash_str = self.__get_hash(torrent, downloader_type) + if self.__can_seeding(torrent, downloader_type): + can_seeding_torrents.append(hash_str) + if can_seeding_torrents: + self.info(f"共 {len(can_seeding_torrents)} 个任务校验完成，开始辅种 ...") + self.downloader.start_torrents(downloader_id=downloader, ids=can_seeding_torrents) + # 去除已经处理过的种子 + self._recheck_torrents[downloader] = list( + set(recheck_torrents).difference(set(can_seeding_torrents))) + elif torrents is None: + self.info(f"下载器 {downloader} 查询校验任务失败，将在下次继续查询 ...") + else: + self.info(f"下载器 {downloader} 中没有需要检查的校验任务，清空待处理列表 ...") + self._recheck_torrents[downloader] = [] + self._is_recheck_running = False + + @staticmethod + def __get_hash(torrent, dl_type): + """ + 获取种子hash + """ + try: + return torrent.get("hash") if dl_type == DownloaderType.QB else torrent.hashString + except Exception as e: + print(str(e)) + return "" + + @staticmethod + def __get_label(torrent, dl_type): + """ + 获取种子标签 + """ + try: + return torrent.get("tags") or [] if dl_type == DownloaderType.QB else torrent.labels or [] + except Exception as e: + print(str(e)) + return [] + + @staticmethod + def __get_save_path(torrent, dl_type): + """ + 获取种子保存路径 + """ + try: + return torrent.get("save_path") if dl_type == DownloaderType.QB else torrent.download_dir + except Exception as e: + print(str(e)) + return "" + + @staticmethod + def __can_seeding(torrent, dl_type): + """ + 判断种子是否可以做种并处于暂停状态 + """ + try: + return torrent.get("state") == "pausedUP" if dl_type == DownloaderType.QB \ + else (torrent.status.stopped and torrent.percent_done == 1) + except Exception as e: + print(str(e)) + return False + + @staticmethod + def __convert_save_path(save_path, from_root, to_root): + """ + 转换保存路径 + """ + try: + # 没有保存目录，以目的根目录为准 + if not save_path: + return to_root + # 没有设置根目录时返回save_path + if not to_root or not from_root: + return save_path + # 统一目录格式 + save_path = os.path.normpath(save_path).replace("\\", "/") + from_root = os.path.normpath(from_root).replace("\\", "/") + to_root = os.path.normpath(to_root).replace("\\", "/") + # 替换根目录 + if save_path.startswith(from_root): + return save_path.replace(from_root, to_root, 1) + except Exception as e: + print(str(e)) + return None + + def stop_service(self): + """ + 退出插件 + """ + try: + if self._scheduler: + self._scheduler.remove_all_jobs() + if self._scheduler.running: + self._event.set() + self._scheduler.shutdown() + self._event.clear() + self._scheduler = None + except Exception as e: + print(str(e)) diff --git a/app/plugins/modules/webhook.py b/app/plugins/modules/webhook.py index 3af3611cb7..ccdc70fd5c 100644 --- a/app/plugins/modules/webhook.py +++ b/app/plugins/modules/webhook.py @@ -84,19 +84,7 @@ def get_fields(): def stop_service(self): pass - @EventHandler.register([ - EventType.DownloadAdd, - EventType.DownloadFail, - EventType.TransferFinished, - EventType.TransferFail, - EventType.SubtitleDownload, - EventType.SubscribeAdd, - EventType.SubscribeFinished, - EventType.MessageIncoming, - EventType.SearchStart, - EventType.SourceFileDeleted, - EventType.LibraryFileDeleted - ]) + @EventHandler.register(EventType) def send(self, event): """ 向第三方Webhook发送请求 diff --git a/app/plugins/plugin_manager.py b/app/plugins/plugin_manager.py index 1118aea794..02c88c6f83 100644 --- a/app/plugins/plugin_manager.py +++ b/app/plugins/plugin_manager.py @@ -158,7 +158,7 @@ def get_plugin_config(self, pid): def get_plugin_page(self, pid): """ - 获取插件数据 + 获取插件额外页面数据 """ if not self._running_plugins.get(pid): return None @@ -167,6 +167,16 @@ def get_plugin_page(self, pid): title, html = self._running_plugins[pid].get_page() return title, html + def get_plugin_script(self, pid): + """ + 获取插件额外脚本 + """ + if not self._running_plugins.get(pid): + return None + if not hasattr(self._running_plugins[pid], "get_script"): + return None + return self._running_plugins[pid].get_script() + def get_plugin_state(self, pid): """ 获取插件状态 @@ -209,9 +219,13 @@ def get_plugins_conf(self, auth_level): conf.update({"color": plugin.module_color}) if hasattr(plugin, "module_config_prefix"): conf.update({"prefix": plugin.module_config_prefix}) + # 插件额外的页面 if hasattr(plugin, "get_page"): title, _ = plugin.get_page() conf.update({"page": title}) + # 插件额外的脚本 + if hasattr(plugin, "get_script"): + conf.update({"script": plugin.get_script()}) # 配置项 conf.update({"fields": plugin.get_fields() or {}}) # 配置值 diff --git a/app/rss.py b/app/rss.py index 571adc7552..c39587234c 100644 --- a/app/rss.py +++ b/app/rss.py @@ -111,6 +111,8 @@ def rssdownload(self): if not rss_url: log.info(f"【Rss】{site_name} 未配置rssurl，跳过...") continue + # 站点信息 + site_id = site_info.get("id") site_cookie = site_info.get("cookie") site_ua = site_info.get("ua") # 是否解析种子详情 @@ -177,6 +179,7 @@ def rssdownload(self): media_info=media_info, rss_movies=rss_movies, rss_tvs=rss_tvs, + site_id=site_id, site_filter_rule=site_fliter_rule, site_cookie=site_cookie, site_parse=site_parse, @@ -275,6 +278,10 @@ def rssdownload(self): # 不做处理，直接下载 pass + # 站点流控 + if self.sites.check_ratelimit(site_id): + continue + # 设置种子信息 media_info.set_torrent_info(res_order=match_info.get("res_order"), filter_rule=match_info.get("filter_rule"), @@ -386,6 +393,7 @@ def check_torrent_rss(self, media_info, rss_movies, rss_tvs, + site_id, site_filter_rule, site_cookie, site_parse, @@ -396,6 +404,7 @@ def check_torrent_rss(self, :param media_info: 已识别的种子媒体信息 :param rss_movies: 电影订阅清单 :param rss_tvs: 电视剧订阅清单 + :param site_id: 站点ID :param site_filter_rule: 站点过滤规则 :param site_cookie: 站点的Cookie :param site_parse: 是否解析种子详情 @@ -500,10 +509,15 @@ def check_torrent_rss(self, match_flag = True match_rss_info = rss_info break + # 名称匹配成功，开始过滤 if match_flag: # 解析种子详情 if site_parse: + # 站点流控 + if self.sites.check_ratelimit(site_id): + match_msg.append("触发站点流控") + return False, match_msg, match_rss_info # 检测Free torrent_attr = self.siteconf.check_torrent_attr(torrent_url=media_info.page_url, cookie=site_cookie, diff --git a/app/sites/__init__.py b/app/sites/__init__.py index 261922a81a..bea8d3227d 100644 --- a/app/sites/__init__.py +++ b/app/sites/__init__.py @@ -4,3 +4,4 @@ from .site_signin import SiteSignin from .site_subtitle import SiteSubtitle from .siteconf import SiteConf +from .site_limiter import SiteRateLimiter diff --git a/app/sites/site_limiter.py b/app/sites/site_limiter.py new file mode 100644 index 0000000000..e4d1acf125 --- /dev/null +++ b/app/sites/site_limiter.py @@ -0,0 +1,55 @@ +import time + + +class SiteRateLimiter: + def __init__(self, limit_interval: int, limit_count: int, limit_seconds: int): + """ + 限制访问频率 + :param limit_interval: 单位时间（秒） + :param limit_count: 单位时间内访问次数 + :param limit_seconds: 访问间隔（秒） + """ + self.limit_count = limit_count + self.limit_interval = limit_interval + self.limit_seconds = limit_seconds + self.last_visit_time = 0 + self.count = 0 + + def check_rate_limit(self) -> (bool, str): + """ + 检查是否超出访问频率控制 + :return: 超出返回True，否则返回False，超出时返回错误信息 + """ + current_time = time.time() + # 防问间隔时间 + if self.limit_seconds: + if current_time - self.last_visit_time < self.limit_seconds: + return True, f"触发流控规则，访问间隔不得小于 {self.limit_seconds} 秒，" \ + f"上次访问时间：{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.last_visit_time))}" + # 单位时间内访问次数 + if self.limit_interval and self.limit_count: + if current_time - self.last_visit_time > self.limit_interval: + # 计数清零 + self.count = 0 + if self.count >= self.limit_count: + return True, f"触发流控规则，{self.limit_interval} 秒内访问次数不得超过 {self.limit_count} 次，" \ + f"上次访问时间：{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.last_visit_time))}" + # 访问计数 + self.count += 1 + # 更新最后访问时间 + self.last_visit_time = current_time + # 未触发流控 + return False, "" + + +if __name__ == "__main__": + # 限制 1 分钟内最多访问 10 次，单次访问间隔不得小于 10 秒 + site_rate_limit = SiteRateLimiter(10, 60, 10) + + # 模拟访问 + for i in range(12): + if site_rate_limit.check_rate_limit(): + print("访问频率超限") + else: + print("访问成功") + time.sleep(3) diff --git a/app/sites/site_subtitle.py b/app/sites/site_subtitle.py index 1f403ffa9d..3690a5769e 100644 --- a/app/sites/site_subtitle.py +++ b/app/sites/site_subtitle.py @@ -4,6 +4,7 @@ from lxml import etree import log +from app.sites.sites import Sites from app.sites.siteconf import SiteConf from app.helper import SiteHelper from app.utils import RequestUtils, StringUtils, PathUtils, ExceptionUtils @@ -13,15 +14,17 @@ class SiteSubtitle: siteconf = None + sites = None _save_tmp_path = None def __init__(self): self.siteconf = SiteConf() + self.sites = Sites() self._save_tmp_path = Config().get_temp_path() if not os.path.exists(self._save_tmp_path): os.makedirs(self._save_tmp_path) - def download(self, media_info, cookie, ua, download_dir): + def download(self, media_info, site_id, cookie, ua, download_dir): """ 从站点下载字幕文件，并保存到本地 """ @@ -33,6 +36,11 @@ def download(self, media_info, cookie, ua, download_dir): if not download_dir: log.warn("【Sites】未找到字幕下载目录") return + + # 站点流控 + if self.sites.check_ratelimit(site_id): + return + # 读取网站代码 request = RequestUtils(cookies=cookie, headers=ua) res = request.get_res(media_info.page_url) diff --git a/app/sites/site_userinfo.py b/app/sites/site_userinfo.py index 8bee089cef..1b6a86185d 100644 --- a/app/sites/site_userinfo.py +++ b/app/sites/site_userinfo.py @@ -53,11 +53,17 @@ def __build_class(self, html_text): ExceptionUtils.exception_traceback(e) return None - def build(self, url, site_name, site_cookie=None, ua=None, emulate=None, proxy=False): + def build(self, url, site_id, site_name, + site_cookie=None, ua=None, emulate=None, proxy=False): if not site_cookie: return None session = requests.Session() log.debug(f"【Sites】站点 {site_name} url={url} site_cookie={site_cookie} ua={ua}") + + # 站点流控 + if self.sites.check_ratelimit(site_id): + return + # 检测环境，有浏览器内核的优先使用仿真签到 chrome = ChromeHelper() if emulate and chrome.get_status(): @@ -142,6 +148,7 @@ def __refresh_site_data(self, site_info): :param site_info: :return: """ + site_id = site_info.get("id") site_name = site_info.get("name") site_url = site_info.get("strict_url") if not site_url: @@ -153,6 +160,7 @@ def __refresh_site_data(self, site_info): proxy = site_info.get("proxy") try: site_user_info = self.build(url=site_url, + site_id=site_id, site_name=site_name, site_cookie=site_cookie, ua=ua, diff --git a/app/sites/sites.py b/app/sites/sites.py index 7ce5796916..d66faa22d5 100644 --- a/app/sites/sites.py +++ b/app/sites/sites.py @@ -1,8 +1,10 @@ import json from datetime import datetime +import log from app.helper import ChromeHelper, SiteHelper, DbHelper from app.message import Message +from app.sites.site_limiter import SiteRateLimiter from app.utils import RequestUtils, StringUtils from app.utils.commons import singleton from config import Config @@ -21,6 +23,7 @@ class Sites: _brush_sites = [] _statistic_sites = [] _signin_sites = [] + _limiters = {} _MAX_CONCURRENCY = 10 @@ -44,6 +47,8 @@ def init_config(self): self._statistic_sites = [] # 开启签到功能站点： self._signin_sites = [] + # 站点限速器 + self._limiters = {} # 站点图标 self.init_favicons() # 站点数据 @@ -91,6 +96,9 @@ def init_config(self): "chrome": True if site_note.get("chrome") == "Y" else False, "proxy": True if site_note.get("proxy") == "Y" else False, "subtitle": True if site_note.get("subtitle") == "Y" else False, + "limit_interval": site_note.get("limit_interval"), + "limit_count": site_note.get("limit_count"), + "limit_seconds": site_note.get("limit_seconds"), "strict_url": StringUtils.get_base_url(site_signurl or site_rssurl) } # 以ID存储 @@ -99,6 +107,18 @@ def init_config(self): site_strict_url = StringUtils.get_url_domain(site.SIGNURL or site.RSSURL) if site_strict_url: self._siteByUrls[site_strict_url] = site_info + # 初始化站点限速器 + if (site_note.get("limit_interval") + and str(site_note.get("limit_interval")).isdigit() + and site_note.get("limit_count") + and str(site_note.get("limit_count")).isdigit()) \ + or (site_note.get("limit_seconds") + and str(site_note.get("limit_seconds")).isdigit()): + self._limiters[site.ID] = SiteRateLimiter( + limit_interval=int(site_note.get("limit_interval")) * 60, + limit_count=int(site_note.get("limit_count")), + limit_seconds=int(site_note.get("limit_seconds")) + ) def init_favicons(self): """ @@ -136,6 +156,19 @@ def get_sites(self, return {} return ret_sites + def check_ratelimit(self, site_id): + """ + 检查站点是否触发流控 + :param site_id: 站点ID + :return: True为触发了流控，False为未触发 + """ + if not self._limiters.get(site_id): + return False + state, msg = self._limiters[site_id].check_rate_limit() + if msg: + log.warn(f"【Sites】站点 {self._siteByIds[site_id].get('name')} {msg}") + return state + def get_sites_by_suffix(self, suffix): """ 根据url的后缀获取站点配置 @@ -145,6 +178,12 @@ def get_sites_by_suffix(self, suffix): return self._siteByUrls[key] return {} + def get_max_site_pri(self): + """ + 获取最大站点优先级 + """ + return max([int(site.get("pri")) for site in self._siteByIds.values()]) + def get_site_dict(self, rss=False, brush=False, diff --git a/app/subscribe.py b/app/subscribe.py index b401344c5c..a61e93d64a 100644 --- a/app/subscribe.py +++ b/app/subscribe.py @@ -109,7 +109,11 @@ def add_rss_subscribe(self, mtype, name, year, return -1, "标题或类型有误", None year = int(year) if str(year).isdigit() else "" rss_sites = rss_sites or [] + if isinstance(rss_sites, str): + rss_sites = rss_sites.split(",") search_sites = search_sites or [] + if isinstance(search_sites, str): + search_sites = search_sites.split(",") over_edition = 1 if over_edition else 0 filter_rule = int(filter_rule) if str(filter_rule).isdigit() else None total_ep = int(total_ep) if str(total_ep).isdigit() else None diff --git a/app/utils/torrent.py b/app/utils/torrent.py index 47923c4731..601442238f 100644 --- a/app/utils/torrent.py +++ b/app/utils/torrent.py @@ -5,6 +5,7 @@ from bencode import bdecode +import log from app.utils.http_utils import RequestUtils from app.utils.types import MediaType from config import Config @@ -75,13 +76,49 @@ def save_torrent_file(self, url, cookie=None, ua=None, referer=None, proxy=False return None, None, "未下载到种子数据" # 解析内容格式 if req.text and str(req.text).startswith("magnet:"): + # 磁力链接 return None, req.text, "磁力链接" + elif req.text and "下载种子文件" in req.text: + # 首次下载提示页面 + skip_flag = False + try: + form = re.findall(r'(.*?)', req.text, re.S) + if form: + action = form[0][0] + inputs = re.findall(r'', form[0][1], re.S) + if action and inputs: + data = {} + for item in inputs: + data[item[0]] = item[1] + # 改写req + req = RequestUtils( + headers=ua, + cookies=cookie, + referer=referer, + proxies=Config().get_proxies() if proxy else None + ).post_res(url=action, data=data) + if req and req.status_code == 200: + # 检查是不是种子文件，如果不是抛出异常 + bdecode(req.content) + # 跳过成功 + skip_flag = True + elif req is not None: + log.warn(f"触发了站点首次种子下载，且无法自动跳过，" + f"返回码：{req.status_code}，错误原因：{req.reason}") + else: + log.warn(f"触发了站点首次种子下载，且无法自动跳过：{url}") + except Exception as err: + log.warn(f"触发了站点首次种子下载，尝试自动跳过时出现错误：{str(err)}，链接：{url}") + + if not skip_flag: + return None, None, "种子数据有误，请确认链接是否正确，如为PT站点则需手工在站点下载一次种子" else: + # 检查是不是种子文件，如果不是仍然抛出异常 try: bdecode(req.content) except Exception as err: print(str(err)) - return None, None, "种子数据有误，请确认链接是否正确，如为PT站点则需手工在站点下载一次种子" + return None, None, "种子数据有误，请确认链接是否正确" # 读取种子文件名 file_name = self.__get_url_torrent_filename(req, url) # 种子文件路径 diff --git a/app/utils/types.py b/app/utils/types.py index 43dccc2ed4..37b697c6a2 100644 --- a/app/utils/types.py +++ b/app/utils/types.py @@ -133,6 +133,10 @@ class EventType(Enum): PluginReload = "plugin.reload" # 豆瓣想看同步 DoubanSync = "douban.sync" + # 辅种任务开始 + AutoSeedStart = "autoseed.start" + # 刷新媒体库 + RefreshMediaServer = "refresh.mediaserver" # 系统配置Key字典 diff --git a/config/sites.dat b/config/sites.dat index 0ac5e55d0b..f9887aa0c9 100644 Binary files a/config/sites.dat and b/config/sites.dat differ diff --git a/db_scripts/versions/1f5cc26cdd3d_1_2_3.py b/db_scripts/versions/1f5cc26cdd3d_1_2_3.py new file mode 100644 index 0000000000..692008b4a5 --- /dev/null +++ b/db_scripts/versions/1f5cc26cdd3d_1_2_3.py @@ -0,0 +1,31 @@ +"""1.2.3 + +Revision ID: 1f5cc26cdd3d +Revises: 13a58bd5311f +Create Date: 2023-04-07 08:23:05.282129 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '1f5cc26cdd3d' +down_revision = '13a58bd5311f' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # 1.2.2 + try: + with op.batch_alter_table("SITE_BRUSH_TASK") as batch_op: + batch_op.add_column(sa.Column('SAVEPATH', sa.Text, nullable=True)) + except Exception as e: + pass + # ### end Alembic commands ### + + +def downgrade() -> None: + pass diff --git a/requirements.txt b/requirements.txt index aa74b164cf..db2a620513 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ func_timeout==4.3.5 greenlet==1.1.3.post0 h11==0.12.0 humanize==4.4.0 -ics==0.7.2 +icalendar==5.0.4 idna==3.3 influxdb==5.3.1 itsdangerous==2.1.2 diff --git a/run.py b/run.py index 5bf230dc88..545957e281 100644 --- a/run.py +++ b/run.py @@ -6,24 +6,21 @@ warnings.filterwarnings('ignore') # 运行环境判断 -is_windows_exe = getattr(sys, 'frozen', False) and (os.name == "nt") +is_executable = getattr(sys, 'frozen', False) +is_windows_exe = is_executable and (os.name == "nt") if is_windows_exe: # 托盘相关库 import threading from package.trayicon import TrayIcon, NullWriter - # 初始化环境变量 - os.environ["NASTOOL_CONFIG"] = os.path.join(os.path.dirname(sys.executable), - "config", - "config.yaml").replace("\\", "/") - os.environ["NASTOOL_LOG"] = os.path.join(os.path.dirname(sys.executable), - "config", - "logs").replace("\\", "/") +if is_executable: + # 可执行文件初始化环境变量 + config_path = os.path.join(os.path.dirname(sys.executable), "config").replace("\\", "/") + os.environ["NASTOOL_CONFIG"] = os.path.join(config_path, "config.yaml").replace("\\", "/") + os.environ["NASTOOL_LOG"] = os.path.join(config_path, "logs").replace("\\", "/") try: - config_dir = os.path.join(os.path.dirname(sys.executable), - "config").replace("\\", "/") - if not os.path.exists(config_dir): - os.makedirs(config_dir) + if not os.path.exists(config_path): + os.makedirs(config_path) except Exception as err: print(str(err)) @@ -135,9 +132,9 @@ def traystart(): if len(os.popen("tasklist| findstr %s" % os.path.basename(sys.executable), 'r').read().splitlines()) <= 2: p1 = threading.Thread(target=traystart, daemon=True) p1.start() - else: - # 初始化浏览器驱动 - init_chrome() + + # 初始化浏览器驱动 + init_chrome() # gunicorn 启动 App.run(**get_run_config(is_windows_exe)) diff --git a/third_party/qbittorrent-api b/third_party/qbittorrent-api index 096fdb85de..99bfc7f50e 160000 --- a/third_party/qbittorrent-api +++ b/third_party/qbittorrent-api @@ -1 +1 @@ -Subproject commit 096fdb85dedcf5e786c03d25e3b9a50e5047f415 +Subproject commit 99bfc7f50ecfa120817362b61e5d2b109986aa24 diff --git a/third_party/slack_bolt b/third_party/slack_bolt index f52b7ff69b..eae0d4e47b 160000 --- a/third_party/slack_bolt +++ b/third_party/slack_bolt @@ -1 +1 @@ -Subproject commit f52b7ff69b58ee465e8738ad0fbfc1703b15f37f +Subproject commit eae0d4e47b4dd8234cd555c2ee0c0bb92a0ef6bd diff --git a/third_party/transmission-rpc b/third_party/transmission-rpc index d8f77c20ae..34607a76a8 160000 --- a/third_party/transmission-rpc +++ b/third_party/transmission-rpc @@ -1 +1 @@ -Subproject commit d8f77c20ae0d8c165812a38571b9514aad81a32e +Subproject commit 34607a76a88438d81c5aaea83057e020d3986f26 diff --git a/version.py b/version.py index 1e47be5099..24290d55ad 100644 --- a/version.py +++ b/version.py @@ -1 +1 @@ -APP_VERSION = 'v3.1.3' +APP_VERSION = 'v3.1.4' diff --git a/web/action.py b/web/action.py index adeebcfde5..424539d80c 100644 --- a/web/action.py +++ b/web/action.py @@ -21,9 +21,9 @@ from app.filetransfer import FileTransfer from app.filter import Filter from app.helper import DbHelper, ProgressHelper, ThreadHelper, \ - MetaHelper, DisplayHelper, WordsHelper, CookieCloudHelper, IndexerHelper + MetaHelper, DisplayHelper, WordsHelper, IndexerHelper from app.indexer import Indexer -from app.media import Category, Media, Bangumi, DouBan +from app.media import Category, Media, Bangumi, DouBan, Scraper from app.media.meta import MetaInfo, MetaBase from app.mediaserver import MediaServer from app.message import Message, MessageCenter @@ -199,7 +199,6 @@ def __init__(self): "set_system_config": self.__set_system_config, "get_site_user_statistics": self.get_site_user_statistics, "send_custom_message": self.send_custom_message, - "cookiecloud_sync": self.__cookiecloud_sync, "media_detail": self.media_detail, "media_similar": self.__media_similar, "media_recommendations": self.__media_recommendations, @@ -1964,6 +1963,7 @@ def __add_brushtask(self, data): brushtask_state = data.get("brushtask_state") brushtask_rssurl = data.get("brushtask_rssurl") brushtask_label = data.get("brushtask_label") + brushtask_savepath = data.get("brushtask_savepath") brushtask_transfer = 'Y' if data.get("brushtask_transfer") else 'N' brushtask_sendmessage = 'Y' if data.get( "brushtask_sendmessage") else 'N' @@ -2015,6 +2015,7 @@ def __add_brushtask(self, data): "downloader": brushtask_downloader, "seed_size": brushtask_totalsize, "label": brushtask_label, + "savepath": brushtask_savepath, "transfer": brushtask_transfer, "state": brushtask_state, "rss_rule": rss_rule, @@ -2398,6 +2399,7 @@ def get_recommend(self, data): mtype = MediaType.MOVIE if SubType in MovieTypes else MediaType.TV # 过滤参数 with_genres with_original_language params = data.get("params") or {} + res_list = Media().get_tmdb_discover(mtype=mtype, page=CurrentPage, params=params) elif Type == "DOUBANTAG": # 豆瓣发现 @@ -4096,11 +4098,10 @@ def __media_path_scrap(data): """ 刮削媒体文件夹或文件 """ - # 触发字幕下载事件 - EventManager().send_event(EventType.MediaScrapStart, { - "path": data.get("path"), - "force": True - }) + path = data.get("path") + if not path: + return {"code": -1, "msg": "请指定刮削路径"} + ThreadHelper().start_thread(Scraper().folder_scraper, (path, None, 'force_all')) return {"code": 0, "msg": "刮削任务已提交，正在后台运行。"} @staticmethod @@ -4452,51 +4453,6 @@ def get_rmt_modes(): "name": name.value } for value, name in RmtModes.items()] - def __cookiecloud_sync(self, data): - """ - CookieCloud数据同步 - """ - server = data.get("server") - key = data.get("key") - password = data.get("password") - # 保存设置 - SystemConfig().set_system_config(key=SystemConfigKey.CookieCloud, - value={ - "server": server, - "key": key, - "password": password - }) - # 同步数据 - contents, retmsg = CookieCloudHelper(server=server, - key=key, - password=password).download_data() - if not contents: - return {"code": 1, "msg": retmsg} - # 整理数据,使用domain域名的最后两级作为分组依据 - domain_groups = defaultdict(list) - for site, cookies in contents.items(): - for cookie in cookies: - domain_parts = cookie["domain"].split(".")[-2:] - domain_key = tuple(domain_parts) - domain_groups[domain_key].append(cookie) - success_count = 0 - for domain, content_list in domain_groups.items(): - if not content_list: - continue - cookie_str = ";".join( - [f"{content['name']}={content['value']}" for content in content_list] - ) - site_info = Sites().get_sites_by_suffix(".".join(domain)) - if site_info: - self.dbhelper.update_site_cookie_ua(tid=site_info.get("id"), - cookie=cookie_str) - success_count += 1 - if success_count: - # 重载站点信息 - Sites().init_config() - return {"code": 0, "msg": f"成功更新 {success_count} 个站点的Cookie数据"} - return {"code": 0, "msg": "同步完成，但未更新任何站点的Cookie！"} - def media_detail(self, data): """ 获取媒体详情 diff --git a/web/apiv1.py b/web/apiv1.py index bc235c13e6..2e07efc651 100644 --- a/web/apiv1.py +++ b/web/apiv1.py @@ -1125,8 +1125,8 @@ class SubscribeAdd(ClientResource): parser.add_argument('rssid', type=int, help='已有订阅ID', location='form') parser.add_argument('mediaid', type=str, help='TMDBID/DB:豆瓣ID', location='form') parser.add_argument('fuzzy_match', type=int, help='模糊匹配（0-否/1-是）', location='form') - parser.add_argument('rss_sites', type=list, help='RSS站点', location='form') - parser.add_argument('search_sites', type=list, help='搜索站点', location='form') + parser.add_argument('rss_sites', type=str, help='RSS站点（,号分隔）', location='form') + parser.add_argument('search_sites', type=str, help='搜索站点（,号分隔）', location='form') parser.add_argument('over_edition', type=int, help='洗版（0-否/1-是）', location='form') parser.add_argument('filter_restype', type=str, help='资源类型', location='form') parser.add_argument('filter_pix', type=str, help='分辨率', location='form') diff --git a/web/backend/user.cp310-win_amd64.pyd b/web/backend/user.cp310-win_amd64.pyd index ca4171f6d6..8b9fdbf57c 100644 Binary files a/web/backend/user.cp310-win_amd64.pyd and b/web/backend/user.cp310-win_amd64.pyd differ diff --git a/web/backend/user.cpython-310-aarch64-linux-gnu.so b/web/backend/user.cpython-310-aarch64-linux-gnu.so index 58ace16064..fa323d795d 100644 Binary files a/web/backend/user.cpython-310-aarch64-linux-gnu.so and b/web/backend/user.cpython-310-aarch64-linux-gnu.so differ diff --git a/web/backend/user.cpython-310-darwin.so b/web/backend/user.cpython-310-darwin.so index 529a62067d..22e3560fd3 100755 Binary files a/web/backend/user.cpython-310-darwin.so and b/web/backend/user.cpython-310-darwin.so differ diff --git a/web/backend/user.cpython-310-x86_64-linux-gnu.so b/web/backend/user.cpython-310-x86_64-linux-gnu.so index f5b954af57..9a32e3698f 100644 Binary files a/web/backend/user.cpython-310-x86_64-linux-gnu.so and b/web/backend/user.cpython-310-x86_64-linux-gnu.so differ diff --git a/web/backend/web_utils.py b/web/backend/web_utils.py index 6398dea904..43c1298d1a 100644 --- a/web/backend/web_utils.py +++ b/web/backend/web_utils.py @@ -66,7 +66,7 @@ def get_latest_version(): return None, None, False @staticmethod - def get_mediainfo_from_id(mtype, mediaid): + def get_mediainfo_from_id(mtype, mediaid, wait=False): """ 根据TMDB/豆瓣/BANGUMI获取媒体信息 """ @@ -76,7 +76,7 @@ def get_mediainfo_from_id(mtype, mediaid): if str(mediaid).startswith("DB:"): # 豆瓣 doubanid = mediaid[3:] - info = DouBan().get_douban_detail(doubanid=doubanid, mtype=mtype) + info = DouBan().get_douban_detail(doubanid=doubanid, mtype=mtype, wait=wait) if not info: return None title = info.get("title") diff --git a/web/main.py b/web/main.py index f5317406bc..884dd5a0d4 100644 --- a/web/main.py +++ b/web/main.py @@ -20,7 +20,7 @@ redirect, Response from flask_compress import Compress from flask_login import LoginManager, login_user, login_required, current_user -from ics import Calendar, Event +from icalendar import Calendar, Event, Alarm from werkzeug.middleware.proxy_fix import ProxyFix import log @@ -1727,17 +1727,35 @@ def upload(): @App.route('/ical') @require_auth(force=False) def ical(): - ICal = Calendar() + # 是否设置提醒开关 + remind = request.args.get("remind") + cal = Calendar() RssItems = WebAction().get_ical_events().get("result") for item in RssItems: event = Event() - event.name = f'{item.get("type")}：{item.get("title")}' + event.add('summary', f'{item.get("type")}：{item.get("title")}') if not item.get("start"): continue - event.begin = datetime.datetime.strptime(item.get("start"), '%Y-%m-%d') - event.duration = datetime.timedelta(hours=1) - ICal.events.add(event) - response = Response(ICal.serialize_iter(), mimetype='text/calendar') + event.add('dtstart', + datetime.datetime.strptime(item.get("start"), + '%Y-%m-%d') + + datetime.timedelta(hours=8)) + event.add('dtend', + datetime.datetime.strptime(item.get("start"), + '%Y-%m-%d') + + datetime.timedelta(hours=9)) + + # 添加事件提醒 + if remind: + alarm = Alarm() + alarm.add('trigger', datetime.timedelta(minutes=30)) + alarm.add('action', 'DISPLAY') + event.add_component(alarm) + + cal.add_component(event) + + # 返回日历文件 + response = Response(cal.to_ical(), mimetype='text/calendar') response.headers['Content-Disposition'] = 'attachment; filename=nastool.ics' return response diff --git a/web/static/components/plugin/modal/index.js b/web/static/components/plugin/modal/index.js index 39de886368..6ca851befb 100644 --- a/web/static/components/plugin/modal/index.js +++ b/web/static/components/plugin/modal/index.js @@ -73,7 +73,10 @@ export class PluginModal extends CustomElement { __render_details(field) { let title = field["summary"]; let tooltip = field["tooltip"]; - return html`
+ let id = field["id"]; + let hidden = field["hidden"]; + let open = field["open"]; + return html`
${title} ${this.__render_note(tooltip)} @@ -116,11 +119,12 @@ export class PluginModal extends CustomElement { let required = field_content["required"]; let tooltip = field_content["tooltip"]; let id = field_content["id"]; + let onclick = field_content["onclick"]; let checkbox; if (this.config[id]) { - checkbox = html`` + checkbox = html`` } else { - checkbox = html`` + checkbox = html`` } return html`
@@ -142,6 +146,7 @@ export class PluginModal extends CustomElement { let id = content[index]["id"]; let options = content[index]["options"]; let default_value = content[index]["default"]; + let onchange = content[index]["onchange"]; let text_options = html``; for (let option in options) { if (this.config[id]) { @@ -159,7 +164,7 @@ export class PluginModal extends CustomElement { text_content = html`
- ${text_options}
` @@ -198,13 +203,19 @@ export class PluginModal extends CustomElement { console.log() let content = field_content["content"]; let id = field_content["id"]; + let radio = field_content["radio"]; let text_options = html``; for (let option in content) { let checkbox; + let onclick = ""; + // 单选 + if (radio) { + onclick = `check_selectgroup_raido(this)` + } if (this.config[id] && this.config[id].includes(option)) { - checkbox = html`` + checkbox = html`` } else { - checkbox = html`` + checkbox = html`` } text_options = html`${text_options}
+
+
+
+ + +
+
- - +
-
+
+
+
- - +
@@ -718,6 +736,7 @@
刷流种子明细
$("#brushtask_include").val(ret.task.rss_rule.include); $("#brushtask_exclude").val(ret.task.rss_rule.exclude); $("#brushtask_label").val(ret.task.label); + $("#brushtask_savepath").val(ret.task.savepath); //种子大小 if (ret.task.rss_rule.size) { let size_str = ret.task.rss_rule.size.split('#'); @@ -881,6 +900,7 @@
刷流种子明细
} let brushtask_rssurl = $("#brushtask_rssurl").val(); let brushtask_label = $("#brushtask_label").val(); + let brushtask_savepath = $("#brushtask_savepath").val(); let brushtask_state = $("#brushtask_state").val(); let brushtask_transfer = $("#brushtask_transfer").prop('checked'); let brushtask_sendmessage = $("#brushtask_sendmessage").prop('checked'); @@ -1051,6 +1071,7 @@
刷流种子明细
brushtask_downloader: brushtask_downloader, brushtask_totalsize: brushtask_totalsize, brushtask_label: brushtask_label, + brushtask_savepath: brushtask_savepath, brushtask_state: brushtask_state, brushtask_transfer: brushtask_transfer, brushtask_sendmessage: brushtask_sendmessage, diff --git a/web/templates/site/site.html b/web/templates/site/site.html index 490581094f..2e32e6c2d2 100644 --- a/web/templates/site/site.html +++ b/web/templates/site/site.html @@ -19,13 +19,6 @@

{{ SVG.plus() }} - - {{ SVG.refresh_dot() }} - CookieCloud - - - {{ SVG.refresh_dot() }} - {{ SVG.activity_heartbeat() }} 站点测试 @@ -302,6 +295,34 @@

+
+ + 站点流控规则 ? + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+
+
-