Added parallel processing of video clips

cabernetwork · Dec 2, 2023 · a128521 · a128521
1 parent a6ae9df
commit a128521
Show file tree

Hide file tree

Showing 10 changed files with 253 additions and 190 deletions.
diff --git a/lib/clients/channels/channels.py b/lib/clients/channels/channels.py
@@ -167,6 +167,7 @@ def get_channels_json(_config, _base_url, _namespace, _instance, _plugins):
             config_section = utils.instance_config_section(sid_data['namespace'], sid_data['instance'])
             if not _config[config_section]['enabled']:
                 continue
+            sids_processed.append(sid)
             stream = _config[config_section]['player-stream_type']
             if stream == 'm3u8redirect':
                 uri = sid_data['json']['stream_url']
@@ -195,7 +196,6 @@ def get_channels_xml(_config, _base_url, _namespace, _instance, _plugins):
         for sid_data in sid_data_list:
             if sid in sids_processed:
                 continue
-            sids_processed.append(sid)
             if not sid_data['enabled']:
                 continue
             if not _plugins.get(sid_data['namespace']):
@@ -205,9 +205,11 @@ def get_channels_xml(_config, _base_url, _namespace, _instance, _plugins):
             if not _plugins[sid_data['namespace']] \
                     .plugin_obj.instances[sid_data['instance']].enabled:
                 continue
+
             config_section = utils.instance_config_section(sid_data['namespace'], sid_data['instance'])
             if not _config[config_section]['enabled']:
                 continue
+            sids_processed.append(sid)
             stream = _config[config_section]['player-stream_type']
             if stream == 'm3u8redirect':
                 uri = sid_data['json']['stream_url']

diff --git a/lib/common/decorators.py b/lib/common/decorators.py
@@ -69,36 +69,12 @@ def wrapper_func(self, *args, **kwargs):
                 ex_save = ex
                 self.logger.info("ConnectionResetError in function {}(), retrying {} {} {}"
                                  .format(f.__qualname__, os.getpid(), str(ex_save), str(arg0)))
-            except (requests.exceptions.HTTPError, urllib.error.HTTPError) as ex:
-                ex_save = ex
-                self.logger.info("HTTPError in function {}(), retrying {} {} {}"
-                                 .format(f.__qualname__, os.getpid(), str(ex_save), str(arg0), ))
-            except urllib.error.URLError as ex:
-                ex_save = ex
-                if isinstance(ex.reason, ConnectionRefusedError):
-                    self.logger.info("URLError:ConnectionRefusedError in function {}(): {} {} {}"
-                                     .format(f.__qualname__, os.getpid(), str(ex_save), str(arg0)))
-                    count = 5
-                    while count > 0:
-                        try:
-                            x = f(self, *args, **kwargs)
-                            return x
-                        except urllib.error.URLError as ex2:
-                            self.logger.debug("{} URLError:ConnectionRefusedError in function {}(): {} {} {}"
-                                              .format(count, f.__qualname__, os.getpid(), str(ex_save), str(arg0)))
-                            count -= 1
-                            time.sleep(.5)
-                        except Exception as ex3:
-                            break
-                else:
-                    self.logger.info("URLError in function {}(), retrying (): {} {} {}"
-                                     .format(f.__qualname__, os.getpid(), str(ex_save), str(arg0)))
             except requests.exceptions.InvalidURL as ex:
                 ex_save = ex
                 self.logger.info("InvalidURL Error in function {}(), retrying {} {} {}"
                                  .format(f.__qualname__, os.getpid(), str(ex_save), str(arg0)))
 
-            except (socket.timeout, requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout) as ex:
+            except (socket.timeout, requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, httpx.ReadTimeout, httpx.ConnectTimeout) as ex:
                 ex_save = ex
                 self.logger.info("Socket Timeout Error in function {}(), retrying {} {} {}"
                                  .format(f.__qualname__, os.getpid(), str(ex_save), str(arg0)))
@@ -147,6 +123,30 @@ def wrapper_func(self, *args, **kwargs):
                 ex_save = ex
                 self.logger.info('InvalidURL Error, encoding and trying again. In function {}() {} {} {}'
                                  .format(f.__qualname__, os.getpid(), str(ex_save), str(arg0)))
+            except (requests.exceptions.HTTPError, urllib.error.HTTPError, httpx.HTTPError) as ex:
+                ex_save = ex
+                self.logger.info("HTTPError in function {}(), retrying {} {} {}"
+                                 .format(f.__qualname__, os.getpid(), str(ex_save), str(arg0), ))
+            except urllib.error.URLError as ex:
+                ex_save = ex
+                if isinstance(ex.reason, ConnectionRefusedError):
+                    self.logger.info("URLError:ConnectionRefusedError in function {}(): {} {} {}"
+                                     .format(f.__qualname__, os.getpid(), str(ex_save), str(arg0)))
+                    count = 5
+                    while count > 0:
+                        try:
+                            x = f(self, *args, **kwargs)
+                            return x
+                        except urllib.error.URLError as ex2:
+                            self.logger.debug("{} URLError:ConnectionRefusedError in function {}(): {} {} {}"
+                                              .format(count, f.__qualname__, os.getpid(), str(ex_save), str(arg0)))
+                            count -= 1
+                            time.sleep(.5)
+                        except Exception as ex3:
+                            break
+                else:
+                    self.logger.info("URLError in function {}(), retrying (): {} {} {}"
+                                     .format(f.__qualname__, os.getpid(), str(ex_save), str(arg0)))
             except (requests.exceptions.ProxyError, requests.exceptions.SSLError, \
                     requests.exceptions.TooManyRedirects, requests.exceptions.InvalidHeader, \
                     requests.exceptions.InvalidProxyURL, requests.exceptions.ChunkedEncodingError, \

diff --git a/lib/common/utils.py b/lib/common/utils.py
@@ -37,7 +37,7 @@
 
 import lib.common.exceptions as exceptions
 
-VERSION = '0.9.14.00-RC01'
+VERSION = '0.9.14.00-RC02'
 CABERNET_URL = 'https://github.com/cabernetwork/cabernet'
 CABERNET_ID = 'cabernet'
 CABERNET_REPO = 'manifest.json'

diff --git a/lib/main.py b/lib/main.py
@@ -43,9 +43,9 @@
     try:
         import httpx
     except ImportError:
-        pip(['install', 'httpx'])
+        pip(['install', 'httpx[http2]'])
     except ModuleNotFoundError:
-        print('Unable to install required httpx module')
+        print('Unable to install required httpx[http2] module')
 except (ImportError, ModuleNotFoundError):
     print('Unable to load pip module to install required modules')
 

diff --git a/lib/plugins/plugin_channels.py b/lib/plugins/plugin_channels.py
@@ -86,7 +86,7 @@ def get_uri_json_data(self, _uri):
         header = {
             'Content-Type': 'application/json',
             'User-agent': utils.DEFAULT_USER_AGENT}
-        resp = self.plugin_obj.http_session.get(_uri, headers=header, timeout=(8, 8))
+        resp = self.plugin_obj.http_session.get(_uri, headers=header, timeout=8)
         x = resp.json()
         resp.raise_for_status()
         return x
@@ -99,9 +99,9 @@ def get_uri_data(self, _uri, _header=None, _data=None):
         else:
             header = _header
         if _data:
-            resp = self.plugin_obj.http_session.post(_uri, headers=header, data=_data, timeout=(8, 8))
+            resp = self.plugin_obj.http_session.post(_uri, headers=header, data=_data, timeout=8)
         else:
-            resp = self.plugin_obj.http_session.get(_uri, headers=header, timeout=(8, 8))
+            resp = self.plugin_obj.http_session.get(_uri, headers=header, timeout=8)
         x = resp.content
         return x
 
@@ -180,7 +180,7 @@ def get_thumbnail_size(self, _thumbnail, _ch_uid, ):
              'Accept-Encoding': 'identity',
              'Connection': 'Keep-Alive'
              }
-        resp = self.plugin_obj.http_session.get(_thumbnail, headers=h, timeout=(8, 8))
+        resp = self.plugin_obj.http_session.get(_thumbnail, headers=h, timeout=8)
         resp.raise_for_status()
         img_blob = resp.content
         fp = io.BytesIO(img_blob)

diff --git a/lib/plugins/plugin_epg.py b/lib/plugins/plugin_epg.py
@@ -65,7 +65,7 @@ def get_uri_data(self, _uri, _header=None):
             header = {'User-agent': utils.DEFAULT_USER_AGENT}
         else:
             header = _header
-        resp = self.plugin_obj.http_session.get(_uri, headers=header, timeout=(4, 8))
+        resp = self.plugin_obj.http_session.get(_uri, headers=header, timeout=8)
         x = resp.json()
         resp.raise_for_status()
         return x

diff --git a/lib/plugins/plugin_programs.py b/lib/plugins/plugin_programs.py
@@ -60,7 +60,7 @@ def get_uri_data(self, _uri, _header=None):
             header = {'User-agent': utils.DEFAULT_USER_AGENT}
         else:
             header = _header
-        resp = self.plugin_obj.http_session.get(_uri, headers=header, timeout=(4, 8))
+        resp = self.plugin_obj.http_session.get(_uri, headers=header, timeout=8)
         x = resp.json()
         resp.raise_for_status()
         return x

diff --git a/lib/plugins/repo_handler.py b/lib/plugins/repo_handler.py
@@ -149,7 +149,7 @@ def update_plugins(self, _repo_settings):
     def get_uri_data(self, _uri):
         header = {
             'User-agent': utils.DEFAULT_USER_AGENT}
-        resp = RepoHandler.http_session.get(_uri, headers=header, timeout=(4, 8))
+        resp = RepoHandler.http_session.get(_uri, headers=header, timeout=8)
         x = resp.content
         resp.raise_for_status()
         return x