build: Allow plugins to control how many assets to process in one job

blend2bam has a BATCH_SIZE of 3, which was picked by experimentation. A more complex/dynamic/configurable mechanism would sill be useful.
Moguri · Aug 8, 2023 · 9e9a73c · 9e9a73c
1 parent c9e030c
commit 9e9a73c
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/pman/_build.py b/pman/_build.py
@@ -110,7 +110,6 @@ def skip_build(converter, asset):
         return False
 
     max_workers = config['build']['jobs']
-    max_batch = 1
     if max_workers <= 0:
         max_workers = None
     pool = concurrent.futures.ProcessPoolExecutor(max_workers=max_workers)
@@ -125,6 +124,7 @@ def skip_build(converter, asset):
         if not assets:
             continue
 
+        max_batch = getattr(converter.plugin, 'BATCH_SIZE', 1)
         chunk_it = iter(assets)
         while chunk := tuple(itertools.islice(chunk_it, max_batch)):
             jobstr = f'{converter.name}: {", ".join(get_rel_path(config, i) for i in chunk)}'

diff --git a/pman/plugins/blend2bam.py b/pman/plugins/blend2bam.py
@@ -14,6 +14,8 @@ class Blend2BamPlugin:
         )
     ]
 
+    BATCH_SIZE = 3
+
 
     CONFIG_DEFAULTS = {
         'blend2bam': {