diff --git a/src/pycestorieseditor/ceevents.py b/src/pycestorieseditor/ceevents.py index 57a4718..9a1c4b9 100644 --- a/src/pycestorieseditor/ceevents.py +++ b/src/pycestorieseditor/ceevents.py @@ -215,7 +215,7 @@ def filter_ceevent(ceevent: Ceevent, ceeventname): yield name, cname -def process_module(xmlfiles: list): +def process_module(xmlfiles: list, cb=None): """Process the various xml files present in a given module Args: @@ -227,26 +227,32 @@ def process_module(xmlfiles: list): xsd = get_xsdfile() # Pool(processes) uses os.cpu_count() if none value is provided + errcount = 0 with multiprocessing.Pool() as pool: - res = pool.starmap( - process_file, ((xmlfile, xsd, parser) for xmlfile in xmlfiles), chunksize=4 + chunks = 1 + if os.cpu_count() < len(xmlfiles): + chunks = int(len(xmlfiles) / os.cpu_count()) + 1 + res = pool.starmap_async( + process_file, ((xmlfile, xsd, parser) for xmlfile in xmlfiles), chunksize=chunks ) - - errcount = 0 - for bucket, skills, errs in res: - errcount += errs - for ceevent in bucket: - if ceevent.name.value in ebucket.keys(): - logger.warning( - "Override of '%s' already present in bucket. (trigger: %s)", - ceevent.name.value, - ceevent.xmlfile, - ) - ebucket[ceevent.name.value] = ceevent - for skill, eventname in skills: - for s in skill: - indexes['skills'].setdefault(s, []) - indexes['skills'][s].append(eventname) + for bucket, skills, errs in res.get(): + errcount += errs + for ceevent in bucket: + if cb: + cb() + if ceevent.name.value in ebucket.keys(): + logger.warning( + "Override of '%s' already present in bucket. (trigger: %s)", + ceevent.name.value, + ceevent.xmlfile, + ) + ebucket[ceevent.name.value] = ceevent + for skill, eventname in skills: + if cb: + cb() + for s in skill: + indexes['skills'].setdefault(s, []) + indexes['skills'][s].append(eventname) return errcount diff --git a/src/pycestorieseditor/wxlaunch.py b/src/pycestorieseditor/wxlaunch.py index f0c77f6..c77850c 100644 --- a/src/pycestorieseditor/wxlaunch.py +++ b/src/pycestorieseditor/wxlaunch.py @@ -260,11 +260,21 @@ def _button_validate_pressed(self, evt): if not get_xsdfile(): self._show_warning("Please select a valid XSD file before validating.") return + + dialog = wx.ProgressDialog( + "Validation", "Validating xml files...", maximum=100, style=wx.PD_APP_MODAL | wx.PD_AUTO_HIDE | wx.PD_ELAPSED_TIME | wx.PD_SMOOTH + ) + + def pulse(): + wx.Yield() + dialog.Pulse() + create_ebucket() init_index() errs = 0 for module in self._paths.values(): - err = process_module([str(f) for f in module.events_files]) + dialog.Pulse("Processing module... {}".format(module.name)) + err = process_module([str(f) for f in module.events_files], pulse) errs += err if errs > 0: self._show_warning(f"{errs} xml files couldn't be validated, please check the logs.")