From 37d5fb74c20475aa0974d7f40c09f39a9ca2fe6c Mon Sep 17 00:00:00 2001 From: eAlasdair Date: Wed, 15 Jan 2020 15:04:21 +1300 Subject: [PATCH 1/9] Hide unnecessary weasyprint output --- csunplugged/resources/utils/BaseResourceGenerator.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/csunplugged/resources/utils/BaseResourceGenerator.py b/csunplugged/resources/utils/BaseResourceGenerator.py index 6ecbc8686..3687edf72 100644 --- a/csunplugged/resources/utils/BaseResourceGenerator.py +++ b/csunplugged/resources/utils/BaseResourceGenerator.py @@ -166,6 +166,10 @@ def pdf(self, resource_name): """ # Only import weasyprint when required as production environment # does not have it installed. + # Also adapt logging so that weasyprint doesn't print warnings when it detects unknown css properties: + import logging + logger = logging.getLogger("weasyprint").setLevel(100) + from weasyprint import HTML, CSS context = dict() context["resource"] = resource_name @@ -247,6 +251,10 @@ def write_thumbnail(self, thumbnail_data, resource_name, path): """ # Only import weasyprint when required as production environment # does not have it installed. + # Also adapt logging so that weasyprint doesn't print warnings when it detects unknown css properties: + import logging + logger = logging.getLogger("weasyprint").setLevel(100) + from weasyprint import HTML, CSS context = dict() context["resource"] = resource_name From 334a1537590c8513f903d4c056088217a1585558 Mon Sep 17 00:00:00 2001 From: eAlasdair Date: Thu, 16 Jan 2020 16:01:59 +1300 Subject: [PATCH 2/9] add first succsessful partial-multiprocessing implementation --- .../management/commands/makeresources.py | 65 +++++++++++++------ 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/csunplugged/resources/management/commands/makeresources.py b/csunplugged/resources/management/commands/makeresources.py index 7e173b4fe..39e829933 100644 --- a/csunplugged/resources/management/commands/makeresources.py +++ b/csunplugged/resources/management/commands/makeresources.py @@ -2,6 +2,7 @@ import os import os.path +import time from urllib.parse import urlencode from django.core.management.base import BaseCommand from django.http.request import QueryDict @@ -11,6 +12,9 @@ from resources.utils.get_resource_generator import get_resource_generator from resources.utils.resource_valid_configurations import resource_valid_configurations from resources.utils.resource_parameters import EnumResourceParameter +from multiprocessing.dummy import Pool + +THREADS = 6 class Command(BaseCommand): @@ -51,25 +55,42 @@ def handle(self, *args, **options): generation_languages.append(language_code) for resource in resources: - print("Creating {}".format(resource.name)) - - # TODO: Import repeated in next for loop, check alternatives - empty_generator = get_resource_generator(resource.generator_module) - if not all([isinstance(option, EnumResourceParameter) - for option in empty_generator.get_options().values()]): - raise TypeError("Only EnumResourceParameters are supported for pre-generation") - valid_options = {option.name: list(option.valid_values.keys()) - for option in empty_generator.get_options().values()} - combinations = resource_valid_configurations(valid_options) - - # TODO: Create PDFs in parallel - - # Create PDF for all possible combinations - for combination in combinations: - for language_code in generation_languages: - self.create_resource_pdf(resource, combination, language_code, base_path) - - def create_resource_pdf(self, resource, combination, language_code, base_path): + extras = [base_path, generation_languages] + self.create_pdfs_for_resource(resource, extras) + + def create_pdfs_for_resource(self, resource, extras): + """TODO""" + base_path = extras[0] + generation_languages = extras[1] + pool = Pool(THREADS) + + # TODO: Import repeated in next for loop, check alternatives + empty_generator = get_resource_generator(resource.generator_module) + if not all([isinstance(option, EnumResourceParameter) + for option in empty_generator.get_options().values()]): + raise TypeError("Only EnumResourceParameters are supported for pre-generation") + valid_options = {option.name: list(option.valid_values.keys()) + for option in empty_generator.get_options().values()} + combinations = resource_valid_configurations(valid_options) + + # Create parameter sets for all possible combinations of resource + parameters = [] + for combination in combinations: + for language_code in generation_languages: + parameters.append([resource, combination, language_code, base_path]) + + # Generate resources + print("Creating {} PDFs with {} processes for '{}'...".format(len(parameters), THREADS, resource.name)) + start = time.process_time() + try: + pool.map(self.create_resource_pdf, parameters) + except: # sqlite3.ProgrammingError: + print("Error using parallel processing, creating {} PDFs in series for '{}'...".format(len(parameters), resource.name)) + for parameter_set in parameters: + self.create_resource_pdf(parameter_set) + print("Done, time taken: {}s.".format(time.process_time() - start)) + + def create_resource_pdf(self, parameter_set): """Create a given resource PDF. Args: @@ -78,13 +99,17 @@ def create_resource_pdf(self, resource, combination, language_code, base_path): language_code (str): Code for language. base_path (str): Base path for outputting P """ + resource = parameter_set[0] + combination = parameter_set[1] + language_code = parameter_set[2] + base_path = parameter_set[3] print(" - Creating PDF in '{}'".format(language_code)) with translation.override(language_code): if resource.copies: combination["copies"] = settings.RESOURCE_COPY_AMOUNT requested_options = QueryDict(urlencode(combination, doseq=True)) generator = get_resource_generator(resource.generator_module, requested_options) - (pdf_file, filename) = generator.pdf(resource.name) + (pdf_file, filename) = generator.pdf(resource.name) ##Breaks here SQLite object pdf_directory = os.path.join(base_path, resource.slug, language_code) if not os.path.exists(pdf_directory): From 479f5f68a5168504e00dea65c68662e422aa2007 Mon Sep 17 00:00:00 2001 From: eAlasdair Date: Thu, 16 Jan 2020 16:29:38 +1300 Subject: [PATCH 3/9] add first succsessful partial-multiprocessing implementation for thumbnails --- .../commands/makeresourcethumbnails.py | 38 +++++++++++++++---- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/csunplugged/resources/management/commands/makeresourcethumbnails.py b/csunplugged/resources/management/commands/makeresourcethumbnails.py index b5d3bd272..e413069fd 100644 --- a/csunplugged/resources/management/commands/makeresourcethumbnails.py +++ b/csunplugged/resources/management/commands/makeresourcethumbnails.py @@ -2,6 +2,7 @@ import os import os.path +import time from urllib.parse import urlencode from tqdm import tqdm from django.conf import settings @@ -13,6 +14,9 @@ from resources.utils.resource_valid_configurations import resource_valid_configurations from resources.utils.resource_parameters import EnumResourceParameter from resources.utils.get_thumbnail import get_thumbnail_filename +from multiprocessing.dummy import Pool + +THREADS = 6 BASE_PATH_TEMPLATE = "build/img/resources/{resource}/thumbnails/{language}" @@ -34,6 +38,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): """Automatically called when makeresourcethumbnails command is given.""" resources = Resource.objects.order_by("name") + pool = Pool(THREADS) if options.get("all_languages"): languages = settings.DEFAULT_LANGUAGES @@ -43,6 +48,7 @@ def handle(self, *args, **options): with translation.override(language_code): print("Creating thumbnails for language '{}''".format(language_code)) for resource in resources: + settings = [] base_path = BASE_PATH_TEMPLATE.format( resource=resource.slug, language=language_code @@ -61,12 +67,28 @@ def handle(self, *args, **options): combinations = resource_valid_configurations(valid_options) # Create thumbnail for all possible combinations - print("Creating thumbnails for {}".format(resource.name)) - progress_bar = tqdm(combinations, ascii=True) - for combination in progress_bar: - requested_options = QueryDict(urlencode(combination, doseq=True)) - generator = get_resource_generator(resource.generator_module, requested_options) - filename = get_thumbnail_filename(resource.slug, combination) - thumbnail_file_path = os.path.join(base_path, filename) - generator.save_thumbnail(resource.name, thumbnail_file_path) + for combination in combinations: + settings.append([resource, combination, base_path]) + + print("Creating {} thumbnails with {} processes for '{}'...".format(len(settings), THREADS, resource.name)) + start = time.process_time() + try: + pool.map(self.generate_thumbnail_set, settings) + except: # sqlite3.ProgrammingError: + print("Error using parallel processing, creating {} thumbnails in series for '{}'...".format(len(settings), resource.name)) + for settings_set in settings: + self.generate_thumbnail_set(settings_set) + print("Done, time taken: {}s.".format(time.process_time() - start)) + + def generate_thumbnail_set(self, settings): + """TODO""" + resource = settings[0] + combination = settings[1] + base_path = settings[2] + print(" - Creating thumbnail for {}".format(resource.name)) + requested_options = QueryDict(urlencode(combination, doseq=True)) + generator = get_resource_generator(resource.generator_module, requested_options) + filename = get_thumbnail_filename(resource.slug, combination) + thumbnail_file_path = os.path.join(base_path, filename) + generator.save_thumbnail(resource.name, thumbnail_file_path) #breaks From feeaade85c310599cf3914e27739a4ab6f6872ab Mon Sep 17 00:00:00 2001 From: eAlasdair Date: Fri, 17 Jan 2020 12:41:59 +1300 Subject: [PATCH 4/9] Neaten parallelism code --- .../management/commands/makeresources.py | 42 +++++++++---------- .../commands/makeresourcethumbnails.py | 38 +++++++++-------- 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/csunplugged/resources/management/commands/makeresources.py b/csunplugged/resources/management/commands/makeresources.py index 39e829933..be30c6267 100644 --- a/csunplugged/resources/management/commands/makeresources.py +++ b/csunplugged/resources/management/commands/makeresources.py @@ -2,7 +2,6 @@ import os import os.path -import time from urllib.parse import urlencode from django.core.management.base import BaseCommand from django.http.request import QueryDict @@ -39,7 +38,6 @@ def add_arguments(self, parser): def handle(self, *args, **options): """Automatically called when the makeresources command is given.""" - base_path = settings.RESOURCE_GENERATION_LOCATION if options["resource_name"]: resources = [Resource.objects.get(name=options["resource_name"])] @@ -55,13 +53,16 @@ def handle(self, *args, **options): generation_languages.append(language_code) for resource in resources: - extras = [base_path, generation_languages] - self.create_pdfs_for_resource(resource, extras) + self.create_pdfs_for_resource(resource, generation_languages) - def create_pdfs_for_resource(self, resource, extras): - """TODO""" - base_path = extras[0] - generation_languages = extras[1] + def create_pdfs_for_resource(self, resource, generation_languages): + """Create all PDFs for a given resource. + + Args: + resource (Resource): The resource PDFs will be generated for + generation_languages (list): All languages to generate PDFs in + """ + base_path = settings.RESOURCE_GENERATION_LOCATION pool = Pool(THREADS) # TODO: Import repeated in next for loop, check alternatives @@ -74,30 +75,29 @@ def create_pdfs_for_resource(self, resource, extras): combinations = resource_valid_configurations(valid_options) # Create parameter sets for all possible combinations of resource - parameters = [] + parameter_sets = [] for combination in combinations: for language_code in generation_languages: - parameters.append([resource, combination, language_code, base_path]) + parameter_sets.append([resource, combination, language_code, base_path]) # Generate resources - print("Creating {} PDFs with {} processes for '{}'...".format(len(parameters), THREADS, resource.name)) - start = time.process_time() + print("Creating {} PDFs with {} processes for '{}'...".format(len(parameter_sets), THREADS, resource.name)) try: - pool.map(self.create_resource_pdf, parameters) + pool.map(self.create_resource_pdf, parameter_sets) except: # sqlite3.ProgrammingError: - print("Error using parallel processing, creating {} PDFs in series for '{}'...".format(len(parameters), resource.name)) - for parameter_set in parameters: + print("Error using parallel processing, creating {} PDFs in series for '{}'...".format(len(parameter_sets), resource.name)) + for parameter_set in parameter_sets: self.create_resource_pdf(parameter_set) - print("Done, time taken: {}s.".format(time.process_time() - start)) def create_resource_pdf(self, parameter_set): """Create a given resource PDF. Args: - resource (Resource): Resource to create. - combination (dict): Specific option attributes for this resource. - language_code (str): Code for language. - base_path (str): Base path for outputting P + parameter_set (list): A list of... + resource (Resource): Resource to create. + combination (dict): Specific option attributes for this resource. + language_code (str): Code for language. + base_path (str): Base path for outputting the resource """ resource = parameter_set[0] combination = parameter_set[1] @@ -109,7 +109,7 @@ def create_resource_pdf(self, parameter_set): combination["copies"] = settings.RESOURCE_COPY_AMOUNT requested_options = QueryDict(urlencode(combination, doseq=True)) generator = get_resource_generator(resource.generator_module, requested_options) - (pdf_file, filename) = generator.pdf(resource.name) ##Breaks here SQLite object + (pdf_file, filename) = generator.pdf(resource.name) pdf_directory = os.path.join(base_path, resource.slug, language_code) if not os.path.exists(pdf_directory): diff --git a/csunplugged/resources/management/commands/makeresourcethumbnails.py b/csunplugged/resources/management/commands/makeresourcethumbnails.py index e413069fd..6b9d97d47 100644 --- a/csunplugged/resources/management/commands/makeresourcethumbnails.py +++ b/csunplugged/resources/management/commands/makeresourcethumbnails.py @@ -2,7 +2,6 @@ import os import os.path -import time from urllib.parse import urlencode from tqdm import tqdm from django.conf import settings @@ -48,7 +47,7 @@ def handle(self, *args, **options): with translation.override(language_code): print("Creating thumbnails for language '{}''".format(language_code)) for resource in resources: - settings = [] + parameter_sets = [] base_path = BASE_PATH_TEMPLATE.format( resource=resource.slug, language=language_code @@ -69,26 +68,31 @@ def handle(self, *args, **options): # Create thumbnail for all possible combinations for combination in combinations: - settings.append([resource, combination, base_path]) + parameter_sets.append([resource, combination, base_path]) - print("Creating {} thumbnails with {} processes for '{}'...".format(len(settings), THREADS, resource.name)) - start = time.process_time() + print("Creating {} thumbnails with {} processes for '{}'...".format(len(parameter_sets), THREADS, resource.name)) try: - pool.map(self.generate_thumbnail_set, settings) + pool.map(self.generate_thumbnail, parameter_sets) except: # sqlite3.ProgrammingError: - print("Error using parallel processing, creating {} thumbnails in series for '{}'...".format(len(settings), resource.name)) - for settings_set in settings: - self.generate_thumbnail_set(settings_set) - print("Done, time taken: {}s.".format(time.process_time() - start)) - - def generate_thumbnail_set(self, settings): - """TODO""" - resource = settings[0] - combination = settings[1] - base_path = settings[2] + print("Error using parallel processing, creating {} thumbnails in series for '{}'...".format(len(parameter_sets), resource.name)) + for parameter_set in parameter_sets: + self.generate_thumbnail(parameter_set) + + def generate_thumbnail(self, parameter_set): + """Create a given resource thumbnial. + + Args: + parameter_set (list): A list of... + resource (Resource): Resource to create. + combination (dict): Specific option attributes for this resource. + base_path (str): Base path for outputting the thumbnail + """ + resource = parameter_set[0] + combination = parameter_set[1] + base_path = parameter_set[2] print(" - Creating thumbnail for {}".format(resource.name)) requested_options = QueryDict(urlencode(combination, doseq=True)) generator = get_resource_generator(resource.generator_module, requested_options) filename = get_thumbnail_filename(resource.slug, combination) thumbnail_file_path = os.path.join(base_path, filename) - generator.save_thumbnail(resource.name, thumbnail_file_path) #breaks + generator.save_thumbnail(resource.name, thumbnail_file_path) From afd279c10333b6043fc63c8590368c6b78bccbd5 Mon Sep 17 00:00:00 2001 From: eAlasdair Date: Tue, 21 Jan 2020 11:36:10 +1300 Subject: [PATCH 5/9] Switch to update_lite for efficiency --- infrastructure/dev-deploy/deploy-resources-1.sh | 2 +- infrastructure/dev-deploy/deploy-resources-10.sh | 2 +- infrastructure/dev-deploy/deploy-resources-11.sh | 2 +- infrastructure/dev-deploy/deploy-resources-2.sh | 2 +- infrastructure/dev-deploy/deploy-resources-3.sh | 2 +- infrastructure/dev-deploy/deploy-resources-4.sh | 2 +- infrastructure/dev-deploy/deploy-resources-5.sh | 2 +- infrastructure/dev-deploy/deploy-resources-6.sh | 2 +- infrastructure/dev-deploy/deploy-resources-7.sh | 2 +- infrastructure/dev-deploy/deploy-resources-8.sh | 2 +- infrastructure/dev-deploy/deploy-resources-9.sh | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/infrastructure/dev-deploy/deploy-resources-1.sh b/infrastructure/dev-deploy/deploy-resources-1.sh index f7d14dd07..d1d15682c 100644 --- a/infrastructure/dev-deploy/deploy-resources-1.sh +++ b/infrastructure/dev-deploy/deploy-resources-1.sh @@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh # Deploy generated resource files to the development static file server. ./csu start -./csu update +./csu update_lite # Generate static PDF resources for deployment. docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Arrows" diff --git a/infrastructure/dev-deploy/deploy-resources-10.sh b/infrastructure/dev-deploy/deploy-resources-10.sh index dfb4e1c55..032df47ec 100644 --- a/infrastructure/dev-deploy/deploy-resources-10.sh +++ b/infrastructure/dev-deploy/deploy-resources-10.sh @@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh # Deploy generated resource files to the development static file server. ./csu start -./csu update +./csu update_lite # Generate static PDF resources for deployment. docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Pixel Painter" "mi" diff --git a/infrastructure/dev-deploy/deploy-resources-11.sh b/infrastructure/dev-deploy/deploy-resources-11.sh index ebe9865cd..ba112cb3a 100644 --- a/infrastructure/dev-deploy/deploy-resources-11.sh +++ b/infrastructure/dev-deploy/deploy-resources-11.sh @@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh # Deploy generated resource files to the development static file server. ./csu start -./csu update +./csu update_lite # Generate static PDF resources for deployment. docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Pixel Painter" "zh-hans" diff --git a/infrastructure/dev-deploy/deploy-resources-2.sh b/infrastructure/dev-deploy/deploy-resources-2.sh index c1e6b62c0..d9c6597cb 100644 --- a/infrastructure/dev-deploy/deploy-resources-2.sh +++ b/infrastructure/dev-deploy/deploy-resources-2.sh @@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh # Deploy generated resource files to the development static file server. ./csu start -./csu update +./csu update_lite # Generate static PDF resources for deployment. docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Modulo Clock" diff --git a/infrastructure/dev-deploy/deploy-resources-3.sh b/infrastructure/dev-deploy/deploy-resources-3.sh index 627bcd822..7b057f507 100644 --- a/infrastructure/dev-deploy/deploy-resources-3.sh +++ b/infrastructure/dev-deploy/deploy-resources-3.sh @@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh # Deploy generated resource files to the development static file server. ./csu start -./csu update +./csu update_lite # Generate static PDF resources for deployment. docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Sorting Network" diff --git a/infrastructure/dev-deploy/deploy-resources-4.sh b/infrastructure/dev-deploy/deploy-resources-4.sh index 6f63be39a..afbf58a53 100644 --- a/infrastructure/dev-deploy/deploy-resources-4.sh +++ b/infrastructure/dev-deploy/deploy-resources-4.sh @@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh # Deploy generated resource files to the development static file server. ./csu start -./csu update +./csu update_lite # Generate static PDF resources for deployment. docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Number Hunt" "en" diff --git a/infrastructure/dev-deploy/deploy-resources-5.sh b/infrastructure/dev-deploy/deploy-resources-5.sh index 3b1a3984c..e6f4a50fe 100644 --- a/infrastructure/dev-deploy/deploy-resources-5.sh +++ b/infrastructure/dev-deploy/deploy-resources-5.sh @@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh # Deploy generated resource files to the development static file server. ./csu start -./csu update +./csu update_lite # Generate static PDF resources for deployment. docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Number Hunt" "mi" diff --git a/infrastructure/dev-deploy/deploy-resources-6.sh b/infrastructure/dev-deploy/deploy-resources-6.sh index 6251ed745..3e41ce7fe 100644 --- a/infrastructure/dev-deploy/deploy-resources-6.sh +++ b/infrastructure/dev-deploy/deploy-resources-6.sh @@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh # Deploy generated resource files to the development static file server. ./csu start -./csu update +./csu update_lite # Generate static PDF resources for deployment. docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Searching Cards" diff --git a/infrastructure/dev-deploy/deploy-resources-7.sh b/infrastructure/dev-deploy/deploy-resources-7.sh index 5186ab711..f5d84e26c 100644 --- a/infrastructure/dev-deploy/deploy-resources-7.sh +++ b/infrastructure/dev-deploy/deploy-resources-7.sh @@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh # Deploy generated resource files to the development static file server. ./csu start -./csu update +./csu update_lite # Generate static PDF resources for deployment. docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Pixel Painter" "en" diff --git a/infrastructure/dev-deploy/deploy-resources-8.sh b/infrastructure/dev-deploy/deploy-resources-8.sh index 65243036b..91fa92b76 100644 --- a/infrastructure/dev-deploy/deploy-resources-8.sh +++ b/infrastructure/dev-deploy/deploy-resources-8.sh @@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh # Deploy generated resource files to the development static file server. ./csu start -./csu update +./csu update_lite # Generate static PDF resources for deployment. docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Pixel Painter" "de" diff --git a/infrastructure/dev-deploy/deploy-resources-9.sh b/infrastructure/dev-deploy/deploy-resources-9.sh index 94c167a51..2e047e18c 100644 --- a/infrastructure/dev-deploy/deploy-resources-9.sh +++ b/infrastructure/dev-deploy/deploy-resources-9.sh @@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh # Deploy generated resource files to the development static file server. ./csu start -./csu update +./csu update_lite # Generate static PDF resources for deployment. docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Pixel Painter" "es" From 972a758622263942dae4113fcc6e563857b4d8b0 Mon Sep 17 00:00:00 2001 From: eAlasdair Date: Wed, 22 Jan 2020 12:35:04 +1300 Subject: [PATCH 6/9] style fixes --- .../management/commands/makeresources.py | 19 ++++++++++++------- .../commands/makeresourcethumbnails.py | 12 ++++++++---- .../resources/utils/BaseResourceGenerator.py | 6 ++---- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/csunplugged/resources/management/commands/makeresources.py b/csunplugged/resources/management/commands/makeresources.py index be30c6267..ec13ca32e 100644 --- a/csunplugged/resources/management/commands/makeresources.py +++ b/csunplugged/resources/management/commands/makeresources.py @@ -12,6 +12,7 @@ from resources.utils.resource_valid_configurations import resource_valid_configurations from resources.utils.resource_parameters import EnumResourceParameter from multiprocessing.dummy import Pool +from sqlite3 import ProgrammingError as sqlite3_ProgrammingError THREADS = 6 @@ -54,13 +55,13 @@ def handle(self, *args, **options): for resource in resources: self.create_pdfs_for_resource(resource, generation_languages) - + def create_pdfs_for_resource(self, resource, generation_languages): """Create all PDFs for a given resource. - + Args: resource (Resource): The resource PDFs will be generated for - generation_languages (list): All languages to generate PDFs in + generation_languages (list): All languages to generate PDFs in """ base_path = settings.RESOURCE_GENERATION_LOCATION pool = Pool(THREADS) @@ -71,7 +72,7 @@ def create_pdfs_for_resource(self, resource, generation_languages): for option in empty_generator.get_options().values()]): raise TypeError("Only EnumResourceParameters are supported for pre-generation") valid_options = {option.name: list(option.valid_values.keys()) - for option in empty_generator.get_options().values()} + for option in empty_generator.get_options().values()} combinations = resource_valid_configurations(valid_options) # Create parameter sets for all possible combinations of resource @@ -81,11 +82,15 @@ def create_pdfs_for_resource(self, resource, generation_languages): parameter_sets.append([resource, combination, language_code, base_path]) # Generate resources - print("Creating {} PDFs with {} processes for '{}'...".format(len(parameter_sets), THREADS, resource.name)) + print("Creating {} PDFs with {} processes for '{}'...".format( + len(parameter_sets), THREADS, resource.name) + ) try: pool.map(self.create_resource_pdf, parameter_sets) - except: # sqlite3.ProgrammingError: - print("Error using parallel processing, creating {} PDFs in series for '{}'...".format(len(parameter_sets), resource.name)) + except sqlite3_ProgrammingError: + print("Error using parallel processing, creating {} PDFs in series for '{}'...".format( + len(parameter_sets), resource.name) + ) for parameter_set in parameter_sets: self.create_resource_pdf(parameter_set) diff --git a/csunplugged/resources/management/commands/makeresourcethumbnails.py b/csunplugged/resources/management/commands/makeresourcethumbnails.py index 6b9d97d47..f68774e08 100644 --- a/csunplugged/resources/management/commands/makeresourcethumbnails.py +++ b/csunplugged/resources/management/commands/makeresourcethumbnails.py @@ -3,7 +3,6 @@ import os import os.path from urllib.parse import urlencode -from tqdm import tqdm from django.conf import settings from django.core.management.base import BaseCommand from django.http.request import QueryDict @@ -14,6 +13,7 @@ from resources.utils.resource_parameters import EnumResourceParameter from resources.utils.get_thumbnail import get_thumbnail_filename from multiprocessing.dummy import Pool +from sqlite3 import ProgrammingError as sqlite3_ProgrammingError THREADS = 6 @@ -70,11 +70,15 @@ def handle(self, *args, **options): for combination in combinations: parameter_sets.append([resource, combination, base_path]) - print("Creating {} thumbnails with {} processes for '{}'...".format(len(parameter_sets), THREADS, resource.name)) + print("Creating {} thumbnails with {} processes for '{}'...".format( + len(parameter_sets), THREADS, resource.name) + ) try: pool.map(self.generate_thumbnail, parameter_sets) - except: # sqlite3.ProgrammingError: - print("Error using parallel processing, creating {} thumbnails in series for '{}'...".format(len(parameter_sets), resource.name)) + except sqlite3_ProgrammingError: + print("Error using parallel processing, creating {} thumbnails in series for '{}'...".format( + len(parameter_sets), resource.name) + ) for parameter_set in parameter_sets: self.generate_thumbnail(parameter_set) diff --git a/csunplugged/resources/utils/BaseResourceGenerator.py b/csunplugged/resources/utils/BaseResourceGenerator.py index 3687edf72..c6786aa3a 100644 --- a/csunplugged/resources/utils/BaseResourceGenerator.py +++ b/csunplugged/resources/utils/BaseResourceGenerator.py @@ -168,8 +168,7 @@ def pdf(self, resource_name): # does not have it installed. # Also adapt logging so that weasyprint doesn't print warnings when it detects unknown css properties: import logging - logger = logging.getLogger("weasyprint").setLevel(100) - + logging.getLogger("weasyprint").setLevel(100) from weasyprint import HTML, CSS context = dict() context["resource"] = resource_name @@ -253,8 +252,7 @@ def write_thumbnail(self, thumbnail_data, resource_name, path): # does not have it installed. # Also adapt logging so that weasyprint doesn't print warnings when it detects unknown css properties: import logging - logger = logging.getLogger("weasyprint").setLevel(100) - + logging.getLogger("weasyprint").setLevel(100) from weasyprint import HTML, CSS context = dict() context["resource"] = resource_name From 2253600fbd5a9fb1934899375f9674f98b4c71ee Mon Sep 17 00:00:00 2001 From: eAlasdair Date: Wed, 22 Jan 2020 12:38:30 +1300 Subject: [PATCH 7/9] style fi --- csunplugged/resources/management/commands/makeresources.py | 1 - 1 file changed, 1 deletion(-) diff --git a/csunplugged/resources/management/commands/makeresources.py b/csunplugged/resources/management/commands/makeresources.py index ec13ca32e..4ae6c5bab 100644 --- a/csunplugged/resources/management/commands/makeresources.py +++ b/csunplugged/resources/management/commands/makeresources.py @@ -39,7 +39,6 @@ def add_arguments(self, parser): def handle(self, *args, **options): """Automatically called when the makeresources command is given.""" - if options["resource_name"]: resources = [Resource.objects.get(name=options["resource_name"])] else: From 7bfd932e090fbe328872308f1d5b011ca36770da Mon Sep 17 00:00:00 2001 From: eAlasdair Date: Thu, 8 Oct 2020 16:07:32 +1300 Subject: [PATCH 8/9] improve console log display during parallel processing --- .../resources/management/commands/makeresources.py | 9 +++++++-- .../management/commands/makeresourcethumbnails.py | 7 ++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/csunplugged/resources/management/commands/makeresources.py b/csunplugged/resources/management/commands/makeresources.py index 4ae6c5bab..582b61cff 100644 --- a/csunplugged/resources/management/commands/makeresources.py +++ b/csunplugged/resources/management/commands/makeresources.py @@ -11,10 +11,11 @@ from resources.utils.get_resource_generator import get_resource_generator from resources.utils.resource_valid_configurations import resource_valid_configurations from resources.utils.resource_parameters import EnumResourceParameter -from multiprocessing.dummy import Pool +from multiprocessing.dummy import Pool, Lock from sqlite3 import ProgrammingError as sqlite3_ProgrammingError THREADS = 6 +LOCK = Lock() class Command(BaseCommand): @@ -107,7 +108,11 @@ def create_resource_pdf(self, parameter_set): combination = parameter_set[1] language_code = parameter_set[2] base_path = parameter_set[3] - print(" - Creating PDF in '{}'".format(language_code)) + LOCK.acquire() + print(" - Creating PDF in '{}':".format(language_code)) + for key in combination.keys(): + print(" - {}: {}".format(key, combination[key])) + LOCK.release() with translation.override(language_code): if resource.copies: combination["copies"] = settings.RESOURCE_COPY_AMOUNT diff --git a/csunplugged/resources/management/commands/makeresourcethumbnails.py b/csunplugged/resources/management/commands/makeresourcethumbnails.py index f68774e08..9bb5b06d0 100644 --- a/csunplugged/resources/management/commands/makeresourcethumbnails.py +++ b/csunplugged/resources/management/commands/makeresourcethumbnails.py @@ -12,10 +12,11 @@ from resources.utils.resource_valid_configurations import resource_valid_configurations from resources.utils.resource_parameters import EnumResourceParameter from resources.utils.get_thumbnail import get_thumbnail_filename -from multiprocessing.dummy import Pool +from multiprocessing.dummy import Pool, Lock from sqlite3 import ProgrammingError as sqlite3_ProgrammingError THREADS = 6 +LOCK = Lock() BASE_PATH_TEMPLATE = "build/img/resources/{resource}/thumbnails/{language}" @@ -94,7 +95,11 @@ def generate_thumbnail(self, parameter_set): resource = parameter_set[0] combination = parameter_set[1] base_path = parameter_set[2] + LOCK.acquire() print(" - Creating thumbnail for {}".format(resource.name)) + for key in combination.keys(): + print(" - {}: {}".format(key, combination[key])) + LOCK.release() requested_options = QueryDict(urlencode(combination, doseq=True)) generator = get_resource_generator(resource.generator_module, requested_options) filename = get_thumbnail_filename(resource.slug, combination) From f276de7427daf4a7c5a3bd8a5b6736b253a67465 Mon Sep 17 00:00:00 2001 From: eAlasdair Date: Fri, 9 Oct 2020 12:57:54 +1300 Subject: [PATCH 9/9] minor consistency fix --- .../resources/management/commands/makeresourcethumbnails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csunplugged/resources/management/commands/makeresourcethumbnails.py b/csunplugged/resources/management/commands/makeresourcethumbnails.py index 9bb5b06d0..3b7486889 100644 --- a/csunplugged/resources/management/commands/makeresourcethumbnails.py +++ b/csunplugged/resources/management/commands/makeresourcethumbnails.py @@ -96,7 +96,7 @@ def generate_thumbnail(self, parameter_set): combination = parameter_set[1] base_path = parameter_set[2] LOCK.acquire() - print(" - Creating thumbnail for {}".format(resource.name)) + print(" - Creating thumbnail for {}:".format(resource.name)) for key in combination.keys(): print(" - {}: {}".format(key, combination[key])) LOCK.release()