Parallelise makeresource commands

csunplugged/resources/management/commands/makeresources.py

@@ -11,6 +11,11 @@
 from resources.utils.get_resource_generator import get_resource_generator
 from resources.utils.resource_valid_configurations import resource_valid_configurations
 from resources.utils.resource_parameters import EnumResourceParameter
+from multiprocessing.dummy import Pool, Lock
+from sqlite3 import ProgrammingError as sqlite3_ProgrammingError
+
+THREADS = 6
+LOCK = Lock()
 
 
 class Command(BaseCommand):
@@ -35,8 +40,6 @@ def add_arguments(self, parser):
 
     def handle(self, *args, **options):
         """Automatically called when the makeresources command is given."""
-        base_path = settings.RESOURCE_GENERATION_LOCATION
-
         if options["resource_name"]:
             resources = [Resource.objects.get(name=options["resource_name"])]
         else:
@@ -51,34 +54,65 @@ def handle(self, *args, **options):
                     generation_languages.append(language_code)
 
         for resource in resources:
-            print("Creating {}".format(resource.name))
-
-            # TODO: Import repeated in next for loop, check alternatives
-            empty_generator = get_resource_generator(resource.generator_module)
-            if not all([isinstance(option, EnumResourceParameter)
-                        for option in empty_generator.get_options().values()]):
-                raise TypeError("Only EnumResourceParameters are supported for pre-generation")
-            valid_options = {option.name: list(option.valid_values.keys())
-                             for option in empty_generator.get_options().values()}
-            combinations = resource_valid_configurations(valid_options)
+            self.create_pdfs_for_resource(resource, generation_languages)
 
-            # TODO: Create PDFs in parallel
+    def create_pdfs_for_resource(self, resource, generation_languages):
+        """Create all PDFs for a given resource.
 
-            # Create PDF for all possible combinations
-            for combination in combinations:
-                for language_code in generation_languages:
-                    self.create_resource_pdf(resource, combination, language_code, base_path)
-
-    def create_resource_pdf(self, resource, combination, language_code, base_path):
+        Args:
+            resource (Resource): The resource PDFs will be generated for
+            generation_languages (list): All languages to generate PDFs in
+        """
+        base_path = settings.RESOURCE_GENERATION_LOCATION
+        pool = Pool(THREADS)
+
+        # TODO: Import repeated in next for loop, check alternatives
+        empty_generator = get_resource_generator(resource.generator_module)
+        if not all([isinstance(option, EnumResourceParameter)
+                    for option in empty_generator.get_options().values()]):
+            raise TypeError("Only EnumResourceParameters are supported for pre-generation")
+        valid_options = {option.name: list(option.valid_values.keys())
+                         for option in empty_generator.get_options().values()}
+        combinations = resource_valid_configurations(valid_options)
+
+        # Create parameter sets for all possible combinations of resource
+        parameter_sets = []
+        for combination in combinations:
+            for language_code in generation_languages:
+                parameter_sets.append([resource, combination, language_code, base_path])
+
+        # Generate resources
+        print("Creating {} PDFs with {} processes for '{}'...".format(
+            len(parameter_sets), THREADS, resource.name)
+            )
+        try:
+            pool.map(self.create_resource_pdf, parameter_sets)
+        except sqlite3_ProgrammingError:
+            print("Error using parallel processing, creating {} PDFs in series for '{}'...".format(
+                len(parameter_sets), resource.name)
+                )
+            for parameter_set in parameter_sets:
+                self.create_resource_pdf(parameter_set)
+
+    def create_resource_pdf(self, parameter_set):
         """Create a given resource PDF.
 
         Args:
-            resource (Resource): Resource to create.
-            combination (dict): Specific option attributes for this resource.
-            language_code (str): Code for language.
-            base_path (str): Base path for outputting P
+            parameter_set (list): A list of...
+                resource (Resource): Resource to create.
+                combination (dict): Specific option attributes for this resource.
+                language_code (str): Code for language.
+                base_path (str): Base path for outputting the resource
         """
-        print("  - Creating PDF in '{}'".format(language_code))
+        resource = parameter_set[0]
+        combination = parameter_set[1]
+        language_code = parameter_set[2]
+        base_path = parameter_set[3]
+        LOCK.acquire()
+        print("  - Creating PDF in '{}':".format(language_code))
+        for key in combination.keys():
+            print("    - {}: {}".format(key, combination[key]))
+        LOCK.release()
         with translation.override(language_code):
             if resource.copies:
                 combination["copies"] = settings.RESOURCE_COPY_AMOUNT

csunplugged/resources/management/commands/makeresourcethumbnails.py

@@ -3,7 +3,6 @@
 import os
 import os.path
 from urllib.parse import urlencode
-from tqdm import tqdm
 from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.http.request import QueryDict
@@ -13,6 +12,11 @@
 from resources.utils.resource_valid_configurations import resource_valid_configurations
 from resources.utils.resource_parameters import EnumResourceParameter
 from resources.utils.get_thumbnail import get_thumbnail_filename
+from multiprocessing.dummy import Pool, Lock
+from sqlite3 import ProgrammingError as sqlite3_ProgrammingError
+
+THREADS = 6
+LOCK = Lock()
 
 BASE_PATH_TEMPLATE = "build/img/resources/{resource}/thumbnails/{language}"
 
@@ -34,6 +38,7 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
         """Automatically called when makeresourcethumbnails command is given."""
         resources = Resource.objects.order_by("name")
+        pool = Pool(THREADS)
 
         if options.get("all_languages"):
             languages = settings.DEFAULT_LANGUAGES
@@ -43,6 +48,7 @@ def handle(self, *args, **options):
             with translation.override(language_code):
                 print("Creating thumbnails for language '{}''".format(language_code))
                 for resource in resources:
+                    parameter_sets = []
                     base_path = BASE_PATH_TEMPLATE.format(
                         resource=resource.slug,
                         language=language_code
@@ -61,12 +67,41 @@ def handle(self, *args, **options):
                     combinations = resource_valid_configurations(valid_options)
 
                     # Create thumbnail for all possible combinations
-                    print("Creating thumbnails for {}".format(resource.name))
-                    progress_bar = tqdm(combinations, ascii=True)
-
-                    for combination in progress_bar:
-                        requested_options = QueryDict(urlencode(combination, doseq=True))
-                        generator = get_resource_generator(resource.generator_module, requested_options)
-                        filename = get_thumbnail_filename(resource.slug, combination)
-                        thumbnail_file_path = os.path.join(base_path, filename)
-                        generator.save_thumbnail(resource.name, thumbnail_file_path)
+
+                    for combination in combinations:
+                        parameter_sets.append([resource, combination, base_path])
+
+                    print("Creating {} thumbnails with {} processes for '{}'...".format(
+                        len(parameter_sets), THREADS, resource.name)
+                        )
+                    try:
+                        pool.map(self.generate_thumbnail, parameter_sets)
+                    except sqlite3_ProgrammingError:
+                        print("Error using parallel processing, creating {} thumbnails in series for '{}'...".format(
+                            len(parameter_sets), resource.name)
+                            )
+                        for parameter_set in parameter_sets:
+                            self.generate_thumbnail(parameter_set)
+
+    def generate_thumbnail(self, parameter_set):
+        """Create a given resource thumbnial.
+
+        Args:
+            parameter_set (list): A list of...
+                resource (Resource): Resource to create.
+                combination (dict): Specific option attributes for this resource.
+                base_path (str): Base path for outputting the thumbnail
+        """
+        resource = parameter_set[0]
+        combination = parameter_set[1]
+        base_path = parameter_set[2]
+        LOCK.acquire()
+        print("  - Creating thumbnail for {}:".format(resource.name))
+        for key in combination.keys():
+            print("    - {}: {}".format(key, combination[key]))
+        LOCK.release()
+        requested_options = QueryDict(urlencode(combination, doseq=True))
+        generator = get_resource_generator(resource.generator_module, requested_options)
+        filename = get_thumbnail_filename(resource.slug, combination)
+        thumbnail_file_path = os.path.join(base_path, filename)
+        generator.save_thumbnail(resource.name, thumbnail_file_path)

csunplugged/resources/utils/BaseResourceGenerator.py

@@ -166,6 +166,9 @@ def pdf(self, resource_name):
         """
         # Only import weasyprint when required as production environment
         # does not have it installed.
+        # Also adapt logging so that weasyprint doesn't print warnings when it detects unknown css properties:
+        import logging
+        logging.getLogger("weasyprint").setLevel(100)
         from weasyprint import HTML, CSS
         context = dict()
         context["resource"] = resource_name
@@ -247,6 +250,9 @@ def write_thumbnail(self, thumbnail_data, resource_name, path):
         """
         # Only import weasyprint when required as production environment
         # does not have it installed.
+        # Also adapt logging so that weasyprint doesn't print warnings when it detects unknown css properties:
+        import logging
+        logging.getLogger("weasyprint").setLevel(100)
         from weasyprint import HTML, CSS
         context = dict()
         context["resource"] = resource_name

infrastructure/dev-deploy/deploy-resources-1.sh

@@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh
 # Deploy generated resource files to the development static file server.
 
 ./csu start
-./csu update
+./csu update_lite
 
 # Generate static PDF resources for deployment.
 docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Arrows"

infrastructure/dev-deploy/deploy-resources-10.sh

@@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh
 # Deploy generated resource files to the development static file server.
 
 ./csu start
-./csu update
+./csu update_lite
 
 # Generate static PDF resources for deployment.
 docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Pixel Painter" "mi"

infrastructure/dev-deploy/deploy-resources-11.sh

@@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh
 # Deploy generated resource files to the development static file server.
 
 ./csu start
-./csu update
+./csu update_lite
 
 # Generate static PDF resources for deployment.
 docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Pixel Painter" "zh-hans"

infrastructure/dev-deploy/deploy-resources-2.sh

@@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh
 # Deploy generated resource files to the development static file server.
 
 ./csu start
-./csu update
+./csu update_lite
 
 # Generate static PDF resources for deployment.
 docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Modulo Clock"

infrastructure/dev-deploy/deploy-resources-3.sh

@@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh
 # Deploy generated resource files to the development static file server.
 
 ./csu start
-./csu update
+./csu update_lite
 
 # Generate static PDF resources for deployment.
 docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Sorting Network"

infrastructure/dev-deploy/deploy-resources-4.sh

@@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh
 # Deploy generated resource files to the development static file server.
 
 ./csu start
-./csu update
+./csu update_lite
 
 # Generate static PDF resources for deployment.
 docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Number Hunt" "en"

infrastructure/dev-deploy/deploy-resources-5.sh

@@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh
 # Deploy generated resource files to the development static file server.
 
 ./csu start
-./csu update
+./csu update_lite
 
 # Generate static PDF resources for deployment.
 docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Number Hunt" "mi"

infrastructure/dev-deploy/deploy-resources-6.sh

@@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh
 # Deploy generated resource files to the development static file server.
 
 ./csu start
-./csu update
+./csu update_lite
 
 # Generate static PDF resources for deployment.
 docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Searching Cards"

infrastructure/dev-deploy/deploy-resources-7.sh

@@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh
 # Deploy generated resource files to the development static file server.
 
 ./csu start
-./csu update
+./csu update_lite
 
 # Generate static PDF resources for deployment.
 docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Pixel Painter" "en"

infrastructure/dev-deploy/deploy-resources-8.sh

@@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh
 # Deploy generated resource files to the development static file server.
 
 ./csu start
-./csu update
+./csu update_lite
 
 # Generate static PDF resources for deployment.
 docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Pixel Painter" "de"

infrastructure/dev-deploy/deploy-resources-9.sh

@@ -5,7 +5,7 @@ source ./infrastructure/dev-deploy/load-dev-deploy-config-envs.sh
 # Deploy generated resource files to the development static file server.
 
 ./csu start
-./csu update
+./csu update_lite
 
 # Generate static PDF resources for deployment.
 docker-compose exec django /docker_venv/bin/python3 ./manage.py makeresources "Pixel Painter" "es"