Remove exception for test files from .flake8; write docstrings for te…

…st_pipelines.py and test_main_spider.py.

.flake8

@@ -23,7 +23,3 @@ select = C,D,E,F,W,B,B950
 # https://github.com/ambv/black/issues/21. Guido agrees here:
 # https://github.com/python/peps/commit/c59c4376ad233a62ca4b3a6060c81368bd21e85b.
 ignore = E501,W503
-# Ignore D100 and D103, which check for docstrings in modules and functions, in all test files
-per-file-ignores =
-    # Ignore D100 and D103 in all test files
-    */test_*.py: D100, D103

backend/worker/webscraper/webscraper/spiders/test_main_spider.py

@@ -1,3 +1,9 @@
+"""
+This module contains tests for the MainSpider class in the main_spider module.
+
+It includes tests for different scenarios such as when a response from a sample website is received.
+"""
+
 # Standard Python Libraries
 import json
 from tempfile import NamedTemporaryFile
@@ -30,11 +36,23 @@
 
 @pytest.fixture
 def spider():
+    """
+    Create a MainSpider instance with a temporary domains file.
+
+    This fixture creates a NamedTemporaryFile instance and uses its name as the domains_file parameter for the
+    MainSpider instance. The MainSpider instance is then returned for use in the tests.
+    """
     with NamedTemporaryFile() as f:
         return MainSpider(domains_file=f.name)
 
 
 def test_sample_website(spider):
+    """
+    Test the MainSpider class with a sample website response.
+
+    This function creates a sample Response instance with a specific body and headers. It then calls the parse_item
+    method of the MainSpider instance (provided by the spider fixture) with the sample response and checks the results.
+    """
     response = Response(
         url="https://www.cisa.gov",
         request=Request(url="https://www.cisa.gov"),

backend/worker/webscraper/webscraper/test_pipelines.py

@@ -1,3 +1,9 @@
+"""
+This module contains tests for the ExportFilePipeline class in the pipelines module.
+
+It includes tests for different scenarios such as processing an item and handling duplicate items.
+"""
+
 # Standard Python Libraries
 from unittest.mock import MagicMock
 
@@ -10,11 +16,23 @@
 
 @pytest.fixture
 def pipeline():
+    """
+    Create an ExportFilePipeline instance with a mocked print function.
+
+    This fixture creates a MagicMock instance and uses it as the print parameter for the
+    ExportFilePipeline instance. The ExportFilePipeline instance is then returned for use in the tests.
+    """
     return ExportFilePipeline(print=MagicMock())
 
 
 @pytest.fixture
 def item():
+    """
+    Create a sample item for testing.
+
+    This fixture creates a dictionary that represents a sample item with specific headers and other details.
+    The item is then returned for use in the tests.
+    """
     return {
         "status": 200,
         "url": "https://www.cisa.gov",
@@ -49,11 +67,24 @@ def item():
 
 
 def test_print_item(pipeline, item):
+    """
+    Test the process_item method of the ExportFilePipeline class with a sample item.
+
+    This function calls the process_item method of the ExportFilePipeline instance (provided by the pipeline fixture)
+    with the sample item (provided by the item fixture) and checks if the print function was called.
+    """
     pipeline.process_item(item)
     pipeline.print.assert_called_once()
 
 
 def test_discard_duplicate_items(pipeline, item):
+    """
+    Test the process_item method of the ExportFilePipeline class with duplicate items.
+
+    This function calls the process_item method of the ExportFilePipeline instance (provided by the pipeline fixture)
+    with the sample item (provided by the item fixture) twice and checks if a DropItem exception is raised the second time.
+    It also checks if the print function was called only once.
+    """
     pipeline.process_item(item)
     pipeline.print.assert_called_once()
     pipeline.print.reset_mock()