Replace all Union[x, None] with Optional[x]

docs/_build/html/versioning.html

@@ -323,7 +323,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>
 <dd class="field-odd"><ul class="simple">
-<li><p><strong>modif_date</strong> (<em>Union</em><em>[</em><em>str</em><em>, </em><em>None</em><em>]</em><em>, </em><em>optional</em>) – Last modification date of the
+<li><p><strong>modif_date</strong> (<em>Optional</em><em>[</em><em>str</em><em>]</em><em>, </em><em>optional</em>) – Last modification date of the
 corresponding elements. Defaults to None.</p></li>
 <li><p><strong>include_counts</strong> (<em>bool</em><em>, </em><em>optional</em>) – Whether to include the current counts with
 key “stats”. Defaults to False.</p></li>
@@ -430,7 +430,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>
 <dd class="field-odd"><ul class="simple">
-<li><p><strong>modif_date</strong> (<em>Union</em><em>[</em><em>str</em><em>, </em><em>None</em><em>]</em><em>, </em><em>optional</em>) – Last modification date of the
+<li><p><strong>modif_date</strong> (<em>Optional</em><em>[</em><em>str</em><em>]</em><em>, </em><em>optional</em>) – Last modification date of the
 corresponding elements. Defaults to None.</p></li>
 <li><p><strong>include_counts</strong> (<em>bool</em><em>, </em><em>optional</em>) – Whether to include the current newspaper
 counts with key “nps_stats”. Defaults to True.</p></li>
@@ -608,7 +608,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <li><p><strong>export_to_git_and_s3</strong> (<em>bool</em><em>, </em><em>optional</em>) – Whether to export the final
 <cite>manifest_data</cite> as JSON to S3 and GitHub. Defaults to True. If False,
 <cite>validate_and_export_manifest</cite> can be called separately to do it.</p></li>
-<li><p><strong>commit_msg</strong> (<em>Union</em><em>[</em><em>str</em><em>, </em><em>None</em><em>]</em><em>, </em><em>optional</em>) – Commit message to use instead of
+<li><p><strong>commit_msg</strong> (<em>Optional</em><em>[</em><em>str</em><em>]</em><em>, </em><em>optional</em>) – Commit message to use instead of
 the default from <cite>validate_and_export_manifest</cite>. Defaults to None.</p></li>
 </ul>
 </dd>
@@ -945,7 +945,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <dd class="field-odd"><ul class="simple">
 <li><p><strong>push_to_git</strong> (<em>bool</em><em>, </em><em>optional</em>) – Whether to also push the generated manifest to
 GitHub (impresso/impresso-data-release). Defaults to False.</p></li>
-<li><p><strong>commit_msg</strong> (<em>Union</em><em>[</em><em>str</em><em>, </em><em>None</em><em>]</em><em>, </em><em>optional</em>) – Commit message to override the
+<li><p><strong>commit_msg</strong> (<em>Optional</em><em>[</em><em>str</em><em>]</em><em>, </em><em>optional</em>) – Commit message to override the
 default message. Defaults to None.</p></li>
 </ul>
 </dd>
@@ -1331,7 +1331,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <dd class="field-odd"><ul class="simple">
 <li><p><strong>bucket_name</strong> (<em>str</em>) – Name of the bucket in which to look.</p></li>
 <li><p><strong>data_stage</strong> (<em>str</em>) – Data stage corresponding to the manifest to fetch.</p></li>
-<li><p><strong>partition</strong> (<em>Union</em><em>[</em><em>str</em><em>, </em><em>None</em><em>]</em><em>, </em><em>optional</em>) – Partition within the bucket to look
+<li><p><strong>partition</strong> (<em>Optional</em><em>[</em><em>str</em><em>]</em><em>, </em><em>optional</em>) – Partition within the bucket to look
 into. Defaults to None.</p></li>
 </ul>
 </dd>
@@ -1343,7 +1343,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 </p>
 </dd>
 <dt class="field-odd">Return type<span class="colon">:</span></dt>
-<dd class="field-odd"><p>Union[str, None]</p>
+<dd class="field-odd"><p>Optional[str]</p>
 </dd>
 </dl>
 </dd></dl>
@@ -1437,7 +1437,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <dd class="field-odd"><ul class="simple">
 <li><p><strong>full_git_filepath</strong> (<em>str</em>) – Path to the file added to the git repository.</p></li>
 <li><p><strong>git_repo</strong> (<em>git.Repo</em>) – git.Repo object of the repository to commit and push to.</p></li>
-<li><p><strong>commit_msg</strong> (<em>Union</em><em>[</em><em>str</em><em>, </em><em>None</em><em>]</em><em>, </em><em>optional</em>) – Message to use when commiting. If not
+<li><p><strong>commit_msg</strong> (<em>Optional</em><em>[</em><em>str</em><em>]</em><em>, </em><em>optional</em>) – Message to use when commiting. If not
 defined, a basic message on the added manifest will be used. Defaults to None.</p></li>
 </ul>
 </dd>
@@ -1486,9 +1486,9 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <li><p><strong>add</strong> (<em>bool</em><em>, </em><em>optional</em>) – Whether new data was added. Defaults to True.</p></li>
 <li><p><strong>full_title</strong> (<em>bool</em><em>, </em><em>optional</em>) – Whether all the title’s years were modified.
 Defaults to True.</p></li>
-<li><p><strong>years</strong> (<em>Union</em><em>[</em><em>list</em><em>[</em><em>str</em><em>]</em><em>, </em><em>None</em><em>]</em><em>, </em><em>optional</em>) – When <cite>full_title</cite>, the specific years
+<li><p><strong>years</strong> (<em>Optional</em><em>[</em><em>list</em><em>[</em><em>str</em><em>]</em><em>]</em><em>, </em><em>optional</em>) – When <cite>full_title</cite>, the specific years
 which were modified/updated. Defaults to None.</p></li>
-<li><p><strong>fields</strong> (<em>Union</em><em>[</em><em>list</em><em>[</em><em>str</em><em>]</em><em>, </em><em>None</em><em>]</em><em>, </em><em>optional</em>) – List of specific fields that were
+<li><p><strong>fields</strong> (<em>Optional</em><em>[</em><em>list</em><em>[</em><em>str</em><em>]</em><em>]</em><em>, </em><em>optional</em>) – List of specific fields that were
 modified/updated. Defaults to None.</p></li>
 </ul>
 </dd>
@@ -1578,15 +1578,15 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 
 <dl class="py function">
 <dt class="sig sig-object py" id="impresso_commons.versioning.helpers.read_manifest_from_s3">
-<span class="sig-prename descclassname"><span class="pre">impresso_commons.versioning.helpers.</span></span><span class="sig-name descname"><span class="pre">read_manifest_from_s3</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">bucket_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">data_stage</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#impresso_commons.versioning.helpers.DataStage" title="impresso_commons.versioning.helpers.DataStage"><span class="pre">DataStage</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">partition</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">Any</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">None</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">None</span><span class="p"><span class="pre">]</span></span></span></span><a class="headerlink" href="#impresso_commons.versioning.helpers.read_manifest_from_s3" title="Link to this definition"></a></dt>
+<span class="sig-prename descclassname"><span class="pre">impresso_commons.versioning.helpers.</span></span><span class="sig-name descname"><span class="pre">read_manifest_from_s3</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">bucket_name</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">data_stage</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><a class="reference internal" href="#impresso_commons.versioning.helpers.DataStage" title="impresso_commons.versioning.helpers.DataStage"><span class="pre">DataStage</span></a><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">partition</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span> <span class="sig-return"><span class="sig-return-icon">&#x2192;</span> <span class="sig-return-typehint"><span class="pre">tuple</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">dict</span><span class="p"><span class="pre">[</span></span><span class="pre">str</span><span class="p"><span class="pre">,</span></span><span class="w"> </span><span class="pre">Any</span><span class="p"><span class="pre">]</span></span><span class="p"><span class="pre">]</span></span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span></span><a class="headerlink" href="#impresso_commons.versioning.helpers.read_manifest_from_s3" title="Link to this definition"></a></dt>
 <dd><p>Read and load manifest given an S3 bucket.</p>
 <dl class="field-list simple">
 <dt class="field-odd">Parameters<span class="colon">:</span></dt>
 <dd class="field-odd"><ul class="simple">
 <li><p><strong>bucket_name</strong> (<em>str</em>) – NAme of the s3 bucket to look into</p></li>
 <li><p><strong>data_stage</strong> (<em>Union</em><em>[</em><a class="reference internal" href="#impresso_commons.versioning.helpers.DataStage" title="impresso_commons.versioning.helpers.DataStage"><em>DataStage</em></a><em>, </em><em>str</em><em>]</em>) – Data stage corresponding to the
 manifest to fetch.</p></li>
-<li><p><strong>partition</strong> (<em>Union</em><em>[</em><em>str</em><em>, </em><em>None</em><em>]</em><em>, </em><em>optional</em>) – Partition within the bucket to look
+<li><p><strong>partition</strong> (<em>Optional</em><em>[</em><em>str</em><em>]</em><em>, </em><em>optional</em>) – Partition within the bucket to look
 into. Defaults to None.</p></li>
 </ul>
 </dd>
@@ -1598,7 +1598,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 </p>
 </dd>
 <dt class="field-odd">Return type<span class="colon">:</span></dt>
-<dd class="field-odd"><p>Union[tuple[str, dict[str, Any]], tuple[None, None]]</p>
+<dd class="field-odd"><p>tuple[str, dict[str, Any]] | tuple[None, None]</p>
 </dd>
 </dl>
 </dd></dl>
@@ -1615,7 +1615,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <dd class="field-even"><p>Contents of manifest if found on S3, None otherwise.</p>
 </dd>
 <dt class="field-odd">Return type<span class="colon">:</span></dt>
-<dd class="field-odd"><p>Union[dict[str, Any], None]</p>
+<dd class="field-odd"><p>Optional[dict[str, Any]]</p>
 </dd>
 </dl>
 </dd></dl>
@@ -1659,7 +1659,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <dd class="field-odd"><p>The provided value, in lower case, or None if not valid.</p>
 </dd>
 <dt class="field-even">Return type<span class="colon">:</span></dt>
-<dd class="field-even"><p>Union[str, None]</p>
+<dd class="field-even"><p>Optional[str]</p>
 </dd>
 </dl>
 </dd></dl>
@@ -1706,7 +1706,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <dd class="field-even"><p>The provided version if it’s valid, None otherwise.</p>
 </dd>
 <dt class="field-odd">Return type<span class="colon">:</span></dt>
-<dd class="field-odd"><p>Union[str, None]</p>
+<dd class="field-odd"><p>Optional[str]</p>
 </dd>
 </dl>
 </dd></dl>
@@ -1743,7 +1743,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <li><p><strong>git_repo</strong> (<em>git.Repo</em>) – Object representing the git repository to push to.</p></li>
 <li><p><strong>path_in_repo</strong> (<em>str</em>) – Relative path where to write the file.</p></li>
 <li><p><strong>filename</strong> (<em>str</em>) – Desired name for the file, including extension.</p></li>
-<li><p><strong>commit_msg</strong> (<em>Union</em><em>[</em><em>str</em><em>, </em><em>None</em><em>]</em><em>, </em><em>optional</em>) – Commit message. If not defined, a
+<li><p><strong>commit_msg</strong> (<em>Optional</em><em>[</em><em>str</em><em>]</em><em>, </em><em>optional</em>) – Commit message. If not defined, a
 basic message on the added manifest will be used.Defaults to None.</p></li>
 </ul>
 </dd>
@@ -1773,7 +1773,7 @@ <h1>Data Versioning<a class="headerlink" href="#data-versioning" title="Link to
 <dd class="field-even"><p>Full path of writen file, or None if an IOError occurred.</p>
 </dd>
 <dt class="field-odd">Return type<span class="colon">:</span></dt>
-<dd class="field-odd"><p>Union[str, None]</p>
+<dd class="field-odd"><p>Optional[str]</p>
 </dd>
 </dl>
 </dd></dl>

impresso_commons/data/manifest_config/ner_mft_config.json

@@ -2,9 +2,9 @@
     "data_stage": "entities",
     "output_bucket": "processed-data/ne_entities/ne-entities_v1-0-0",
     "input_bucket": "rebuilt-staging",
-    "git_repository": "/Users/piconti/impresso/impresso-pycommons",
+    "git_repository": "/local/path/to/NER-EL/repo",
     "newspapers": [],
-    "temp_directory": "/Users/piconti/impresso/git_temp_folder",
+    "temp_directory": "/local/path/t/git_temp_folder",
     "previous_mft_s3_path": "",
     "is_staging": true,
     "is_patch": false,

impresso_commons/text/rebuilder.py

@@ -31,7 +31,7 @@
 import os
 import shutil
 import signal
-from typing import Any, Union
+from typing import Any, Optional
 import git
 
 import dask.bag as db
@@ -72,7 +72,7 @@
 
 
 def rebuild_text(
-    page: list[dict], language: Union[str, None], string: Union[str, None] = None
+    page: list[dict], language: Optional[str], string: Optional[str] = None
 ) -> tuple[str, dict[list], dict[list]]:
     """Rebuild the text of an article for Solr ingestion.
 
@@ -162,7 +162,7 @@ def rebuild_text(
 
 
 def rebuild_text_passim(
-    page: list[dict], language: Union[str, None], string: Union[str, None] = None
+    page: list[dict], language: Optional[str], string: Optional[str] = None
 ) -> tuple[str, list[dict]]:
     """The text rebuilding function from pages for passim.
 

impresso_commons/versioning/compute_manifest.py

@@ -16,13 +16,12 @@
 import os
 import traceback
 import logging
-from typing import Any, Union
+from typing import Any, Optional
 import git
 from docopt import docopt
 
 import dask.bag as db
 from dask.distributed import Client
-from impresso_commons.path.path_s3 import list_newspapers
 from impresso_commons.utils.s3 import fixed_s3fs_glob, IMPRESSO_STORAGEOPT
 from impresso_commons.utils.utils import init_logger
 from impresso_commons.versioning.helpers import (
@@ -82,7 +81,7 @@ def extract_np_key(s3_key: str, bucket: str) -> str:
         return key_no_bucket.split("-")[0]
 
 
-def get_files_to_consider(config: dict[str, Any]) -> Union[dict[str, list[str]], None]:
+def get_files_to_consider(config: dict[str, Any]) -> Optional[dict[str, list[str]]]:
     """Get the list of S3 files to consider based on the provided configuration.
 
     Args:
@@ -129,8 +128,8 @@ def get_files_to_consider(config: dict[str, Any]) -> Union[dict[str, list[str]],
 
 
 def compute_stats_for_stage(
-    files_bag: db.core.Bag, stage: DataStage, client: Union[Client, None] = None
-) -> Union[list[dict], None]:
+    files_bag: db.core.Bag, stage: DataStage, client: Optional[Client] = None
+) -> Optional[list[dict]]:
     """Compute statistics for a specific data stage.
 
     Args:
@@ -189,7 +188,7 @@ def validate_config(config: dict[str, Any]) -> dict[str, Any]:
 
 
 def create_manifest(
-    config_dict: dict[str, Any], client: Union[Client, None] = None
+    config_dict: dict[str, Any], client: Optional[Client] = None
 ) -> None:
     """Given its configuration, generate the manifest for a given s3 bucket partition.