From 8c5d9306b35e9a8b4726289d3a0ef5f3bcb42c45 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Wed, 14 Sep 2022 22:35:31 -0500 Subject: [PATCH] fix: Specify encoding as utf-8 to enforce PEP 597 (#2007) * Explicitly specify the encoding as utf-8 while opening a file to enforce PEP 597. This is future-proofing work to some degree as Python 3.15+ will make utf-8 the default. - c.f. https://peps.python.org/pep-0597/ * Add the flake8-encodings pre-commit hook to enforce PEP 597. --- .pre-commit-config.yaml | 1 + docs/development.rst | 2 +- docs/examples/notebooks/ImpactPlot.ipynb | 8 +++- docs/examples/notebooks/multiBinPois.ipynb | 2 +- .../multichannel-coupled-histo.ipynb | 4 +- docs/examples/notebooks/pullplot.ipynb | 5 ++- docs/likelihood.rst | 3 +- src/pyhf/cli/infer.py | 18 ++++++--- src/pyhf/cli/patchset.py | 16 ++++---- src/pyhf/cli/rootio.py | 8 ++-- src/pyhf/cli/spec.py | 24 ++++++------ src/pyhf/schema/loader.py | 2 +- src/pyhf/writexml.py | 2 +- tests/conftest.py | 2 +- tests/contrib/test_contrib_utils.py | 20 +++++++--- tests/contrib/test_viz.py | 28 +++++++++----- tests/test_export.py | 30 +++++++++++---- tests/test_modifiers.py | 14 +++++-- tests/test_patchset.py | 37 +++++++++++++------ tests/test_pdf.py | 25 ++++++++++--- tests/test_schema.py | 20 +++++++--- tests/test_scripts.py | 14 ++++--- tests/test_validation.py | 28 +++++++++----- tests/test_workspace.py | 3 +- validation/makedata.py | 3 +- validation/manualonoff_roofit/onoff.py | 3 +- .../makedata.py | 3 +- .../makedata.py | 3 +- .../multichannel_histfactory/makedata.py | 3 +- validation/run_toys.py | 5 ++- validation/standard_hypo_test_demo.py | 2 +- validation/xmlimport_input2/makedata.py | 3 +- 32 files changed, 230 insertions(+), 111 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2d44a93509..e706b3c0c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,6 +58,7 @@ repos: hooks: - id: flake8 args: ["--count", "--statistics"] + additional_dependencies: [flake8-encodings==0.5.0.post1] - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.971 diff --git a/docs/development.rst b/docs/development.rst index 00ecb9dd0e..d3b54dbf6b 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -52,7 +52,7 @@ available by the ``datadir`` fixture. Therefore, one can do: .. code-block:: python def test_patchset(datadir): - data_file = open(datadir.join("test.txt")) + data_file = open(datadir.join("test.txt"), encoding="utf-8") ... which will load the copy of ``text.txt`` in the temporary directory. This also diff --git a/docs/examples/notebooks/ImpactPlot.ipynb b/docs/examples/notebooks/ImpactPlot.ipynb index 0d10ddcd4c..aa045c124f 100644 --- a/docs/examples/notebooks/ImpactPlot.ipynb +++ b/docs/examples/notebooks/ImpactPlot.ipynb @@ -76,9 +76,13 @@ "outputs": [], "source": [ "def make_model(channel_list):\n", - " spec = json.load(open(\"1Lbb-probability-models/RegionA/BkgOnly.json\"))\n", + " spec = json.load(\n", + " open(\"1Lbb-probability-models/RegionA/BkgOnly.json\", encoding=\"utf-8\")\n", + " )\n", " patchset = pyhf.PatchSet(\n", - " json.load(open(\"1Lbb-probability-models/RegionA/patchset.json\"))\n", + " json.load(\n", + " open(\"1Lbb-probability-models/RegionA/patchset.json\", encoding=\"utf-8\")\n", + " )\n", " )\n", " patch = patchset[\"sbottom_750_745_60\"]\n", " spec = jsonpatch.apply_patch(spec, patch)\n", diff --git a/docs/examples/notebooks/multiBinPois.ipynb b/docs/examples/notebooks/multiBinPois.ipynb index 5b58b28f7f..72cb01f979 100644 --- a/docs/examples/notebooks/multiBinPois.ipynb +++ b/docs/examples/notebooks/multiBinPois.ipynb @@ -85,7 +85,7 @@ } ], "source": [ - "source = json.load(open(validation_datadir + '/1bin_example1.json'))\n", + "source = json.load(open(validation_datadir + \"/1bin_example1.json\", encoding=\"utf-8\"))\n", "model = uncorrelated_background(\n", " source['bindata']['sig'], source['bindata']['bkg'], source['bindata']['bkgerr']\n", ")\n", diff --git a/docs/examples/notebooks/multichannel-coupled-histo.ipynb b/docs/examples/notebooks/multichannel-coupled-histo.ipynb index 60665951cf..3c565fd93f 100644 --- a/docs/examples/notebooks/multichannel-coupled-histo.ipynb +++ b/docs/examples/notebooks/multichannel-coupled-histo.ipynb @@ -165,7 +165,9 @@ } ], "source": [ - "with open(validation_datadir + \"/2bin_2channel_coupledhisto.json\") as spec:\n", + "with open(\n", + " validation_datadir + \"/2bin_2channel_coupledhisto.json\", encoding=\"utf-8\"\n", + ") as spec:\n", " source = json.load(spec)\n", "\n", "data, pdf = prep_data(source[\"channels\"])\n", diff --git a/docs/examples/notebooks/pullplot.ipynb b/docs/examples/notebooks/pullplot.ipynb index ca76f61fd0..a259ba2cc9 100644 --- a/docs/examples/notebooks/pullplot.ipynb +++ b/docs/examples/notebooks/pullplot.ipynb @@ -72,7 +72,10 @@ "outputs": [], "source": [ "def make_model(channel_list):\n", - " spec = json.load(open(\"1Lbb-probability-models/RegionA/BkgOnly.json\"))\n", + " with open(\n", + " \"1Lbb-probability-models/RegionA/BkgOnly.json\", encoding=\"utf-8\"\n", + " ) as spec_file:\n", + " spec = json.load(spec_file)\n", " spec[\"channels\"] = [c for c in spec[\"channels\"] if c[\"name\"] in channel_list]\n", " spec[\"measurements\"][0][\"config\"][\"poi\"] = \"lumi\"\n", "\n", diff --git a/docs/likelihood.rst b/docs/likelihood.rst index b724a7da35..a3e687a6e2 100644 --- a/docs/likelihood.rst +++ b/docs/likelihood.rst @@ -28,7 +28,8 @@ check that it conforms to the provided workspace specification as follows: import json, requests, jsonschema - workspace = json.load(open("/path/to/analysis_workspace.json")) + with open("/path/to/analysis_workspace.json", encoding="utf-8") as ws_file: + workspace = json.load(ws_file) # if no exception is raised, it found and parsed the schema schema = requests.get("https://scikit-hep.org/pyhf/schemas/1.0.0/workspace.json").json() # If no exception is raised by validate(), the instance is valid. diff --git a/src/pyhf/cli/infer.py b/src/pyhf/cli/infer.py index 5081d2d156..05894be405 100644 --- a/src/pyhf/cli/infer.py +++ b/src/pyhf/cli/infer.py @@ -99,10 +99,13 @@ def fit( ) set_backend(tensorlib, new_optimizer(**optconf)) - with click.open_file(workspace, "r") as specstream: + with click.open_file(workspace, "r", encoding="utf-8") as specstream: spec = json.load(specstream) ws = Workspace(spec) - patches = [json.loads(click.open_file(pfile, "r").read()) for pfile in patch] + patches = [ + json.loads(click.open_file(pfile, "r", encoding="utf-8").read()) + for pfile in patch + ] model = ws.model( measurement_name=measurement, @@ -125,7 +128,7 @@ def fit( if output_file is None: click.echo(json.dumps(result, indent=4, sort_keys=True)) else: - with open(output_file, "w+") as out_file: + with open(output_file, "w+", encoding="utf-8") as out_file: json.dump(result, out_file, indent=4, sort_keys=True) log.debug(f"Written to {output_file:s}") @@ -190,12 +193,15 @@ def cls( "CLs_obs": 0.3599845631401915 } """ - with click.open_file(workspace, 'r') as specstream: + with click.open_file(workspace, "r", encoding="utf-8") as specstream: spec = json.load(specstream) ws = Workspace(spec) - patches = [json.loads(click.open_file(pfile, 'r').read()) for pfile in patch] + patches = [ + json.loads(click.open_file(pfile, "r", encoding="utf-8").read()) + for pfile in patch + ] model = ws.model( measurement_name=measurement, patches=patches, @@ -241,6 +247,6 @@ def cls( if output_file is None: click.echo(json.dumps(result, indent=4, sort_keys=True)) else: - with open(output_file, 'w+') as out_file: + with open(output_file, "w+", encoding="utf-8") as out_file: json.dump(result, out_file, indent=4, sort_keys=True) log.debug(f"Written to {output_file:s}") diff --git a/src/pyhf/cli/patchset.py b/src/pyhf/cli/patchset.py index 50f5855cb8..75bdb56887 100644 --- a/src/pyhf/cli/patchset.py +++ b/src/pyhf/cli/patchset.py @@ -39,7 +39,7 @@ def extract(patchset, name, output_file, with_metadata): Returns: jsonpatch (:obj:`list`): A list of jsonpatch operations to apply to a workspace. """ - with click.open_file(patchset, 'r') as fstream: + with click.open_file(patchset, "r", encoding="utf-8") as fstream: patchset_spec = json.load(fstream) patchset = PatchSet(patchset_spec) @@ -52,7 +52,7 @@ def extract(patchset, name, output_file, with_metadata): result = patch.patch if output_file: - with open(output_file, 'w+') as out_file: + with open(output_file, "w", encoding="utf-8") as out_file: json.dump(result, out_file, indent=4, sort_keys=True) log.debug(f"Written to {output_file:s}") else: @@ -79,19 +79,19 @@ def apply(background_only, patchset, name, output_file): Returns: workspace (:class:`~pyhf.workspace.Workspace`): The patched background-only workspace. """ - with click.open_file(background_only, 'r') as specstream: + with click.open_file(background_only, "r", encoding="utf-8") as specstream: spec = json.load(specstream) ws = Workspace(spec) - with click.open_file(patchset, 'r') as fstream: + with click.open_file(patchset, "r", encoding="utf-8") as fstream: patchset_spec = json.load(fstream) patchset = PatchSet(patchset_spec) patched_ws = patchset.apply(ws, name) if output_file: - with open(output_file, 'w+') as out_file: + with open(output_file, "w+", encoding="utf-8") as out_file: json.dump(patched_ws, out_file, indent=4, sort_keys=True) log.debug(f"Written to {output_file:s}") else: @@ -111,12 +111,12 @@ def verify(background_only, patchset): Returns: None """ - with click.open_file(background_only, 'r') as specstream: + with click.open_file(background_only, "r", encoding="utf-8") as specstream: spec = json.load(specstream) ws = Workspace(spec) - with click.open_file(patchset, 'r') as fstream: + with click.open_file(patchset, "r", encoding="utf-8") as fstream: patchset_spec = json.load(fstream) patchset = PatchSet(patchset_spec) @@ -134,7 +134,7 @@ def inspect(patchset): Returns: None """ - with click.open_file(patchset, 'r') as fstream: + with click.open_file(patchset, "r", encoding="utf-8") as fstream: patchset_spec = json.load(fstream) patchset = PatchSet(patchset_spec) diff --git a/src/pyhf/cli/rootio.py b/src/pyhf/cli/rootio.py index 06411773f8..d1312c2641 100644 --- a/src/pyhf/cli/rootio.py +++ b/src/pyhf/cli/rootio.py @@ -65,7 +65,7 @@ def xml2json( if output_file is None: click.echo(json.dumps(spec, indent=4, sort_keys=True)) else: - with open(output_file, 'w+') as out_file: + with open(output_file, "w+", encoding="utf-8") as out_file: json.dump(spec, out_file, indent=4, sort_keys=True) log.debug(f"Written to {output_file:s}") @@ -92,15 +92,15 @@ def json2xml(workspace, output_dir, specroot, dataroot, resultprefix, patch): from pyhf import writexml os.makedirs(output_dir, exist_ok=True) - with click.open_file(workspace, 'r') as specstream: + with click.open_file(workspace, "r", encoding="utf-8") as specstream: spec = json.load(specstream) for pfile in patch: - patch = json.loads(click.open_file(pfile, 'r').read()) + patch = json.loads(click.open_file(pfile, "r", encoding="utf-8").read()) spec = jsonpatch.JsonPatch(patch).apply(spec) os.makedirs(Path(output_dir).joinpath(specroot), exist_ok=True) os.makedirs(Path(output_dir).joinpath(dataroot), exist_ok=True) with click.open_file( - Path(output_dir).joinpath(f'{resultprefix}.xml'), 'w' + Path(output_dir).joinpath(f"{resultprefix}.xml"), "w", encoding="utf-8" ) as outstream: outstream.write( writexml.writexml( diff --git a/src/pyhf/cli/spec.py b/src/pyhf/cli/spec.py index 47beac9e73..ad335851b8 100644 --- a/src/pyhf/cli/spec.py +++ b/src/pyhf/cli/spec.py @@ -60,7 +60,7 @@ def inspect(workspace, output_file, measurement): (*) Measurement mu (none) """ - with click.open_file(workspace, 'r') as specstream: + with click.open_file(workspace, "r", encoding="utf-8") as specstream: spec = json.load(specstream) ws = Workspace(spec) @@ -158,7 +158,7 @@ def inspect(workspace, output_file, measurement): click.echo() if output_file: - with open(output_file, 'w+') as out_file: + with open(output_file, "w+", encoding="utf-8") as out_file: json.dump(result, out_file, indent=4, sort_keys=True) log.debug(f"Written to {output_file:s}") @@ -189,7 +189,7 @@ def prune( See :func:`pyhf.workspace.Workspace.prune` for more information. """ - with click.open_file(workspace, 'r') as specstream: + with click.open_file(workspace, "r", encoding="utf-8") as specstream: spec = json.load(specstream) ws = Workspace(spec) @@ -204,7 +204,7 @@ def prune( if output_file is None: click.echo(json.dumps(pruned_ws, indent=4, sort_keys=True)) else: - with open(output_file, 'w+') as out_file: + with open(output_file, "w+", encoding="utf-8") as out_file: json.dump(pruned_ws, out_file, indent=4, sort_keys=True) log.debug(f"Written to {output_file:s}") @@ -253,7 +253,7 @@ def rename(workspace, output_file, channel, sample, modifier, measurement): See :func:`pyhf.workspace.Workspace.rename` for more information. """ - with click.open_file(workspace, 'r') as specstream: + with click.open_file(workspace, "r", encoding="utf-8") as specstream: spec = json.load(specstream) ws = Workspace(spec) @@ -267,7 +267,7 @@ def rename(workspace, output_file, channel, sample, modifier, measurement): if output_file is None: click.echo(json.dumps(renamed_ws, indent=4, sort_keys=True)) else: - with open(output_file, 'w+') as out_file: + with open(output_file, "w+", encoding="utf-8") as out_file: json.dump(renamed_ws, out_file, indent=4, sort_keys=True) log.debug(f"Written to {output_file:s}") @@ -298,10 +298,10 @@ def combine(workspace_one, workspace_two, join, output_file, merge_channels): See :func:`pyhf.workspace.Workspace.combine` for more information. """ - with click.open_file(workspace_one, 'r') as specstream: + with click.open_file(workspace_one, "r", encoding="utf-8") as specstream: spec_one = json.load(specstream) - with click.open_file(workspace_two, 'r') as specstream: + with click.open_file(workspace_two, "r", encoding="utf-8") as specstream: spec_two = json.load(specstream) ws_one = Workspace(spec_one) @@ -313,7 +313,7 @@ def combine(workspace_one, workspace_two, join, output_file, merge_channels): if output_file is None: click.echo(json.dumps(combined_ws, indent=4, sort_keys=True)) else: - with open(output_file, 'w+') as out_file: + with open(output_file, "w+", encoding="utf-8") as out_file: json.dump(combined_ws, out_file, indent=4, sort_keys=True) log.debug(f"Written to {output_file:s}") @@ -347,7 +347,7 @@ def digest(workspace, algorithm, output_json): $ curl -sL https://raw.githubusercontent.com/scikit-hep/pyhf/master/docs/examples/json/2-bin_1-channel.json | pyhf digest sha256:dad8822af55205d60152cbe4303929042dbd9d4839012e055e7c6b6459d68d73 """ - with click.open_file(workspace, 'r') as specstream: + with click.open_file(workspace, "r", encoding="utf-8") as specstream: spec = json.load(specstream) workspace = Workspace(spec) @@ -393,7 +393,7 @@ def sort(workspace, output_file): """ - with click.open_file(workspace, 'r') as specstream: + with click.open_file(workspace, "r", encoding="utf-8") as specstream: spec = json.load(specstream) workspace = Workspace(spec) @@ -402,6 +402,6 @@ def sort(workspace, output_file): if output_file is None: click.echo(json.dumps(sorted_ws, indent=4, sort_keys=True)) else: - with open(output_file, 'w+') as out_file: + with open(output_file, "w+", encoding="utf-8") as out_file: json.dump(sorted_ws, out_file, indent=4, sort_keys=True) log.debug(f"Written to {output_file}") diff --git a/src/pyhf/schema/loader.py b/src/pyhf/schema/loader.py index c046bec409..920766c4dc 100644 --- a/src/pyhf/schema/loader.py +++ b/src/pyhf/schema/loader.py @@ -50,7 +50,7 @@ def load_schema(schema_id: str): raise pyhf.exceptions.SchemaNotFound( f'The schema {schema_id} was not found. Do you have the right version or the right path? {path}' ) - with path.open() as json_schema: + with path.open(encoding="utf-8") as json_schema: schema = json.load(json_schema) variables.SCHEMA_CACHE[schema['$id']] = schema return variables.SCHEMA_CACHE[schema['$id']] diff --git a/src/pyhf/writexml.py b/src/pyhf/writexml.py index 0567c94031..8d3ecd3ca3 100644 --- a/src/pyhf/writexml.py +++ b/src/pyhf/writexml.py @@ -289,7 +289,7 @@ def writexml(spec, specdir, data_rootdir, resultprefix): channelfilename = str( Path(specdir).joinpath(f'{resultprefix}_{channelspec["name"]}.xml') ) - with open(channelfilename, 'w') as channelfile: + with open(channelfilename, "w", encoding="utf-8") as channelfile: channel = build_channel(spec, channelspec, spec.get('observations')) indent(channel) channelfile.write( diff --git a/tests/conftest.py b/tests/conftest.py index 96c1d11918..673d30d3b6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ @pytest.fixture def get_json_from_tarfile(): def _get_json_from_tarfile(archive_data_path, json_name): - with tarfile.open(archive_data_path, "r:gz") as archive: + with tarfile.open(archive_data_path, "r:gz", encoding="utf-8") as archive: json_file = ( archive.extractfile(archive.getmember(json_name)).read().decode("utf8") ) diff --git a/tests/contrib/test_contrib_utils.py b/tests/contrib/test_contrib_utils.py index 075093966d..5a0b69261b 100644 --- a/tests/contrib/test_contrib_utils.py +++ b/tests/contrib/test_contrib_utils.py @@ -11,25 +11,35 @@ @pytest.fixture(scope="function") def tarfile_path(tmpdir): - with open(tmpdir.join("test_file.txt").strpath, "w") as write_file: + with open( + tmpdir.join("test_file.txt").strpath, "w", encoding="utf-8" + ) as write_file: write_file.write("test file") - with tarfile.open(tmpdir.join("test_tar.tar.gz").strpath, mode="w:gz") as archive: + with tarfile.open( + tmpdir.join("test_tar.tar.gz").strpath, mode="w:gz", encoding="utf-8" + ) as archive: archive.add(tmpdir.join("test_file.txt").strpath) return Path(tmpdir.join("test_tar.tar.gz").strpath) @pytest.fixture(scope="function") def tarfile_uncompressed_path(tmpdir): - with open(tmpdir.join("test_file.txt").strpath, "w") as write_file: + with open( + tmpdir.join("test_file.txt").strpath, "w", encoding="utf-8" + ) as write_file: write_file.write("test file") - with tarfile.open(tmpdir.join("test_tar.tar").strpath, mode="w") as archive: + with tarfile.open( + tmpdir.join("test_tar.tar").strpath, mode="w", encoding="utf-8" + ) as archive: archive.add(tmpdir.join("test_file.txt").strpath) return Path(tmpdir.join("test_tar.tar").strpath) @pytest.fixture(scope="function") def zipfile_path(tmpdir): - with open(tmpdir.join("test_file.txt").strpath, "w") as write_file: + with open( + tmpdir.join("test_file.txt").strpath, "w", encoding="utf-8" + ) as write_file: write_file.write("test file") with zipfile.ZipFile(tmpdir.join("test_zip.zip").strpath, "w") as archive: archive.write(tmpdir.join("test_file.txt").strpath) diff --git a/tests/contrib/test_viz.py b/tests/contrib/test_viz.py index 5c04ad99b6..a5ae7467ed 100644 --- a/tests/contrib/test_viz.py +++ b/tests/contrib/test_viz.py @@ -13,7 +13,7 @@ def test_brazil_band_collection(datadir): - data = json.load(datadir.joinpath("hypotest_results.json").open()) + data = json.load(datadir.joinpath("hypotest_results.json").open(encoding="utf-8")) fig = Figure() ax = fig.subplots() @@ -31,7 +31,9 @@ def test_brazil_band_collection(datadir): assert brazil_band_collection.clb is None assert brazil_band_collection.axes == ax - data = json.load(datadir.joinpath("tail_probs_hypotest_results.json").open()) + data = json.load( + datadir.joinpath("tail_probs_hypotest_results.json").open(encoding="utf-8") + ) fig = Figure() ax = fig.subplots() @@ -52,7 +54,7 @@ def test_brazil_band_collection(datadir): @pytest.mark.mpl_image_compare def test_plot_results(datadir): - data = json.load(datadir.joinpath("hypotest_results.json").open()) + data = json.load(datadir.joinpath("hypotest_results.json").open(encoding="utf-8")) fig = Figure() ax = fig.subplots() @@ -66,7 +68,7 @@ def test_plot_results(datadir): @pytest.mark.mpl_image_compare def test_plot_results_no_axis(datadir): - data = json.load(datadir.joinpath("hypotest_results.json").open()) + data = json.load(datadir.joinpath("hypotest_results.json").open(encoding="utf-8")) matplotlib.use("agg") # Use non-gui backend fig, ax = plt.subplots() @@ -78,7 +80,9 @@ def test_plot_results_no_axis(datadir): @pytest.mark.mpl_image_compare def test_plot_results_components(datadir): - data = json.load(datadir.joinpath("tail_probs_hypotest_results.json").open()) + data = json.load( + datadir.joinpath("tail_probs_hypotest_results.json").open(encoding="utf-8") + ) fig = Figure() ax = fig.subplots() @@ -90,7 +94,9 @@ def test_plot_results_components(datadir): @pytest.mark.mpl_image_compare def test_plot_results_components_no_clb(datadir): - data = json.load(datadir.joinpath("tail_probs_hypotest_results.json").open()) + data = json.load( + datadir.joinpath("tail_probs_hypotest_results.json").open(encoding="utf-8") + ) fig = Figure() ax = fig.subplots() @@ -110,7 +116,9 @@ def test_plot_results_components_no_clb(datadir): @pytest.mark.mpl_image_compare def test_plot_results_components_no_clsb(datadir): - data = json.load(datadir.joinpath("tail_probs_hypotest_results.json").open()) + data = json.load( + datadir.joinpath("tail_probs_hypotest_results.json").open(encoding="utf-8") + ) fig = Figure() ax = fig.subplots() @@ -130,7 +138,9 @@ def test_plot_results_components_no_clsb(datadir): @pytest.mark.mpl_image_compare def test_plot_results_components_no_cls(datadir): - data = json.load(datadir.joinpath("tail_probs_hypotest_results.json").open()) + data = json.load( + datadir.joinpath("tail_probs_hypotest_results.json").open(encoding="utf-8") + ) fig = Figure() ax = fig.subplots() @@ -158,7 +168,7 @@ def test_plot_results_components_data_structure(datadir): """ test results should have format of: [CLs_obs, [CLsb, CLb], [CLs_exp band]] """ - data = json.load(datadir.joinpath("hypotest_results.json").open()) + data = json.load(datadir.joinpath("hypotest_results.json").open(encoding="utf-8")) fig = Figure() ax = fig.subplots() diff --git a/tests/test_export.py b/tests/test_export.py index 4700ff7988..bba0aa224e 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -53,7 +53,10 @@ def spec_staterror(): def spec_histosys(): - source = json.load(open('validation/data/2bin_histosys_example2.json')) + with open( + "validation/data/2bin_histosys_example2.json", encoding="utf-8" + ) as spec_file: + source = json.load(spec_file) spec = { 'channels': [ { @@ -88,7 +91,10 @@ def spec_histosys(): def spec_normsys(): - source = json.load(open('validation/data/2bin_histosys_example2.json')) + with open( + "validation/data/2bin_histosys_example2.json", encoding="utf-8" + ) as spec_file: + source = json.load(spec_file) spec = { 'channels': [ { @@ -120,7 +126,10 @@ def spec_normsys(): def spec_shapesys(): - source = json.load(open('validation/data/2bin_histosys_example2.json')) + with open( + "validation/data/2bin_histosys_example2.json", encoding="utf-8" + ) as spec_file: + source = json.load(spec_file) spec = { 'channels': [ { @@ -148,7 +157,10 @@ def spec_shapesys(): def spec_shapefactor(): - source = json.load(open('validation/data/2bin_histosys_example2.json')) + with open( + "validation/data/2bin_histosys_example2.json", encoding="utf-8" + ) as spec_file: + source = json.load(spec_file) spec = { 'channels': [ { @@ -408,7 +420,9 @@ def test_export_root_histogram(mocker, tmp_path): with uproot.recreate(tmp_path.joinpath("test_export_root_histogram.root")) as file: file["hist"] = pyhf.writexml._ROOT_DATA_FILE["hist"] - with uproot.open(tmp_path.joinpath("test_export_root_histogram.root")) as file: + with uproot.open( + tmp_path.joinpath("test_export_root_histogram.root"), encoding="utf-8" + ) as file: assert file["hist"].values().tolist() == [0, 1, 2, 3, 4, 5, 6, 7, 8] assert file["hist"].axis().edges().tolist() == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] assert file["hist"].name == "hist" @@ -425,7 +439,9 @@ def test_integer_data(datadir, mocker): """ Test that a spec with only integer data will be written correctly """ - with open(datadir.joinpath("workspace_integer_data.json")) as spec_file: + with open( + datadir.joinpath("workspace_integer_data.json"), encoding="utf-8" + ) as spec_file: spec = json.load(spec_file) channel_spec = spec["channels"][0] mocker.patch("pyhf.writexml._ROOT_DATA_FILE") @@ -443,7 +459,7 @@ def test_integer_data(datadir, mocker): ids=['no_inits', 'no_bounds'], ) def test_issue1814(datadir, mocker, fname, val, low, high): - with open(datadir / fname) as spec_file: + with open(datadir / fname, encoding="utf-8") as spec_file: spec = json.load(spec_file) modifierspec = {'data': None, 'name': 'mu_sig', 'type': 'normfactor'} diff --git a/tests/test_modifiers.py b/tests/test_modifiers.py index 493326e2c8..6432e75e3b 100644 --- a/tests/test_modifiers.py +++ b/tests/test_modifiers.py @@ -184,11 +184,13 @@ def test_invalid_bin_wise_modifier(datadir, patch_file): Test that bin-wise modifiers will raise an exception if their data shape differs from their sample's. """ - spec = json.load(open(datadir.joinpath("spec.json"))) + with open(datadir.joinpath("spec.json"), encoding="utf-8") as spec_file: + spec = json.load(spec_file) assert pyhf.Model(spec) - patch = JsonPatch.from_string(open(datadir.joinpath(patch_file)).read()) + with open(datadir.joinpath(patch_file), encoding="utf-8") as spec_file: + patch = JsonPatch.from_string(spec_file.read()) bad_spec = patch.apply(spec) with pytest.raises(pyhf.exceptions.InvalidModifier): @@ -196,7 +198,9 @@ def test_invalid_bin_wise_modifier(datadir, patch_file): def test_issue1720_staterror_builder_mask(datadir): - with open(datadir.joinpath("issue1720_greedy_staterror.json")) as spec_file: + with open( + datadir.joinpath("issue1720_greedy_staterror.json"), encoding="utf-8" + ) as spec_file: spec = json.load(spec_file) spec["channels"][0]["samples"][1]["modifiers"][0]["type"] = "staterror" @@ -234,7 +238,9 @@ def test_issue1720_greedy_staterror(datadir, inits): """ Test that the staterror does not affect more samples than shapesys equivalently. """ - with open(datadir.joinpath("issue1720_greedy_staterror.json")) as spec_file: + with open( + datadir.joinpath("issue1720_greedy_staterror.json"), encoding="utf-8" + ) as spec_file: spec = json.load(spec_file) model_shapesys = pyhf.Workspace(spec).model() diff --git a/tests/test_patchset.py b/tests/test_patchset.py index b4e3d36724..64eb392350 100644 --- a/tests/test_patchset.py +++ b/tests/test_patchset.py @@ -12,7 +12,8 @@ ids=['patchset_good.json', 'patchset_good_2_patches.json'], ) def patchset(datadir, request): - spec = json.load(open(datadir.joinpath(request.param))) + with open(datadir.joinpath(request.param), encoding="utf-8") as spec_file: + spec = json.load(spec_file) return pyhf.PatchSet(spec) @@ -32,7 +33,8 @@ def patch(): ], ) def test_patchset_invalid_spec(datadir, patchset_file): - patchsetspec = json.load(open(datadir.joinpath(patchset_file))) + with open(datadir.joinpath(patchset_file), encoding="utf-8") as patch_file: + patchsetspec = json.load(patch_file) with pytest.raises(pyhf.exceptions.InvalidSpecification): pyhf.PatchSet(patchsetspec) @@ -46,7 +48,8 @@ def test_patchset_invalid_spec(datadir, patchset_file): ], ) def test_patchset_bad(datadir, patchset_file): - patchsetspec = json.load(open(datadir.joinpath(patchset_file))) + with open(datadir.joinpath(patchset_file), encoding="utf-8") as patch_file: + patchsetspec = json.load(patch_file) with pytest.raises(pyhf.exceptions.InvalidPatchSet): pyhf.PatchSet(patchsetspec) @@ -97,20 +100,31 @@ def test_patchset_repr(patchset): def test_patchset_verify(datadir): - patchset = pyhf.PatchSet(json.load(open(datadir.joinpath('example_patchset.json')))) - ws = pyhf.Workspace(json.load(open(datadir.joinpath('example_bkgonly.json')))) + with open( + datadir.joinpath("example_patchset.json"), encoding="utf-8" + ) as patch_file: + patchset = pyhf.PatchSet(json.load(patch_file)) + with open(datadir.joinpath("example_bkgonly.json"), encoding="utf-8") as ws_file: + ws = pyhf.Workspace(json.load(ws_file)) assert patchset.verify(ws) is None def test_patchset_verify_failure(datadir): - patchset = pyhf.PatchSet(json.load(open(datadir.joinpath('example_patchset.json')))) + with open( + datadir.joinpath("example_patchset.json"), encoding="utf-8" + ) as patch_file: + patchset = pyhf.PatchSet(json.load(patch_file)) with pytest.raises(pyhf.exceptions.PatchSetVerificationError): assert patchset.verify({}) def test_patchset_apply(datadir): - patchset = pyhf.PatchSet(json.load(open(datadir.joinpath('example_patchset.json')))) - ws = pyhf.Workspace(json.load(open(datadir.joinpath('example_bkgonly.json')))) + with open( + datadir.joinpath("example_patchset.json"), encoding="utf-8" + ) as patch_file: + patchset = pyhf.PatchSet(json.load(patch_file)) + with open(datadir.joinpath("example_bkgonly.json"), encoding="utf-8") as ws_file: + ws = pyhf.Workspace(json.load(ws_file)) with mock.patch('pyhf.patchset.PatchSet.verify') as m: assert m.call_count == 0 assert patchset.apply(ws, 'patch_channel1_signal_syst1') @@ -134,9 +148,10 @@ def test_patch_equality(patch): def test_patchset_get_string_values(datadir): - patchset = pyhf.PatchSet( - json.load(open(datadir.joinpath('patchset_good_stringvalues.json'))) - ) + with open( + datadir.joinpath('patchset_good_stringvalues.json'), encoding="utf-8" + ) as patch_file: + patchset = pyhf.PatchSet(json.load(patch_file)) assert patchset["Gtt_2100_5000_800"] assert patchset["Gbb_2200_5000_800"] assert patchset[[2100, 800, "Gtt"]] diff --git a/tests/test_pdf.py b/tests/test_pdf.py index 2538b708a2..948aa1c0f2 100644 --- a/tests/test_pdf.py +++ b/tests/test_pdf.py @@ -354,7 +354,10 @@ def test_pdf_integration_shapesys_zeros(backend): @pytest.mark.only_numpy def test_pdf_integration_histosys(backend): - source = json.load(open('validation/data/2bin_histosys_example2.json')) + with open( + "validation/data/2bin_histosys_example2.json", encoding="utf-8" + ) as spec_file: + source = json.load(spec_file) spec = { 'channels': [ { @@ -433,7 +436,10 @@ def test_pdf_integration_histosys(backend): def test_pdf_integration_normsys(backend): - source = json.load(open('validation/data/2bin_histosys_example2.json')) + with open( + "validation/data/2bin_histosys_example2.json", encoding="utf-8" + ) as spec_file: + source = json.load(spec_file) spec = { 'channels': [ { @@ -494,7 +500,10 @@ def test_pdf_integration_normsys(backend): @pytest.mark.only_numpy def test_pdf_integration_shapesys(backend): - source = json.load(open('validation/data/2bin_histosys_example2.json')) + with open( + "validation/data/2bin_histosys_example2.json", encoding="utf-8" + ) as spec_file: + source = json.load(spec_file) spec = { 'channels': [ { @@ -617,7 +626,10 @@ def test_invalid_modifier_name_resuse(): def test_override_paramset_defaults(): - source = json.load(open('validation/data/2bin_histosys_example2.json')) + with open( + "validation/data/2bin_histosys_example2.json", encoding="utf-8" + ) as spec_file: + source = json.load(spec_file) spec = { 'channels': [ { @@ -650,7 +662,10 @@ def test_override_paramset_defaults(): def test_override_paramsets_incorrect_num_parameters(): - source = json.load(open('validation/data/2bin_histosys_example2.json')) + with open( + "validation/data/2bin_histosys_example2.json", encoding="utf-8" + ) as spec_file: + source = json.load(spec_file) spec = { 'channels': [ { diff --git a/tests/test_schema.py b/tests/test_schema.py index 965ab6bd1f..384fcf0276 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -49,14 +49,18 @@ def test_schema_changeable(datadir, monkeypatch, self_restoring_schema_globals): new_path = datadir / 'customschema' with pytest.raises(pyhf.exceptions.SchemaNotFound): - pyhf.Workspace(json.load(open(datadir / 'customschema' / 'custom.json'))) + with open( + datadir / "customschema" / "custom.json", encoding="utf-8" + ) as spec_file: + pyhf.Workspace(json.load(spec_file)) pyhf.schema(new_path) assert old_path != pyhf.schema.path assert new_path == pyhf.schema.path assert pyhf.schema.variables.SCHEMA_CACHE is not old_cache assert len(pyhf.schema.variables.SCHEMA_CACHE) == 0 - assert pyhf.Workspace(json.load(open(new_path / 'custom.json'))) + with open(new_path / "custom.json", encoding="utf-8") as spec_file: + assert pyhf.Workspace(json.load(spec_file)) assert len(pyhf.schema.variables.SCHEMA_CACHE) == 1 @@ -73,7 +77,8 @@ def test_schema_changeable_context(datadir, monkeypatch, self_restoring_schema_g assert new_path == pyhf.schema.path assert pyhf.schema.variables.SCHEMA_CACHE is not old_cache assert len(pyhf.schema.variables.SCHEMA_CACHE) == 0 - assert pyhf.Workspace(json.load(open(new_path / 'custom.json'))) + with open(new_path / "custom.json", encoding="utf-8") as spec_file: + assert pyhf.Workspace(json.load(spec_file)) assert len(pyhf.schema.variables.SCHEMA_CACHE) == 1 assert old_path == pyhf.schema.path assert old_cache == pyhf.schema.variables.SCHEMA_CACHE @@ -91,7 +96,8 @@ def test_schema_changeable_context_error( with pytest.raises(ZeroDivisionError): with pyhf.schema(new_path): # this populates the current cache - pyhf.Workspace(json.load(open(new_path / 'custom.json'))) + with open(new_path / "custom.json", encoding="utf-8") as spec_file: + pyhf.Workspace(json.load(spec_file)) raise ZeroDivisionError() assert old_path == pyhf.schema.path assert old_cache == pyhf.schema.variables.SCHEMA_CACHE @@ -569,7 +575,8 @@ def test_jsonpatch_fail(patch): @pytest.mark.parametrize('patchset_file', ['patchset_good.json']) def test_patchset(datadir, patchset_file): - patchset = json.load(open(datadir.joinpath(patchset_file))) + with open(datadir.joinpath(patchset_file), encoding="utf-8") as patch_file: + patchset = json.load(patch_file) pyhf.schema.validate(patchset, 'patchset.json') @@ -589,7 +596,8 @@ def test_patchset(datadir, patchset_file): ], ) def test_patchset_fail(datadir, patchset_file): - patchset = json.load(open(datadir.joinpath(patchset_file))) + with open(datadir.joinpath(patchset_file), encoding="utf-8") as patch_file: + patchset = json.load(patch_file) with pytest.raises(pyhf.exceptions.InvalidSpecification): pyhf.schema.validate(patchset, 'patchset.json') diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 124dabafc6..56053dcf0a 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -16,9 +16,13 @@ @pytest.fixture(scope="function") def tarfile_path(tmpdir): - with open(tmpdir.join("test_file.txt").strpath, "w") as write_file: + with open( + tmpdir.join("test_file.txt").strpath, "w", encoding="utf-8" + ) as write_file: write_file.write("test file") - with tarfile.open(tmpdir.join("test_tar.tar.gz").strpath, mode="w:gz") as archive: + with tarfile.open( + tmpdir.join("test_tar.tar.gz").strpath, mode="w:gz", encoding="utf-8" + ) as archive: archive.add(tmpdir.join("test_file.txt").strpath) return Path(tmpdir.join("test_tar.tar.gz").strpath) @@ -702,9 +706,9 @@ def test_patchset_extract(datadir, tmpdir, script_runner, output_file, with_meta else: assert ( extracted_output - == json.load(datadir.joinpath("example_patchset.json").open())['patches'][ - 0 - ]['patch'] + == json.load( + datadir.joinpath("example_patchset.json").open(encoding="utf-8") + )["patches"][0]["patch"] ) diff --git a/tests/test_validation.py b/tests/test_validation.py index 7fb947ab1a..3c5de27318 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -10,7 +10,7 @@ @pytest.fixture(scope='module') def source_1bin_shapesys(): - with open('validation/data/1bin_example1.json') as read_json: + with open("validation/data/1bin_example1.json", encoding="utf-8") as read_json: return json.load(read_json) @@ -64,7 +64,7 @@ def expected_result_1bin_shapesys(): @pytest.fixture(scope='module') def source_1bin_shapesys_q0(): - with open('validation/data/1bin_example1_q0.json') as read_json: + with open("validation/data/1bin_example1_q0.json", encoding="utf-8") as read_json: return json.load(read_json) @@ -133,7 +133,7 @@ def expected_result_1bin_shapesys_q0_toys(): @pytest.fixture(scope='module') def source_1bin_lumi(): - with open('validation/data/1bin_lumi.json') as read_json: + with open("validation/data/1bin_lumi.json", encoding="utf-8") as read_json: return json.load(read_json) @@ -195,7 +195,7 @@ def expected_result_1bin_lumi(): @pytest.fixture(scope='module') def source_1bin_normsys(): - with open('validation/data/1bin_normsys.json') as read_json: + with open("validation/data/1bin_normsys.json", encoding="utf-8") as read_json: return json.load(read_json) @@ -249,7 +249,9 @@ def expected_result_1bin_normsys(): @pytest.fixture(scope='module') def source_2bin_histosys(): - with open('validation/data/2bin_histosys_example2.json') as read_json: + with open( + "validation/data/2bin_histosys_example2.json", encoding="utf-8" + ) as read_json: return json.load(read_json) @@ -306,7 +308,9 @@ def expected_result_2bin_histosys(): @pytest.fixture(scope='module') def source_2bin_2channel(): - with open('validation/data/2bin_2channel_example1.json') as read_json: + with open( + "validation/data/2bin_2channel_example1.json", encoding="utf-8" + ) as read_json: return json.load(read_json) @@ -380,7 +384,9 @@ def expected_result_2bin_2channel(): @pytest.fixture(scope='module') def source_2bin_2channel_couplednorm(): - with open('validation/data/2bin_2channel_couplednorm.json') as read_json: + with open( + "validation/data/2bin_2channel_couplednorm.json", encoding="utf-8" + ) as read_json: return json.load(read_json) @@ -465,7 +471,9 @@ def expected_result_2bin_2channel_couplednorm(): @pytest.fixture(scope='module') def source_2bin_2channel_coupledhistosys(): - with open('validation/data/2bin_2channel_coupledhisto.json') as read_json: + with open( + "validation/data/2bin_2channel_coupledhisto.json", encoding="utf-8" + ) as read_json: return json.load(read_json) @@ -567,7 +575,9 @@ def expected_result_2bin_2channel_coupledhistosys(): @pytest.fixture(scope='module') def source_2bin_2channel_coupledshapefactor(): - with open('validation/data/2bin_2channel_coupledshapefactor.json') as read_json: + with open( + "validation/data/2bin_2channel_coupledshapefactor.json", encoding="utf-8" + ) as read_json: return json.load(read_json) diff --git a/tests/test_workspace.py b/tests/test_workspace.py index aaa7a3b301..ccc54be422 100644 --- a/tests/test_workspace.py +++ b/tests/test_workspace.py @@ -889,7 +889,8 @@ def test_workspace_poiless(datadir): """ Test that a workspace with a measurement with empty POI string is treated as POI-less """ - spec = json.load(open(datadir.joinpath("poiless.json"))) + with open(datadir.joinpath("poiless.json"), encoding="utf-8") as spec_file: + spec = json.load(spec_file) ws = pyhf.Workspace(spec) model = ws.model() diff --git a/validation/makedata.py b/validation/makedata.py index 3e6e95ff06..5ae5424459 100644 --- a/validation/makedata.py +++ b/validation/makedata.py @@ -4,7 +4,8 @@ import ROOT if __name__ == "__main__": - source_data = json.load(open(sys.argv[1])) + with open(sys.argv[1], encoding="utf-8") as source_file: + source_data = json.load(source_file) root_file = sys.argv[2] binning = source_data["binning"] diff --git a/validation/manualonoff_roofit/onoff.py b/validation/manualonoff_roofit/onoff.py index 4708d1abad..ff31acacbe 100644 --- a/validation/manualonoff_roofit/onoff.py +++ b/validation/manualonoff_roofit/onoff.py @@ -1,7 +1,8 @@ import json import ROOT -d = json.load(open('data/source.json')) +with open("data/source.json", encoding="utf-8") as source_file: + d = json.load(source_file) nobs = d['bindata']['data'][0] b = d['bindata']['bkg'][0] deltab = d['bindata']['bkgerr'][0] diff --git a/validation/multichan_coupledhistosys_histfactory/makedata.py b/validation/multichan_coupledhistosys_histfactory/makedata.py index 50c7bb29f6..b09298329f 100644 --- a/validation/multichan_coupledhistosys_histfactory/makedata.py +++ b/validation/multichan_coupledhistosys_histfactory/makedata.py @@ -3,7 +3,8 @@ import json import sys -source_data = json.load(open(sys.argv[1])) +with open(sys.argv[1], encoding="utf-8") as source_file: + source_data = json.load(source_file) root_file = sys.argv[2] f = ROOT.TFile(root_file, 'RECREATE') diff --git a/validation/multichan_coupledoverall_histfactory/makedata.py b/validation/multichan_coupledoverall_histfactory/makedata.py index 0ae185fea2..0b33ff4f55 100644 --- a/validation/multichan_coupledoverall_histfactory/makedata.py +++ b/validation/multichan_coupledoverall_histfactory/makedata.py @@ -3,7 +3,8 @@ import json import sys -source_data = json.load(open(sys.argv[1])) +with open(sys.argv[1], encoding="utf-8") as source_file: + source_data = json.load(source_file) root_file = sys.argv[2] f = ROOT.TFile(root_file, 'RECREATE') diff --git a/validation/multichannel_histfactory/makedata.py b/validation/multichannel_histfactory/makedata.py index ac50787d63..cce668912d 100644 --- a/validation/multichannel_histfactory/makedata.py +++ b/validation/multichannel_histfactory/makedata.py @@ -3,7 +3,8 @@ import json import sys -source_data = json.load(open(sys.argv[1])) +with open(sys.argv[1], encoding="utf-8") as source_file: + source_data = json.load(source_file) root_file = sys.argv[2] f = ROOT.TFile(root_file, 'RECREATE') diff --git a/validation/run_toys.py b/validation/run_toys.py index ff541d4b2e..efa88bd4a6 100644 --- a/validation/run_toys.py +++ b/validation/run_toys.py @@ -53,7 +53,8 @@ def run_toys_ROOT(infile, ntoys): for idx in range(n_points) ] - json.dump(data, open("scan.json", "w")) + with open("scan.json", "w", encoding="utf-8") as write_file: + json.dump(data, write_file) canvas = ROOT.TCanvas() canvas.SetLogy(False) @@ -69,7 +70,7 @@ def run_toys_ROOT(infile, ntoys): def run_toys_pyhf(ntoys=2_000, seed=0): np.random.seed(seed) # with open("validation/xmlimport_input_bkg.json") as ws_json: - with open("debug/issue_workpace/issue_ws.json") as ws_json: + with open("debug/issue_workpace/issue_ws.json", encoding="utf-8") as ws_json: workspace = pyhf.Workspace(json.load(ws_json)) model = workspace.model() diff --git a/validation/standard_hypo_test_demo.py b/validation/standard_hypo_test_demo.py index c67f61b361..501004a27c 100644 --- a/validation/standard_hypo_test_demo.py +++ b/validation/standard_hypo_test_demo.py @@ -75,7 +75,7 @@ def standard_hypo_test_demo( def pyhf_version(ntoys=5000, seed=0): np.random.seed(seed) - with open("validation/xmlimport_input_bkg.json") as ws_json: + with open("validation/xmlimport_input_bkg.json", encoding="utf-8") as ws_json: workspace = pyhf.Workspace(json.load(ws_json)) model = workspace.model() diff --git a/validation/xmlimport_input2/makedata.py b/validation/xmlimport_input2/makedata.py index 50c7bb29f6..b09298329f 100644 --- a/validation/xmlimport_input2/makedata.py +++ b/validation/xmlimport_input2/makedata.py @@ -3,7 +3,8 @@ import json import sys -source_data = json.load(open(sys.argv[1])) +with open(sys.argv[1], encoding="utf-8") as source_file: + source_data = json.load(source_file) root_file = sys.argv[2] f = ROOT.TFile(root_file, 'RECREATE')