Redesign nb-tester interface

.github/workflows/notebook-test-cron.yml

@@ -30,7 +30,7 @@ jobs:
 
       - name: Execute notebooks
         timeout-minutes: 350
-        run: tox -- --write --only-submit-jobs
+        run: tox -- --write --test-strategy hardware
 
       - name: Detect changed notebooks
         id: changed-notebooks

.github/workflows/notebook-test.yml

@@ -47,7 +47,7 @@ jobs:
           """.strip().split("\n")
           import os
           github_output = os.getenv("GITHUB_OUTPUT")
-          all_files = "${{ steps.all-changed.outputs.all_changed_files }}".split("\n")
+          all_files = """${{ steps.all-changed.outputs.all_changed_files }}""".split("\n")
 
           config_changed = any(path.startswith("scripts/") for path in all_files)
           extra_deps = config_changed or any(path in EXTRA_DEPS_NOTEBOOKS for path in all_files)

scripts/ci/check-all-notebooks-are-tested.py

@@ -23,8 +23,8 @@
 config = tomllib.loads(config_path.read_text())
 
 categorized_notebooks = set()
-for group in config.values():
-    for path in group:
+for group in config["groups"].values():
+    for path in group.get("notebooks", []):
         categorized_notebooks.add(Path(path))
 
 uncategorized = [

scripts/config/notebook-testing.toml

@@ -1,68 +1,83 @@
-# For notebooks to be tested "normally" (no mocking)
-notebooks_normal_test = [
-    "docs/guides/construct-circuits.ipynb",
+default-strategy = "ci"
+
+[test-strategies]
+ci = { timeout = 300 }
+
+# For notebooks to be tested in CI "normally" (no mocking)
+[groups.normal]
+test-strategies.ci = {}
+test-strategies.hardware = {}
+notebooks = [
+    "docs/guides/build-noise-models.ipynb",
     "docs/guides/circuit-library.ipynb",
-    "docs/guides/visualize-circuits.ipynb",
     "docs/guides/classical-feedforward-and-control-flow.ipynb",
-    "docs/guides/operators-overview.ipynb",
-    "docs/guides/pulse.ipynb",
-    "docs/guides/save-circuits.ipynb",
-    "docs/guides/save-jobs.ipynb",
-    "docs/guides/dynamic-circuits-considerations.ipynb",
-    "docs/guides/get-qpu-information.ipynb",
-    "docs/guides/visualize-results.ipynb",
     "docs/guides/common-parameters.ipynb",
+    "docs/guides/construct-circuits.ipynb",
     "docs/guides/create-transpiler-plugin.ipynb",
     "docs/guides/custom-backend.ipynb",
     "docs/guides/custom-transpiler-pass.ipynb",
     "docs/guides/defaults-and-configuration-options.ipynb",
+    "docs/guides/dynamic-circuits-considerations.ipynb",
     "docs/guides/dynamical-decoupling-pass-manager.ipynb",
-    "docs/guides/represent-quantum-computers.ipynb",
-    "docs/guides/set-optimization.ipynb",
-    "docs/guides/transpile-with-pass-managers.ipynb",
-    "docs/guides/transpiler-plugins.ipynb",
-    "docs/guides/transpiler-stages.ipynb",
-    "docs/guides/build-noise-models.ipynb",
+    "docs/guides/error-mitigation-and-suppression-techniques.ipynb",
+    "docs/guides/get-qpu-information.ipynb",
     "docs/guides/local-testing-mode.ipynb",
-    "docs/guides/plot-quantum-states.ipynb",
-    "docs/guides/simulate-with-qiskit-aer.ipynb",
-    "docs/guides/simulate-stabilizer-circuits.ipynb",
     "docs/guides/operator-class.ipynb",
-    "docs/guides/error-mitigation-and-suppression-techniques.ipynb",
+    "docs/guides/operators-overview.ipynb",
+    "docs/guides/plot-quantum-states.ipynb",
+    "docs/guides/pulse.ipynb",
+    "docs/guides/qiskit-addons-aqc-get-started.ipynb",
+    "docs/guides/represent-quantum-computers.ipynb",
+    "docs/guides/save-circuits.ipynb",
+    "docs/guides/save-jobs.ipynb",
     "docs/guides/serverless-first-program.ipynb",
     "docs/guides/serverless-manage-resources.ipynb",
     "docs/guides/serverless-run-first-workload.ipynb",
+    "docs/guides/set-optimization.ipynb",
+    "docs/guides/simulate-stabilizer-circuits.ipynb",
+    "docs/guides/simulate-with-qiskit-aer.ipynb",
     "docs/guides/specify-observables-pauli.ipynb",
-    "docs/guides/qiskit-addons-aqc-get-started.ipynb",
-]
-
-# Don't test the following notebooks (this section can include glob patterns)
-notebooks_exclude = [
-    "docs/guides/qedma-qesem.ipynb",
-    "docs/guides/q-ctrl-optimization-solver.ipynb",
-    "docs/guides/q-ctrl-performance-management.ipynb",
-    "docs/guides/algorithmiq-tem.ipynb",
-    "docs/guides/functions.ipynb",
-    "docs/guides/qunasys-quri-chemistry.ipynb",
-    "docs/guides/multiverse-computing-singularity.ipynb",
-    "docs/guides/ibm-circuit-function.ipynb",
-    "docs/guides/qiskit-addons-sqd-get-started.ipynb",
-    "docs/guides/fractional-gates.ipynb",
+    "docs/guides/transpile-with-pass-managers.ipynb",
+    "docs/guides/transpiler-plugins.ipynb",
+    "docs/guides/transpiler-stages.ipynb",
+    "docs/guides/visualize-circuits.ipynb",
+    "docs/guides/visualize-results.ipynb",
 ]
 
-# The following notebooks submit jobs that can be mocked with a 5Q simulator
-notebooks_that_submit_jobs = [
-    "docs/guides/primitive-input-output.ipynb",
+# The following notebooks submit jobs that can be mocked with a simulator
+[groups.mockable]
+test-strategies.ci = { patch="qiskit-fake-provider", num_qubits=6 }
+test-strategies.hardware = {}
+notebooks = [
     "docs/guides/debug-qiskit-runtime-jobs.ipynb",
+    "docs/guides/primitive-input-output.ipynb",
 ]
 
-# The following notebooks submit jobs that are too big to mock with a 5Q simulator (or use functions that aren't supported on sims)
+# The following notebooks submit jobs that are too big to mock with a simulator (or use functions that aren't supported on sims)
 # A job is "too big" if a cell can't run in under 5 mins, or we run out of
 # memory on a reasonable device.
-notebooks_no_mock = [
+[groups.cron-job-only]
+test-strategies.hardware = {}
+notebooks = [
     "docs/guides/get-started-with-primitives.ipynb",
     "docs/guides/hello-world.ipynb",
     "docs/guides/noise-learning.ipynb",
     "docs/guides/qiskit-addons-obp-get-started.ipynb",
     "docs/guides/primitives-examples.ipynb",
 ]
+
+# Don't ever test the following notebooks
+[groups.exclude]
+notebooks = [
+    "docs/guides/algorithmiq-tem.ipynb",
+    "docs/guides/fractional-gates.ipynb",
+    "docs/guides/functions.ipynb",
+    "docs/guides/ibm-circuit-function.ipynb",
+    "docs/guides/multiverse-computing-singularity.ipynb",
+    "docs/guides/q-ctrl-optimization-solver.ipynb",
+    "docs/guides/q-ctrl-performance-management.ipynb",
+    "docs/guides/qedma-qesem.ipynb",
+    "docs/guides/qiskit-addons-sqd-get-started.ipynb",
+    "docs/guides/qunasys-quri-chemistry.ipynb",
+]
+

scripts/nb-tester/README.md

@@ -0,0 +1,168 @@
+# Qiskit documentation notebook tester
+
+A tool to execute notebooks for testing, with certain features useful for
+notebooks that run jobs on IBM Quantum backends.
+
+## Basic usage
+
+To use this tool, just run the `test-docs-notebooks` command, followed by a
+list of notebooks you'd like to execute.
+
+```
+test-docs-notebooks path/to/notebook.ipynb path/to/another.ipynb
+```
+
+* To write the executed notebooks to disk, include the `--write` argument.
+* To set a time limit for each cell, include the `--cell-timeout` argument,
+  followed by the time in seconds.
+
+## Patches
+
+A key feature of this tool is tricking notebooks into running jobs on a
+specific backend so we can test them without waiting in a queue. We do this by
+monkey-patching `QiskitRuntimeService.least_busy` to return the backend of our
+choosing. This technique requires notebooks always use `least_busy` to select a
+backend.
+
+A patch is just Python code that we run in the kernel before executing your
+notebook's code. You can provide your own patch code, or use one of our
+built-in patches.
+
+You can specify patch information as a TOML dict string. This dict _must_ include a
+`patch` key, which is either a filepath to your custom patch, or the name of
+one of our built-in patches (choose from `qiskit-fake-provider`,
+`qiskit-ibm-runtime`, or `qiskit-fake-provider`).
+
+```
+{ patch="patch-name" }
+```
+
+To make them more flexible, the patch code can include variables inside curly
+braces. If a patch includes variables, you must provide them as extra arguments
+when specifying the patch. These arguments will be injected into your patch
+code using `.format(**args)`. For example, here's a simplified version of our
+built-in `qiskit-fake-provider` patch:
+
+```python
+# Example patch file
+from qiskit.providers.fake_provider import GenericBackendV2
+from qiskit_ibm_runtime import QiskitRuntimeService
+QiskitRuntimeService.least_busy = lambda *_: GenericBackendV2(num_qubits={num_qubits})
+```
+
+Note the variable `{num_qubits}`. To use this patch, include the values of any
+variables as extra arguments in the TOML dict.
+
+```
+{ patch="qiskit-fake-provider", num_qubits=3 }
+```
+
+There are two ways to provide patches: Through the CLI or through a config file.
+You cannot use both at the same time.
+
+### Set patches through the CLI
+
+Use the `--patch` argument to provide patch information, followed by a list of
+filenames that the patch should apply to. Filenames passed before a `--patch`
+argument are not patched.
+
+```
+test-docs-notebooks [filenames] --patch [patch-information] [filenames]
+```
+
+<details><summary>Example usage</summary>
+
+Take the following command as an example.
+
+```
+test-docs-notebooks\
+  notebook.ipynb\
+  --patch\
+  '{ patch="qiskit-fake-provider", num_qubits=6 }'\
+  notebook-2.ipynb\
+  notebook-3.ipynb\
+  --patch\
+  '{ patch="qiskit-ibm-runtime", backend="test-eagle", qiskit_runtime_service_args="" }'\
+  notebook-4.ipynb
+```
+
+This will execute:
+ * `notebook.ipynb` with no patch
+ * `notebook-2.ipynb` and `notebook-3.ipynb` with `least_busy` patched to return a 6-qubit simulator
+ * `notebook-4.ipynb` with `least_busy` patched to return the `test-eagle` cloud backend
+
+</details>
+
+### Set patches through a config file
+
+You can also choose how to patch notebooks through a TOML file. This config
+file contains groups of notebook paths, and each group includes information on
+how to patch that group in different situations, known as "test strategies".
+
+```
+# Example config file
+default-strategy = "<strategy-name>"
+
+[groups.<name>]
+test-strategies.<strategy-name> = { patch="<patch-name>", <patch-args> }
+notebooks = [
+  <notebook-paths>
+]
+```
+
+<details><summary>Example usage</summary>
+
+For example, the following config file has two groups, each with one notebook,
+and two test strategies: "main" and "mock".
+
+```toml
+# config.toml
+default-strategy = "main"
+
+[test-strategies]
+mock = { timeout = 300 }
+
+[groups.group1]
+test-strategies.main = {}
+notebooks = [
+    "notebook.ipynb",
+]
+
+[groups.group2]
+test-strategies.main = {}
+test-strategies.mock = { patch="qiskit-fake-provider", num_qubits=6 }
+notebooks = [
+    "another-notebook.ipynb",
+]
+```
+
+Here's a few different commands you could run:
+
+* ```
+  test-docs-notebooks --config-path config.toml
+  ```
+
+  This will run the config file with its default strategy, which
+  is "main". This means both `notebook.ipynb` and `another-notebook.ipynb` will
+  run without patching, as their `test-strategies.main` has no `patch` arg.
+
+* ```
+  test-docs-notebooks --config-path config.toml --test-strategy mock
+  ```
+
+  This runs the same config file but with test strategy "mock". This will skip
+  `notebook.ipynb`, as its group does not have a "mock" strategy defined, and
+  will run `another-notebook.ipynb` with a 6-qubit simulator. The "mock"
+  strategy also has a timeout defined, so each cell will timeout after 300s.
+  You can override this with your own `--timeout` argument.
+
+* ```
+  test-docs-notebooks notebook.ipynb --config-path config.toml
+  ```
+
+  You can also provide filenames when using a config file. When filenames are
+  set, the script will only run notebooks passed as the filepath arg. This
+  command will run `notebook.ipynb` but skip `another-notebook.ipynb` as it
+  wasn't passed as a filename arg.
+
+</details>

scripts/nb-tester/patches/qiskit-fake-provider

@@ -0,0 +1,11 @@
+import warnings
+from qiskit.providers.fake_provider import GenericBackendV2
+from qiskit_ibm_runtime import QiskitRuntimeService
+
+warnings.filterwarnings("ignore", message="Options {{.*}} have no effect in local testing mode.")
+warnings.filterwarnings("ignore", message="Session is not supported in local testing mode or when using a simulator.")
+
+def patched_least_busy(self, *args, **kwargs):
+    return GenericBackendV2(num_qubits={num_qubits})
+
+QiskitRuntimeService.least_busy = patched_least_busy

scripts/nb-tester/patches/qiskit-ibm-runtime

@@ -0,0 +1,7 @@
+from qiskit_ibm_runtime import QiskitRuntimeService
+
+def patched_least_busy(self, *args, **kwargs):
+    service = QiskitRuntimeService({qiskit_runtime_service_args})
+    return service.backend("{backend}")
+
+QiskitRuntimeService.least_busy = patched_least_busy

scripts/nb-tester/patches/runtime-fake-provider

@@ -0,0 +1,11 @@
+import warnings
+from qiskit_ibm_runtime import QiskitRuntimeService
+
+warnings.filterwarnings("ignore", message="Options {{.*}} have no effect in local testing mode.")
+warnings.filterwarnings("ignore", message="Session is not supported in local testing mode or when using a simulator.")
+
+def patched_least_busy(self, *args, **kwargs):
+    provider = FakeProviderForBackendV2()
+    return provider.backend("{backend}")
+
+QiskitRuntimeService.least_busy = patched_least_busy

scripts/nb-tester/pyproject.toml

@@ -14,7 +14,6 @@ dependencies = [
   "nbformat~=5.10.4",
   "ipykernel~=6.29.2",
   "squeaky==0.7.0",
-  "tomli==2.2.1",
 ]
 
 [[project.authors]]

scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py

@@ -15,16 +15,15 @@
 
 import asyncio
 import sys
-from textwrap import dedent
 import platform
 from datetime import datetime
 
-from .config import get_notebook_jobs, get_args
+from .config import get_notebook_jobs, get_parser
 from .execute import execute_notebook, cancel_trailing_jobs
 
 
 async def _main() -> None:
-    args = get_args()
+    args = get_parser().parse_args()
     jobs = list(get_notebook_jobs(args))
 
     if not jobs: