From 5f9763d5a0f1af48a2ef885f3d3901921c6bc6ec Mon Sep 17 00:00:00 2001
From: Frank Harkins <frankharkins@hotmail.co.uk>
Date: Tue, 10 Dec 2024 10:58:39 +0000
Subject: [PATCH] Add `extended` strategy and reclassify notebooks

---
 scripts/config/notebook-testing.toml | 72 ++++++++++++++++++++--------
 1 file changed, 53 insertions(+), 19 deletions(-)

diff --git a/scripts/config/notebook-testing.toml b/scripts/config/notebook-testing.toml
index 3a4343fbbb4..e0fc0b69ca0 100644
--- a/scripts/config/notebook-testing.toml
+++ b/scripts/config/notebook-testing.toml
@@ -1,11 +1,13 @@
 default-strategy = "ci"
 
 [test-strategies]
-ci = { timeout = 300 }
+ci = { timeout = 300 }         # For regular CI
+extended = { timeout = 1000 }  # Extended checks using test-eagle
 
 # For notebooks to be tested in CI "normally" (no mocking)
 [groups.normal]
 test-strategies.ci = {}
+test-strategies.extended = {}
 test-strategies.hardware = { patch="qiskit-ibm-runtime-open" }
 notebooks = [
     "docs/guides/build-noise-models.ipynb",
@@ -20,6 +22,7 @@ notebooks = [
     "docs/guides/dynamic-circuits-considerations.ipynb",
     "docs/guides/dynamical-decoupling-pass-manager.ipynb",
     "docs/guides/error-mitigation-and-suppression-techniques.ipynb",
+    "docs/guides/fractional-gates.ipynb",
     "docs/guides/get-qpu-information.ipynb",
     "docs/guides/local-testing-mode.ipynb",
     "docs/guides/operator-class.ipynb",
@@ -45,9 +48,10 @@ notebooks = [
     "docs/guides/visualize-results.ipynb",
 ]
 
-# The following notebooks submit jobs that can be mocked with a simulator
-[groups.mockable]
+# Mock the following notebooks using a 6-qubit local simulator
+[groups.local-sim]
 test-strategies.ci = { patch="qiskit-fake-provider", num_qubits=6 }
+test-strategies.extended = { patch="qiskit-fake-provider", num_qubits=6 }
 test-strategies.hardware = { patch="qiskit-ibm-runtime-open" }
 notebooks = [
     "docs/guides/debug-qiskit-runtime-jobs.ipynb",
@@ -55,31 +59,61 @@ notebooks = [
     "docs/guides/primitive-input-output.ipynb",
 ]
 
-# The following notebooks submit jobs that are too big to mock with a simulator (or use functions that aren't supported on sims)
-# A job is "too big" if a cell can't run in under 5 mins, or we run out of
-# memory on a reasonable device.
-[groups.cron-job-only]
+# Mock the following notebooks in our extended checks using IBM Quantum's
+# test-eagle device, which returns nonsense results. Use this to test notebooks
+# that submit large jobs to test notebooks that use functions, which accept
+# only a backend name (string) rather than a `Backend` object.
+[groups.test-eagle]
+test-strategies.extended = { patch="qiskit-ibm-runtime", backend="test_eagle_us-east", qiskit_runtime_service_args="" }
 test-strategies.hardware = { patch="qiskit-ibm-runtime-open" }
 notebooks = [
+    "docs/guides/primitives-examples.ipynb",
     "docs/guides/get-started-with-primitives.ipynb",
+    "docs/guides/algorithmiq-tem.ipynb",
+    "docs/guides/functions.ipynb",
+    "docs/guides/ibm-circuit-function.ipynb",
+    "docs/guides/qiskit-addons-sqd-get-started.ipynb",
+    "docs/guides/qiskit-addons-obp-get-started.ipynb",
+]
+
+# Only run the following notebooks in our fortnightly cron job on real hardware.
+[groups.cron-job-only]
+test-strategies.hardware = { patch="qiskit-ibm-runtime-open" }
+notebooks = [
     "docs/guides/hello-world.ipynb",
+
+    # The following notebooks don't seem to work with `test-eagle`, but I'm not
+    # sure why. I've added them here to run on the next cron job so we can
+    # confirm on actual hardware.
+
+    #   This timed out with no jobs submitted. Adding to the cron to see if it
+    #   needs longer.
+    "docs/guides/qedma-qesem.ipynb",
+
+    #   This timed out for me, even though I saw the job finish. Maybe it needs
+    #   longer? Running in cron job to remove time limit.
     "docs/guides/noise-learning.ipynb",
-    "docs/guides/qiskit-addons-obp-get-started.ipynb",
-    "docs/guides/primitives-examples.ipynb",
+
+    #   Got: `QiskitServerlessException: "Q-CTRL: Failed to execute Qiskit
+    #   Function. could not broadcast input array from shape (3996,5) into
+    #   shape (4000,5)"`
+    #   Timed out when I re-ran it.
+    "docs/guides/q-ctrl-performance-management.ipynb",
+
+    #   Got `KeyError: 'qsci_energy'`; might be due to nonsense results from
+    #   test-eagle? Running in cron job to test on a real backend.
+    "docs/guides/qunasys-quri-chemistry.ipynb",
+
+    #   Got: `QiskitServerlessException: "Q-CTRL: Failed to execute Qiskit
+    #   Function. Failed to compile the circuit.
+    #   compiler_exceptions=[TranspilerError('More virtual qubits
+    #   exist than physical.')]"`
+    "docs/guides/q-ctrl-optimization-solver.ipynb"
 ]
 
 # Don't ever test the following notebooks
 [groups.exclude]
 notebooks = [
-    "docs/guides/algorithmiq-tem.ipynb",
-    "docs/guides/fractional-gates.ipynb",
-    "docs/guides/functions.ipynb",
-    "docs/guides/ibm-circuit-function.ipynb",
+    # This notebook contains undefined variables so can't run at all.
     "docs/guides/multiverse-computing-singularity.ipynb",
-    "docs/guides/q-ctrl-optimization-solver.ipynb",
-    "docs/guides/q-ctrl-performance-management.ipynb",
-    "docs/guides/qedma-qesem.ipynb",
-    "docs/guides/qiskit-addons-sqd-get-started.ipynb",
-    "docs/guides/qiskit-addons-mpf.ipynb",
-    "docs/guides/qunasys-quri-chemistry.ipynb",
 ]