From 4bf007ad64f5907d17c1708f414e42d1329fe99a Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 16 Nov 2021 09:17:53 -0800
Subject: [PATCH 01/41] GPU in wheels, attempt 1

---
 .github/workflows/testing_wheels.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index eeb4d741..c64b811d 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -52,6 +52,11 @@ jobs:
       # Used to host cibuildwheel
       - uses: actions/setup-python@v2
 
+      # Install NVCC on Ubuntu to include GPU in the wheel.
+      - name: cuda-toolkit
+        if: ${{ matrix.os == 'ubuntu-20.04' }}
+        uses: Jimver/cuda-toolkit@v0.2.4
+
       - name: Install cibuildwheel and twine
         run: python -m pip install cibuildwheel==1.11.0
 

From 2b230f72d511768d9eb906fdf3de9e16dff19a37 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 16 Nov 2021 09:55:12 -0800
Subject: [PATCH 02/41] Checking for nvcc binary

---
 .github/workflows/testing_wheels.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index c64b811d..89ea34f6 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -53,10 +53,16 @@ jobs:
       - uses: actions/setup-python@v2
 
       # Install NVCC on Ubuntu to include GPU in the wheel.
+      # TODO: open issues
+      #   - `which nvcc` finds nothing? (need to add to path?)
+      #   - testing GPU on non-GPU machine will fail
       - name: cuda-toolkit
         if: ${{ matrix.os == 'ubuntu-20.04' }}
         uses: Jimver/cuda-toolkit@v0.2.4
 
+      - name: Check for NVCC
+        run: which nvcc
+
       - name: Install cibuildwheel and twine
         run: python -m pip install cibuildwheel==1.11.0
 

From 0c57285b9a19ce7eda6d319b17d27fbdc3b91ae7 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 16 Nov 2021 11:45:08 -0800
Subject: [PATCH 03/41] PATH testing

---
 .github/workflows/testing_wheels.yml | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 89ea34f6..58dc0180 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -55,13 +55,25 @@ jobs:
       # Install NVCC on Ubuntu to include GPU in the wheel.
       # TODO: open issues
       #   - `which nvcc` finds nothing? (need to add to path?)
+      #     - nope, confirmed working at workflow level
+      #     - might still be unavailable lower down? seems unlikely
+      #     - still skipping GPU-based tests
       #   - testing GPU on non-GPU machine will fail
       - name: cuda-toolkit
         if: ${{ matrix.os == 'ubuntu-20.04' }}
         uses: Jimver/cuda-toolkit@v0.2.4
 
-      - name: Check for NVCC
-        run: which nvcc
+      - name: First check
+        if: ${{ matrix.os == 'ubuntu-20.04' }}
+        run: echo "Cuda install location: ${{steps.cuda-toolkit.outputs.CUDA_PATH}}"
+
+      - name: Second check
+        if: ${{ matrix.os == 'ubuntu-20.04' }}
+        run: echo $PATH
+
+      - name: Third check
+        if: ${{ matrix.os == 'ubuntu-20.04' }}
+        run: PATH="${PATH}:${{steps.cuda-toolkit.outputs.CUDA_PATH}}"
 
       - name: Install cibuildwheel and twine
         run: python -m pip install cibuildwheel==1.11.0

From 9d6eac36d47b02520ff5d34475ae4f5ac66acd67 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 16 Nov 2021 11:50:20 -0800
Subject: [PATCH 04/41] Appease YAML

---
 .github/workflows/testing_wheels.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 58dc0180..3316acc3 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -65,7 +65,7 @@ jobs:
 
       - name: First check
         if: ${{ matrix.os == 'ubuntu-20.04' }}
-        run: echo "Cuda install location: ${{steps.cuda-toolkit.outputs.CUDA_PATH}}"
+        run: echo "Cuda install location ${{steps.cuda-toolkit.outputs.CUDA_PATH}}"
 
       - name: Second check
         if: ${{ matrix.os == 'ubuntu-20.04' }}

From d2c2c0066e10a8e9c7ff179e3165e6ff624b31ae Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 16 Nov 2021 13:07:32 -0800
Subject: [PATCH 05/41] CIBW PATH

---
 .github/workflows/testing_wheels.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 3316acc3..ad5b29c6 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -58,6 +58,8 @@ jobs:
       #     - nope, confirmed working at workflow level
       #     - might still be unavailable lower down? seems unlikely
       #     - still skipping GPU-based tests
+      #     - checks suggest CUDA is on the path...what now?
+      #     - might not be getting to CIBW
       #   - testing GPU on non-GPU machine will fail
       - name: cuda-toolkit
         if: ${{ matrix.os == 'ubuntu-20.04' }}
@@ -73,7 +75,8 @@ jobs:
 
       - name: Third check
         if: ${{ matrix.os == 'ubuntu-20.04' }}
-        run: PATH="${PATH}:${{steps.cuda-toolkit.outputs.CUDA_PATH}}"
+        env:
+          CIBW_ENVIRONMENT: PATH=$PATH
 
       - name: Install cibuildwheel and twine
         run: python -m pip install cibuildwheel==1.11.0

From bbb7a50bd7530d91b57b669a8c0fb89570b62f0e Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 16 Nov 2021 13:09:06 -0800
Subject: [PATCH 06/41] jumping YAML hoops

---
 .github/workflows/testing_wheels.yml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index ad5b29c6..e5ee2002 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -73,11 +73,6 @@ jobs:
         if: ${{ matrix.os == 'ubuntu-20.04' }}
         run: echo $PATH
 
-      - name: Third check
-        if: ${{ matrix.os == 'ubuntu-20.04' }}
-        env:
-          CIBW_ENVIRONMENT: PATH=$PATH
-
       - name: Install cibuildwheel and twine
         run: python -m pip install cibuildwheel==1.11.0
 
@@ -85,6 +80,8 @@ jobs:
         run: bash build_tools/test_libs.sh
 
       - name: Build wheels
+        env:
+          CIBW_ENVIRONMENT: PATH=$PATH
         run: python -m cibuildwheel --output-dir wheelhouse
 
       - uses: actions/upload-artifact@v2

From 85220a760f2e6fbd459893d6a279817e4c0809af Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 16 Nov 2021 13:32:21 -0800
Subject: [PATCH 07/41] CIBW which nvcc

---
 .github/workflows/testing_wheels.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index e5ee2002..42052214 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -45,6 +45,7 @@ jobs:
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
       # due to package and module name conflict have to temporarily move it away to run tests
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
+      CIBW_TEST_COMMAND_LINUX: "which nvcc"
       CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py && mv /tmp/qsimcirq {package}"
     steps:
       - uses: actions/checkout@v2
@@ -53,13 +54,15 @@ jobs:
       - uses: actions/setup-python@v2
 
       # Install NVCC on Ubuntu to include GPU in the wheel.
-      # TODO: open issues
+      # TODO: some remaining issues
       #   - `which nvcc` finds nothing? (need to add to path?)
       #     - nope, confirmed working at workflow level
       #     - might still be unavailable lower down? seems unlikely
       #     - still skipping GPU-based tests
       #     - checks suggest CUDA is on the path...what now?
       #     - might not be getting to CIBW
+      #     - tested CIBW_ENVIRONMENT: PATH=$PATH, no visible change
+      #     - new test: `which nvcc` in CIBW
       #   - testing GPU on non-GPU machine will fail
       - name: cuda-toolkit
         if: ${{ matrix.os == 'ubuntu-20.04' }}
@@ -81,7 +84,7 @@ jobs:
 
       - name: Build wheels
         env:
-          CIBW_ENVIRONMENT: PATH=$PATH
+          CIBW_ENVIRONMENT: "PATH=$PATH"
         run: python -m cibuildwheel --output-dir wheelhouse
 
       - uses: actions/upload-artifact@v2

From 033d67f7c52aba1c8a5e050edd2590ab469e7b98 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 19 Nov 2021 11:19:16 -0800
Subject: [PATCH 08/41] hardcoded PATH

---
 .github/workflows/testing_wheels.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 42052214..dafda644 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -45,6 +45,7 @@ jobs:
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
       # due to package and module name conflict have to temporarily move it away to run tests
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
+      CIBW_TEST_COMMAND_LINUX: "echo $PATH"
       CIBW_TEST_COMMAND_LINUX: "which nvcc"
       CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py && mv /tmp/qsimcirq {package}"
     steps:
@@ -62,7 +63,10 @@ jobs:
       #     - checks suggest CUDA is on the path...what now?
       #     - might not be getting to CIBW
       #     - tested CIBW_ENVIRONMENT: PATH=$PATH, no visible change
-      #     - new test: `which nvcc` in CIBW
+      #     - new test: `which nvcc` in CIBW (FAILED)
+      #     - next up:
+      #     - try https://github.com/OpenNMT/CTranslate2/blob/master/.github/workflows/ci.yml
+      #     - CIBW_BEFORE_ALL_LINUX: <script to install CUDA / add to PATH>
       #   - testing GPU on non-GPU machine will fail
       - name: cuda-toolkit
         if: ${{ matrix.os == 'ubuntu-20.04' }}
@@ -84,7 +88,7 @@ jobs:
 
       - name: Build wheels
         env:
-          CIBW_ENVIRONMENT: "PATH=$PATH"
+          CIBW_ENVIRONMENT: "PATH=$PATH:/usr/local/cuda-11.2/bin"
         run: python -m cibuildwheel --output-dir wheelhouse
 
       - uses: actions/upload-artifact@v2

From 178f51bfce9e5b977140b9cccf967d1a8a4073da Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 19 Nov 2021 11:20:35 -0800
Subject: [PATCH 09/41] yaml fixup

---
 .github/workflows/testing_wheels.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index dafda644..72757c12 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -46,7 +46,6 @@ jobs:
       # due to package and module name conflict have to temporarily move it away to run tests
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
       CIBW_TEST_COMMAND_LINUX: "echo $PATH"
-      CIBW_TEST_COMMAND_LINUX: "which nvcc"
       CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py && mv /tmp/qsimcirq {package}"
     steps:
       - uses: actions/checkout@v2

From 2a5ff71bf538a417672156ee84492c6715dd21f6 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Wed, 1 Dec 2021 10:28:50 -0800
Subject: [PATCH 10/41] pwd and ls

---
 .github/workflows/testing_wheels.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 72757c12..409eeba7 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -42,6 +42,7 @@ jobs:
       CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
+      CIBW_BEFORE_BUILD: "pwd && ls"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
       # due to package and module name conflict have to temporarily move it away to run tests
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
@@ -63,9 +64,11 @@ jobs:
       #     - might not be getting to CIBW
       #     - tested CIBW_ENVIRONMENT: PATH=$PATH, no visible change
       #     - new test: `which nvcc` in CIBW (FAILED)
+      #     - hardcoded PATH generates absurd errors?
+      #       - but the first wheel-build works. <===========
       #     - next up:
       #     - try https://github.com/OpenNMT/CTranslate2/blob/master/.github/workflows/ci.yml
-      #     - CIBW_BEFORE_ALL_LINUX: <script to install CUDA / add to PATH>
+      #       - CIBW_BEFORE_ALL_LINUX: <script to install CUDA / add to PATH>
       #   - testing GPU on non-GPU machine will fail
       - name: cuda-toolkit
         if: ${{ matrix.os == 'ubuntu-20.04' }}

From bf2061de45287f279c2ec49c3bb0b581a6f92e3e Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Wed, 1 Dec 2021 10:48:11 -0800
Subject: [PATCH 11/41] rm CIBW_TEST_COMMAND_LINUX

---
 .github/workflows/testing_wheels.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 409eeba7..19eeb60c 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -45,8 +45,10 @@ jobs:
       CIBW_BEFORE_BUILD: "pwd && ls"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
       # due to package and module name conflict have to temporarily move it away to run tests
+      # TODO: this might be the culprit, /project/qsimcirq doesn't exist in second go-round
+      #   CIBW_TEST_COMMAND_LINUX was overriding CIBW_TEST_COMMAND
+      #   Still need to prevent GPU tests from trying to run?
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
-      CIBW_TEST_COMMAND_LINUX: "echo $PATH"
       CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py && mv /tmp/qsimcirq {package}"
     steps:
       - uses: actions/checkout@v2

From 74942aa32b8b6dca5078841bc3b227f53270e730 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 3 Dec 2021 08:58:58 -0800
Subject: [PATCH 12/41] the mystery deepens

---
 .github/workflows/testing_wheels.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 19eeb60c..5d95e8d2 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -45,9 +45,6 @@ jobs:
       CIBW_BEFORE_BUILD: "pwd && ls"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
       # due to package and module name conflict have to temporarily move it away to run tests
-      # TODO: this might be the culprit, /project/qsimcirq doesn't exist in second go-round
-      #   CIBW_TEST_COMMAND_LINUX was overriding CIBW_TEST_COMMAND
-      #   Still need to prevent GPU tests from trying to run?
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
       CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py && mv /tmp/qsimcirq {package}"
     steps:
@@ -67,7 +64,8 @@ jobs:
       #     - tested CIBW_ENVIRONMENT: PATH=$PATH, no visible change
       #     - new test: `which nvcc` in CIBW (FAILED)
       #     - hardcoded PATH generates absurd errors?
-      #       - but the first wheel-build works. <===========
+      #       - but the first wheel-build works.
+      #       - reran with fixes and the result wheel didn't have GPU <===========
       #     - next up:
       #     - try https://github.com/OpenNMT/CTranslate2/blob/master/.github/workflows/ci.yml
       #       - CIBW_BEFORE_ALL_LINUX: <script to install CUDA / add to PATH>

From dc6a9e88ceb807d2e1e1a90e3828f92b4e67e3c0 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 3 Dec 2021 09:21:02 -0800
Subject: [PATCH 13/41] env-pass

---
 .github/workflows/testing_wheels.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 5d95e8d2..e2651b00 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -67,6 +67,7 @@ jobs:
       #       - but the first wheel-build works.
       #       - reran with fixes and the result wheel didn't have GPU <===========
       #     - next up:
+      #     - CIBW_ENVIRONMENT_PASS_LINUX: env vars might not be going through?
       #     - try https://github.com/OpenNMT/CTranslate2/blob/master/.github/workflows/ci.yml
       #       - CIBW_BEFORE_ALL_LINUX: <script to install CUDA / add to PATH>
       #   - testing GPU on non-GPU machine will fail
@@ -90,7 +91,7 @@ jobs:
 
       - name: Build wheels
         env:
-          CIBW_ENVIRONMENT: "PATH=$PATH:/usr/local/cuda-11.2/bin"
+          CIBW_ENVIRONMENT_PASS_LINUX: "PATH=$PATH:/usr/local/cuda-11.2/bin"
         run: python -m cibuildwheel --output-dir wheelhouse
 
       - uses: actions/upload-artifact@v2

From 43db2caeac449dd0a858a081a63e7e43b6d6401f Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 3 Dec 2021 09:50:30 -0800
Subject: [PATCH 14/41] script-based CUDA install

---
 .github/workflows/testing_wheels.yml | 28 +++++++++++++---------------
 build_tools/cuda_install.sh          |  2 ++
 2 files changed, 15 insertions(+), 15 deletions(-)
 create mode 100644 build_tools/cuda_install.sh

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index e2651b00..aae24afc 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -41,6 +41,7 @@ jobs:
       CIBW_BUILD: "${{ matrix.cibw.build || '*' }}"
       CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
+      CIBW_BEFORE_ALL_LINUX: "build_tools/cuda_install.sh"
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
       CIBW_BEFORE_BUILD: "pwd && ls"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
@@ -65,23 +66,22 @@ jobs:
       #     - new test: `which nvcc` in CIBW (FAILED)
       #     - hardcoded PATH generates absurd errors?
       #       - but the first wheel-build works.
-      #       - reran with fixes and the result wheel didn't have GPU <===========
-      #     - next up:
+      #       - reran with fixes and the result wheel didn't have GPU
       #     - CIBW_ENVIRONMENT_PASS_LINUX: env vars might not be going through?
-      #     - try https://github.com/OpenNMT/CTranslate2/blob/master/.github/workflows/ci.yml
-      #       - CIBW_BEFORE_ALL_LINUX: <script to install CUDA / add to PATH>
+      #       - failed
+      #     - CUDA install script in CIBW_BEFORE_ALL_LINUX <==========
       #   - testing GPU on non-GPU machine will fail
-      - name: cuda-toolkit
-        if: ${{ matrix.os == 'ubuntu-20.04' }}
-        uses: Jimver/cuda-toolkit@v0.2.4
+      # - name: cuda-toolkit
+      #   if: ${{ matrix.os == 'ubuntu-20.04' }}
+      #   uses: Jimver/cuda-toolkit@v0.2.4
 
-      - name: First check
-        if: ${{ matrix.os == 'ubuntu-20.04' }}
-        run: echo "Cuda install location ${{steps.cuda-toolkit.outputs.CUDA_PATH}}"
+      # - name: First check
+      #   if: ${{ matrix.os == 'ubuntu-20.04' }}
+      #   run: echo "Cuda install location ${{steps.cuda-toolkit.outputs.CUDA_PATH}}"
 
-      - name: Second check
-        if: ${{ matrix.os == 'ubuntu-20.04' }}
-        run: echo $PATH
+      # - name: Second check
+      #   if: ${{ matrix.os == 'ubuntu-20.04' }}
+      #   run: echo $PATH
 
       - name: Install cibuildwheel and twine
         run: python -m pip install cibuildwheel==1.11.0
@@ -90,8 +90,6 @@ jobs:
         run: bash build_tools/test_libs.sh
 
       - name: Build wheels
-        env:
-          CIBW_ENVIRONMENT_PASS_LINUX: "PATH=$PATH:/usr/local/cuda-11.2/bin"
         run: python -m cibuildwheel --output-dir wheelhouse
 
       - uses: actions/upload-artifact@v2
diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
new file mode 100644
index 00000000..a2aaa4eb
--- /dev/null
+++ b/build_tools/cuda_install.sh
@@ -0,0 +1,2 @@
+curl https://raw.githubusercontent.com/GoogleCloudPlatform/compute-gpu-installation/main/linux/install_gpu_driver.py --output install_gpu_driver.py
+sudo python3 install_gpu_driver.py --force

From 159aec6573bd96ad73d00e578712d55bd6b3e9a7 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 3 Dec 2021 10:02:12 -0800
Subject: [PATCH 15/41] bash the .sh

---
 .github/workflows/testing_wheels.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index aae24afc..d2c08c18 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -41,7 +41,7 @@ jobs:
       CIBW_BUILD: "${{ matrix.cibw.build || '*' }}"
       CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
-      CIBW_BEFORE_ALL_LINUX: "build_tools/cuda_install.sh"
+      CIBW_BEFORE_ALL_LINUX: "bash build_tools/cuda_install.sh"
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
       CIBW_BEFORE_BUILD: "pwd && ls"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
@@ -86,8 +86,9 @@ jobs:
       - name: Install cibuildwheel and twine
         run: python -m pip install cibuildwheel==1.11.0
 
-      - name: Run C++ tests
-        run: bash build_tools/test_libs.sh
+      # temporary
+      # - name: Run C++ tests
+      #   run: bash build_tools/test_libs.sh
 
       - name: Build wheels
         run: python -m cibuildwheel --output-dir wheelhouse

From c53ce633491585d2d66c9385578dcad728ac4527 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 3 Dec 2021 10:10:47 -0800
Subject: [PATCH 16/41] don't sudo what's already sudone

---
 build_tools/cuda_install.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index a2aaa4eb..6c9262af 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -1,2 +1,2 @@
 curl https://raw.githubusercontent.com/GoogleCloudPlatform/compute-gpu-installation/main/linux/install_gpu_driver.py --output install_gpu_driver.py
-sudo python3 install_gpu_driver.py --force
+python3 install_gpu_driver.py --force

From 925cc0bda4fe36c13670bbbc47de0105be6efceb Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 3 Dec 2021 10:55:11 -0800
Subject: [PATCH 17/41] preload deps to prevent reboot

---
 .github/workflows/testing_wheels.yml | 5 ++++-
 build_tools/cuda_install.sh          | 8 ++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index d2c08c18..1c588de8 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -69,7 +69,10 @@ jobs:
       #       - reran with fixes and the result wheel didn't have GPU
       #     - CIBW_ENVIRONMENT_PASS_LINUX: env vars might not be going through?
       #       - failed
-      #     - CUDA install script in CIBW_BEFORE_ALL_LINUX <==========
+      #     - CUDA install script in CIBW_BEFORE_ALL_LINUX
+      #       - tries and fails to reboot
+      #       - might be related to "drivers req'd outside container" issue
+      #       - attempt fix w/pre-installed dependencies <==========
       #   - testing GPU on non-GPU machine will fail
       # - name: cuda-toolkit
       #   if: ${{ matrix.os == 'ubuntu-20.04' }}
diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index 6c9262af..1b107edd 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -1,2 +1,10 @@
 curl https://raw.githubusercontent.com/GoogleCloudPlatform/compute-gpu-installation/main/linux/install_gpu_driver.py --output install_gpu_driver.py
+
+# pre-install dependencies to prevent reboot
+kernel_version=$(uname -r)
+yum clean all
+yum install -y kernel
+yum install -y kernel-devel-${kernel_version} kernel-headers-${kernel_version} pciutils
+
+# run the installer script
 python3 install_gpu_driver.py --force

From 4c85cd3abb33b4fc598fa45e68c0da46ca8ba99f Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 3 Dec 2021 11:17:57 -0800
Subject: [PATCH 18/41] double install, yay

---
 .github/workflows/testing_wheels.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 1c588de8..66898170 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -72,11 +72,12 @@ jobs:
       #     - CUDA install script in CIBW_BEFORE_ALL_LINUX
       #       - tries and fails to reboot
       #       - might be related to "drivers req'd outside container" issue
-      #       - attempt fix w/pre-installed dependencies <==========
+      #       - attempt fix w/pre-installed dependencies
+      #       - no luck, but might require separate out-of-container install? <==========
       #   - testing GPU on non-GPU machine will fail
-      # - name: cuda-toolkit
-      #   if: ${{ matrix.os == 'ubuntu-20.04' }}
-      #   uses: Jimver/cuda-toolkit@v0.2.4
+      - name: cuda-toolkit
+        if: ${{ matrix.os == 'ubuntu-20.04' }}
+        uses: Jimver/cuda-toolkit@v0.2.4
 
       # - name: First check
       #   if: ${{ matrix.os == 'ubuntu-20.04' }}

From 1142a18195ea18d3228753cb151d3719bddddccc Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 3 Dec 2021 11:47:24 -0800
Subject: [PATCH 19/41] ooc install, ic no pre-install

---
 .github/workflows/testing_wheels.yml | 3 ++-
 build_tools/cuda_install.sh          | 6 ------
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 66898170..78366171 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -73,7 +73,8 @@ jobs:
       #       - tries and fails to reboot
       #       - might be related to "drivers req'd outside container" issue
       #       - attempt fix w/pre-installed dependencies
-      #       - no luck, but might require separate out-of-container install? <==========
+      #       - no luck, but might require separate out-of-container install?
+      #       - failed again, trying OOC install + no pre-install <==========
       #   - testing GPU on non-GPU machine will fail
       - name: cuda-toolkit
         if: ${{ matrix.os == 'ubuntu-20.04' }}
diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index 1b107edd..924cf37e 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -1,10 +1,4 @@
 curl https://raw.githubusercontent.com/GoogleCloudPlatform/compute-gpu-installation/main/linux/install_gpu_driver.py --output install_gpu_driver.py
 
-# pre-install dependencies to prevent reboot
-kernel_version=$(uname -r)
-yum clean all
-yum install -y kernel
-yum install -y kernel-devel-${kernel_version} kernel-headers-${kernel_version} pciutils
-
 # run the installer script
 python3 install_gpu_driver.py --force

From bf7d0297fd8e3fa5ac61f7419c1f1af7af0697d2 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 10 Dec 2021 08:41:54 -0800
Subject: [PATCH 20/41] move script into qsim

---
 build_tools/cuda_install.sh | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index 924cf37e..a6f8716f 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -1,4 +1,25 @@
-curl https://raw.githubusercontent.com/GoogleCloudPlatform/compute-gpu-installation/main/linux/install_gpu_driver.py --output install_gpu_driver.py
+#!/bin/bash
+set -x
 
-# run the installer script
-python3 install_gpu_driver.py --force
+# install dependencies
+kernel_version=$(uname -r)
+yum clean all
+yum install -y kernel
+yum install -y kernel-devel-${kernel_version} kernel-headers-${kernel_version} pciutils
+
+# install CUDA toolkit
+
+yum install -y yum-utils epel-release
+yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo
+yum clean all
+yum -y install nvidia-driver-latest-dkms cuda
+yum -y install cuda-drivers
+
+# post-install actions
+
+CUDA_PROFILE="/etc/profile.d/cuda.sh"
+echo "# CUDA Toolkit settings" >> $CUDA_PROFILE
+echo "export PATH=/usr/local/cuda-11/bin:$PATH" >> $CUDA_PROFILE
+echo "export LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH" >> $CUDA_PROFILE
+
+set +x
\ No newline at end of file

From 95e2d73255f42ecb4020c37ba007d72adf2190ad Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 10 Dec 2021 09:28:42 -0800
Subject: [PATCH 21/41] export directly

---
 build_tools/cuda_install.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index a6f8716f..0ef3c867 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -22,4 +22,7 @@ echo "# CUDA Toolkit settings" >> $CUDA_PROFILE
 echo "export PATH=/usr/local/cuda-11/bin:$PATH" >> $CUDA_PROFILE
 echo "export LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH" >> $CUDA_PROFILE
 
+export PATH=/usr/local/cuda-11/bin:$PATH
+export LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
+
 set +x
\ No newline at end of file

From 3a6b86f47d0987cd4db6150ef4fc50cf7e342aaa Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 10 Dec 2021 11:08:14 -0800
Subject: [PATCH 22/41] need more data

---
 .github/workflows/testing_wheels.yml | 2 +-
 build_tools/cuda_install.sh          | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 78366171..3affc7b7 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -41,7 +41,7 @@ jobs:
       CIBW_BUILD: "${{ matrix.cibw.build || '*' }}"
       CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
-      CIBW_BEFORE_ALL_LINUX: "bash build_tools/cuda_install.sh"
+      CIBW_BEFORE_ALL_LINUX: "bash build_tools/cuda_install.sh && echo $PATH"
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
       CIBW_BEFORE_BUILD: "pwd && ls"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index 0ef3c867..edf7973d 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -25,4 +25,7 @@ echo "export LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH" >> $CUDA
 export PATH=/usr/local/cuda-11/bin:$PATH
 export LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
 
+nvcc --version
+nvidia-smi
+
 set +x
\ No newline at end of file

From 6033c9f772921ba51f2a39c470e96207ccbf97c7 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 10 Dec 2021 11:51:43 -0800
Subject: [PATCH 23/41] sourcery

---
 .github/workflows/testing_wheels.yml | 2 +-
 build_tools/cuda_install.sh          | 5 -----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 3affc7b7..4a5352bc 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -41,7 +41,7 @@ jobs:
       CIBW_BUILD: "${{ matrix.cibw.build || '*' }}"
       CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
-      CIBW_BEFORE_ALL_LINUX: "bash build_tools/cuda_install.sh && echo $PATH"
+      CIBW_BEFORE_ALL_LINUX: "source build_tools/cuda_install.sh && echo $PATH"
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
       CIBW_BEFORE_BUILD: "pwd && ls"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index edf7973d..0ac1480b 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -17,11 +17,6 @@ yum -y install cuda-drivers
 
 # post-install actions
 
-CUDA_PROFILE="/etc/profile.d/cuda.sh"
-echo "# CUDA Toolkit settings" >> $CUDA_PROFILE
-echo "export PATH=/usr/local/cuda-11/bin:$PATH" >> $CUDA_PROFILE
-echo "export LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH" >> $CUDA_PROFILE
-
 export PATH=/usr/local/cuda-11/bin:$PATH
 export LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
 

From 578264fc7734e6dd3be3adcd37772726452e362e Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 10 Dec 2021 12:52:40 -0800
Subject: [PATCH 24/41] version and which-nvcc check

---
 .github/workflows/testing_wheels.yml | 5 +++--
 build_tools/cuda_install.sh          | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 4a5352bc..0c19848d 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -43,7 +43,8 @@ jobs:
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
       CIBW_BEFORE_ALL_LINUX: "source build_tools/cuda_install.sh && echo $PATH"
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
-      CIBW_BEFORE_BUILD: "pwd && ls"
+      # *cuda.so was not built, suspect `which nvcc` is null
+      CIBW_BEFORE_BUILD: "echo $PATH && which nvcc"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
       # due to package and module name conflict have to temporarily move it away to run tests
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
@@ -78,7 +79,7 @@ jobs:
       #   - testing GPU on non-GPU machine will fail
       - name: cuda-toolkit
         if: ${{ matrix.os == 'ubuntu-20.04' }}
-        uses: Jimver/cuda-toolkit@v0.2.4
+        uses: Jimver/cuda-toolkit@v0.2.5
 
       # - name: First check
       #   if: ${{ matrix.os == 'ubuntu-20.04' }}
diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index 0ac1480b..9b8e5c1b 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -21,6 +21,7 @@ export PATH=/usr/local/cuda-11/bin:$PATH
 export LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
 
 nvcc --version
+which nvcc
 nvidia-smi
 
-set +x
\ No newline at end of file
+set +x

From 247bddaa40294f01581f88d1a962f19f437ac456 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 10 Dec 2021 14:20:49 -0800
Subject: [PATCH 25/41] passthrough PATH

---
 .github/workflows/testing_wheels.yml | 4 +++-
 build_tools/cuda_install.sh          | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 0c19848d..a4e4871d 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -42,9 +42,11 @@ jobs:
       CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
       CIBW_BEFORE_ALL_LINUX: "source build_tools/cuda_install.sh && echo $PATH"
+      CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/cuda-11/bin:$PATH
+      CIBW_ENVIRONMENT_PASS_LINUX: PATH
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
       # *cuda.so was not built, suspect `which nvcc` is null
-      CIBW_BEFORE_BUILD: "echo $PATH && which nvcc"
+      CIBW_BEFORE_BUILD_LINUX: "echo $PATH && which nvcc"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
       # due to package and module name conflict have to temporarily move it away to run tests
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index 9b8e5c1b..d27a53be 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -20,6 +20,7 @@ yum -y install cuda-drivers
 export PATH=/usr/local/cuda-11/bin:$PATH
 export LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
 
+# works here, does not escape to build process
 nvcc --version
 which nvcc
 nvidia-smi

From 47361d184035c80a4b3da30e05cb7892c3e7e040 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Fri, 10 Dec 2021 15:13:23 -0800
Subject: [PATCH 26/41] move the env

---
 .github/workflows/testing_wheels.yml | 29 ++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index a4e4871d..fbd96a25 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -37,20 +37,6 @@ jobs:
             architecture: x64
             cibw:
               build: "cp36-win_amd64 cp37-win_amd64 cp38-win_amd64 cp39-win_amd64"
-    env:
-      CIBW_BUILD: "${{ matrix.cibw.build || '*' }}"
-      CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
-      CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
-      CIBW_BEFORE_ALL_LINUX: "source build_tools/cuda_install.sh && echo $PATH"
-      CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/cuda-11/bin:$PATH
-      CIBW_ENVIRONMENT_PASS_LINUX: PATH
-      CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
-      # *cuda.so was not built, suspect `which nvcc` is null
-      CIBW_BEFORE_BUILD_LINUX: "echo $PATH && which nvcc"
-      CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
-      # due to package and module name conflict have to temporarily move it away to run tests
-      CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
-      CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py && mv /tmp/qsimcirq {package}"
     steps:
       - uses: actions/checkout@v2
 
@@ -99,6 +85,21 @@ jobs:
       #   run: bash build_tools/test_libs.sh
 
       - name: Build wheels
+        env:
+          CIBW_BUILD: "${{ matrix.cibw.build || '*' }}"
+          CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
+          CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
+          CIBW_BEFORE_ALL_LINUX: "source build_tools/cuda_install.sh && echo $PATH"
+          CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/cuda-11/bin:$PATH
+          # overwriting PATH is breaking Python - need to add to PATH somehow?
+          CIBW_ENVIRONMENT_PASS_LINUX: PATH
+          CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
+          # *cuda.so was not built, suspect `which nvcc` is null
+          CIBW_BEFORE_BUILD_LINUX: "echo $PATH && which nvcc"
+          CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
+          # due to package and module name conflict have to temporarily move it away to run tests
+          CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
+          CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py && mv /tmp/qsimcirq {package}"
         run: python -m cibuildwheel --output-dir wheelhouse
 
       - uses: actions/upload-artifact@v2

From aed7e6a96ccb16a9254f4281a6ade7588eb11c66 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Mon, 13 Dec 2021 09:07:15 -0800
Subject: [PATCH 27/41] less PATH is more PATH

---
 .github/workflows/testing_wheels.yml | 31 ++++++++++++++--------------
 build_tools/cuda_install.sh          |  5 +++--
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index fbd96a25..4ab215fc 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -37,6 +37,22 @@ jobs:
             architecture: x64
             cibw:
               build: "cp36-win_amd64 cp37-win_amd64 cp38-win_amd64 cp39-win_amd64"
+    env:
+      # Note: Location of env does not matter.
+      CIBW_BUILD: "${{ matrix.cibw.build || '*' }}"
+      CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
+      CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
+      CIBW_BEFORE_ALL_LINUX: "bash build_tools/cuda_install.sh && echo $PATH"
+      # overwriting PATH is breaking Python - need to add to PATH somehow?
+      CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/cuda-11/bin:$PATH LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
+      CIBW_ENVIRONMENT_PASS_LINUX: PATH LD_LIBRARY_PATH
+      CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
+      # *cuda.so was not built, suspect `which nvcc` is null
+      CIBW_BEFORE_BUILD_LINUX: "echo $PATH && which nvcc"
+      CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
+      # due to package and module name conflict have to temporarily move it away to run tests
+      CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
+      CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py && mv /tmp/qsimcirq {package}"
     steps:
       - uses: actions/checkout@v2
 
@@ -85,21 +101,6 @@ jobs:
       #   run: bash build_tools/test_libs.sh
 
       - name: Build wheels
-        env:
-          CIBW_BUILD: "${{ matrix.cibw.build || '*' }}"
-          CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
-          CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
-          CIBW_BEFORE_ALL_LINUX: "source build_tools/cuda_install.sh && echo $PATH"
-          CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/cuda-11/bin:$PATH
-          # overwriting PATH is breaking Python - need to add to PATH somehow?
-          CIBW_ENVIRONMENT_PASS_LINUX: PATH
-          CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
-          # *cuda.so was not built, suspect `which nvcc` is null
-          CIBW_BEFORE_BUILD_LINUX: "echo $PATH && which nvcc"
-          CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
-          # due to package and module name conflict have to temporarily move it away to run tests
-          CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
-          CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py && mv /tmp/qsimcirq {package}"
         run: python -m cibuildwheel --output-dir wheelhouse
 
       - uses: actions/upload-artifact@v2
diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index d27a53be..d5ec48cc 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -17,8 +17,9 @@ yum -y install cuda-drivers
 
 # post-install actions
 
-export PATH=/usr/local/cuda-11/bin:$PATH
-export LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
+# commented to allow outer PATH-setting
+# export PATH=/usr/local/cuda-11/bin:$PATH
+# export LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
 
 # works here, does not escape to build process
 nvcc --version

From 164974c98679d9176f024ed56f64ea4ba92dd31e Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Mon, 13 Dec 2021 09:19:17 -0800
Subject: [PATCH 28/41] and without changing the PATH...

---
 .github/workflows/testing_wheels.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 4ab215fc..15daaa26 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -44,8 +44,8 @@ jobs:
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
       CIBW_BEFORE_ALL_LINUX: "bash build_tools/cuda_install.sh && echo $PATH"
       # overwriting PATH is breaking Python - need to add to PATH somehow?
-      CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/cuda-11/bin:$PATH LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
-      CIBW_ENVIRONMENT_PASS_LINUX: PATH LD_LIBRARY_PATH
+      # CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/cuda-11/bin:$PATH LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
+      # CIBW_ENVIRONMENT_PASS_LINUX: PATH LD_LIBRARY_PATH
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
       # *cuda.so was not built, suspect `which nvcc` is null
       CIBW_BEFORE_BUILD_LINUX: "echo $PATH && which nvcc"

From 9671f4a3efdc0f1d1f8ae0e58ba2a2b54421a47f Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Mon, 13 Dec 2021 09:35:31 -0800
Subject: [PATCH 29/41] remove which nvcc also

---
 .github/workflows/testing_wheels.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 15daaa26..68c628f1 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -48,7 +48,7 @@ jobs:
       # CIBW_ENVIRONMENT_PASS_LINUX: PATH LD_LIBRARY_PATH
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
       # *cuda.so was not built, suspect `which nvcc` is null
-      CIBW_BEFORE_BUILD_LINUX: "echo $PATH && which nvcc"
+      # CIBW_BEFORE_BUILD_LINUX: "echo $PATH && which nvcc"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
       # due to package and module name conflict have to temporarily move it away to run tests
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"

From a09c4adc8743c50802f65a5070902549e061e505 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Mon, 13 Dec 2021 11:20:09 -0800
Subject: [PATCH 30/41] pass unmodified PATH

---
 .github/workflows/testing_wheels.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 68c628f1..92be511a 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -45,7 +45,8 @@ jobs:
       CIBW_BEFORE_ALL_LINUX: "bash build_tools/cuda_install.sh && echo $PATH"
       # overwriting PATH is breaking Python - need to add to PATH somehow?
       # CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/cuda-11/bin:$PATH LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
-      # CIBW_ENVIRONMENT_PASS_LINUX: PATH LD_LIBRARY_PATH
+      # TEST: pass in PATH without changing it
+      CIBW_ENVIRONMENT_PASS_LINUX: PATH
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
       # *cuda.so was not built, suspect `which nvcc` is null
       # CIBW_BEFORE_BUILD_LINUX: "echo $PATH && which nvcc"

From ba997ef7d4e57680896b07178dc503195e1700a4 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 21 Dec 2021 08:49:52 -0800
Subject: [PATCH 31/41] path after PATH

---
 .github/workflows/testing_wheels.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 92be511a..e334287a 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -44,8 +44,7 @@ jobs:
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
       CIBW_BEFORE_ALL_LINUX: "bash build_tools/cuda_install.sh && echo $PATH"
       # overwriting PATH is breaking Python - need to add to PATH somehow?
-      # CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/cuda-11/bin:$PATH LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
-      # TEST: pass in PATH without changing it
+      CIBW_ENVIRONMENT_LINUX: PATH=$PATH:/usr/local/cuda-11/bin LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-11/lib64
       CIBW_ENVIRONMENT_PASS_LINUX: PATH
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
       # *cuda.so was not built, suspect `which nvcc` is null

From 29d499314ff324a4ea9ce3400e303c48e2e3c5be Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 21 Dec 2021 09:56:02 -0800
Subject: [PATCH 32/41] LD_LIBRARY_PATH passthrough

---
 .github/workflows/testing_wheels.yml | 35 +---------------------------
 1 file changed, 1 insertion(+), 34 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index e334287a..7dfff989 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -38,17 +38,13 @@ jobs:
             cibw:
               build: "cp36-win_amd64 cp37-win_amd64 cp38-win_amd64 cp39-win_amd64"
     env:
-      # Note: Location of env does not matter.
       CIBW_BUILD: "${{ matrix.cibw.build || '*' }}"
       CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
       CIBW_BEFORE_ALL_LINUX: "bash build_tools/cuda_install.sh && echo $PATH"
-      # overwriting PATH is breaking Python - need to add to PATH somehow?
       CIBW_ENVIRONMENT_LINUX: PATH=$PATH:/usr/local/cuda-11/bin LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-11/lib64
-      CIBW_ENVIRONMENT_PASS_LINUX: PATH
+      CIBW_ENVIRONMENT_PASS_LINUX: PATH LD_LIBRARY_PATH
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
-      # *cuda.so was not built, suspect `which nvcc` is null
-      # CIBW_BEFORE_BUILD_LINUX: "echo $PATH && which nvcc"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
       # due to package and module name conflict have to temporarily move it away to run tests
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
@@ -60,39 +56,10 @@ jobs:
       - uses: actions/setup-python@v2
 
       # Install NVCC on Ubuntu to include GPU in the wheel.
-      # TODO: some remaining issues
-      #   - `which nvcc` finds nothing? (need to add to path?)
-      #     - nope, confirmed working at workflow level
-      #     - might still be unavailable lower down? seems unlikely
-      #     - still skipping GPU-based tests
-      #     - checks suggest CUDA is on the path...what now?
-      #     - might not be getting to CIBW
-      #     - tested CIBW_ENVIRONMENT: PATH=$PATH, no visible change
-      #     - new test: `which nvcc` in CIBW (FAILED)
-      #     - hardcoded PATH generates absurd errors?
-      #       - but the first wheel-build works.
-      #       - reran with fixes and the result wheel didn't have GPU
-      #     - CIBW_ENVIRONMENT_PASS_LINUX: env vars might not be going through?
-      #       - failed
-      #     - CUDA install script in CIBW_BEFORE_ALL_LINUX
-      #       - tries and fails to reboot
-      #       - might be related to "drivers req'd outside container" issue
-      #       - attempt fix w/pre-installed dependencies
-      #       - no luck, but might require separate out-of-container install?
-      #       - failed again, trying OOC install + no pre-install <==========
-      #   - testing GPU on non-GPU machine will fail
       - name: cuda-toolkit
         if: ${{ matrix.os == 'ubuntu-20.04' }}
         uses: Jimver/cuda-toolkit@v0.2.5
 
-      # - name: First check
-      #   if: ${{ matrix.os == 'ubuntu-20.04' }}
-      #   run: echo "Cuda install location ${{steps.cuda-toolkit.outputs.CUDA_PATH}}"
-
-      # - name: Second check
-      #   if: ${{ matrix.os == 'ubuntu-20.04' }}
-      #   run: echo $PATH
-
       - name: Install cibuildwheel and twine
         run: python -m pip install cibuildwheel==1.11.0
 

From 102622162c91e87fdf5455e78a710bc04522fee6 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 21 Dec 2021 10:13:26 -0800
Subject: [PATCH 33/41] find Python (not libs)

---
 pybind_interface/cuda/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pybind_interface/cuda/CMakeLists.txt b/pybind_interface/cuda/CMakeLists.txt
index d7f5b836..5eb5e130 100644
--- a/pybind_interface/cuda/CMakeLists.txt
+++ b/pybind_interface/cuda/CMakeLists.txt
@@ -27,7 +27,7 @@ if(NOT pybind11_POPULATED)
   add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR})
 endif()
 
-find_package(PythonLibs 3.6 REQUIRED)
+find_package(Python 3.6 REQUIRED)
 find_package(CUDA REQUIRED)
 
 include_directories(${PYTHON_INCLUDE_DIRS} ${pybind11_SOURCE_DIR}/include)

From 9989069b1e4faeb853d719030e3e2f9dedcdfd05 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 21 Dec 2021 10:34:44 -0800
Subject: [PATCH 34/41] find all components

---
 pybind_interface/cuda/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pybind_interface/cuda/CMakeLists.txt b/pybind_interface/cuda/CMakeLists.txt
index 5eb5e130..eccb60f5 100644
--- a/pybind_interface/cuda/CMakeLists.txt
+++ b/pybind_interface/cuda/CMakeLists.txt
@@ -27,7 +27,7 @@ if(NOT pybind11_POPULATED)
   add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR})
 endif()
 
-find_package(Python 3.6 REQUIRED)
+find_package(Python 3.6 REQUIRED COMPONENTS Interpreter Development)
 find_package(CUDA REQUIRED)
 
 include_directories(${PYTHON_INCLUDE_DIRS} ${pybind11_SOURCE_DIR}/include)

From 2dc97e051be7afac4c91b448eb18806354130a8c Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 21 Dec 2021 11:02:35 -0800
Subject: [PATCH 35/41] install python-devel

---
 build_tools/cuda_install.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index d5ec48cc..9fb6c159 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -7,9 +7,10 @@ yum clean all
 yum install -y kernel
 yum install -y kernel-devel-${kernel_version} kernel-headers-${kernel_version} pciutils
 
-# install CUDA toolkit
+# install python and CUDA toolkit
 
 yum install -y yum-utils epel-release
+yum install -y python3-devel.x86_64
 yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo
 yum clean all
 yum -y install nvidia-driver-latest-dkms cuda

From 08080f82dbf43e987ceb70f3d90476fc551cb681 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Tue, 21 Dec 2021 13:07:11 -0800
Subject: [PATCH 36/41] prioritize CUDA in PATH

---
 .github/workflows/testing_wheels.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 7dfff989..189ce1ed 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -42,7 +42,9 @@ jobs:
       CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
       CIBW_BEFORE_ALL_LINUX: "bash build_tools/cuda_install.sh && echo $PATH"
-      CIBW_ENVIRONMENT_LINUX: PATH=$PATH:/usr/local/cuda-11/bin LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-11/lib64
+      # Testing: wrong immintrin headers may be due to PATH priority order.
+      # Switched new CUDA paths to front in attempt to fix.
+      CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/cuda-11/bin:$PATH LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
       CIBW_ENVIRONMENT_PASS_LINUX: PATH LD_LIBRARY_PATH
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""

From 3095486f00e1d5d7b9567b245ccdbcb6c551bcb4 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Thu, 23 Dec 2021 08:22:02 -0800
Subject: [PATCH 37/41] try version pin

---
 .github/workflows/testing_wheels.yml | 2 ++
 build_tools/cuda_install.sh          | 1 +
 2 files changed, 3 insertions(+)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 189ce1ed..a96e9ef9 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -61,6 +61,8 @@ jobs:
       - name: cuda-toolkit
         if: ${{ matrix.os == 'ubuntu-20.04' }}
         uses: Jimver/cuda-toolkit@v0.2.5
+        with:
+          cuda: "11.5.1"
 
       - name: Install cibuildwheel and twine
         run: python -m pip install cibuildwheel==1.11.0
diff --git a/build_tools/cuda_install.sh b/build_tools/cuda_install.sh
index 9fb6c159..5530df83 100644
--- a/build_tools/cuda_install.sh
+++ b/build_tools/cuda_install.sh
@@ -8,6 +8,7 @@ yum install -y kernel
 yum install -y kernel-devel-${kernel_version} kernel-headers-${kernel_version} pciutils
 
 # install python and CUDA toolkit
+# TODO: need to pin version
 
 yum install -y yum-utils epel-release
 yum install -y python3-devel.x86_64

From f9e88d39b91cdee3b44309436d3e7ac6e93ba4a8 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Wed, 5 Jan 2022 15:23:11 -0800
Subject: [PATCH 38/41] Skip GPU tests on GPU-less machines

---
 .github/workflows/testing_wheels.yml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index a96e9ef9..61716025 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -42,15 +42,14 @@ jobs:
       CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
       CIBW_BEFORE_ALL_LINUX: "bash build_tools/cuda_install.sh && echo $PATH"
-      # Testing: wrong immintrin headers may be due to PATH priority order.
-      # Switched new CUDA paths to front in attempt to fix.
       CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/cuda-11/bin:$PATH LD_LIBRARY_PATH=/usr/local/cuda-11/lib64:$LD_LIBRARY_PATH
       CIBW_ENVIRONMENT_PASS_LINUX: PATH LD_LIBRARY_PATH
       CIBW_BEFORE_BUILD_MACOS: "brew install libomp"
       CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
       # due to package and module name conflict have to temporarily move it away to run tests
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
-      CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py && mv /tmp/qsimcirq {package}"
+      # prevent GPU tests (no GPU on GHA runners)
+      CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py -k 'not gpu' && mv /tmp/qsimcirq {package}"
     steps:
       - uses: actions/checkout@v2
 
@@ -67,10 +66,6 @@ jobs:
       - name: Install cibuildwheel and twine
         run: python -m pip install cibuildwheel==1.11.0
 
-      # temporary
-      # - name: Run C++ tests
-      #   run: bash build_tools/test_libs.sh
-
       - name: Build wheels
         run: python -m cibuildwheel --output-dir wheelhouse
 

From 168c9ee93207cada0d8886b18ddcea8c5f115f22 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Thu, 6 Jan 2022 08:12:27 -0800
Subject: [PATCH 39/41] Fix Windows test; potential Ubuntu flake

---
 .github/workflows/testing_wheels.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index 61716025..1b4620df 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -49,7 +49,7 @@ jobs:
       # due to package and module name conflict have to temporarily move it away to run tests
       CIBW_BEFORE_TEST: "mv {package}/qsimcirq /tmp"
       # prevent GPU tests (no GPU on GHA runners)
-      CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py -k 'not gpu' && mv /tmp/qsimcirq {package}"
+      CIBW_TEST_COMMAND: "pytest {package}/qsimcirq_tests/qsimcirq_test.py -k \"not gpu\" && mv /tmp/qsimcirq {package}"
     steps:
       - uses: actions/checkout@v2
 

From c0a864b9a79c12eabebc730ce5834a3dbc77fe48 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Wed, 26 Jan 2022 08:29:16 -0800
Subject: [PATCH 40/41] Split out qsimcirq-gpu

---
 .github/workflows/testing_wheels.yml   |  7 ++++++
 CMakeLists.txt                         |  7 +++---
 pybind_interface/decide/CMakeLists.txt | 13 +++++-----
 setup.py                               | 34 ++++++++++++++++----------
 4 files changed, 37 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/testing_wheels.yml b/.github/workflows/testing_wheels.yml
index eeb4d741..2378f7f7 100644
--- a/.github/workflows/testing_wheels.yml
+++ b/.github/workflows/testing_wheels.yml
@@ -32,12 +32,19 @@ jobs:
               arch: x86_64
               build: "cp36* cp37* cp38* cp39*"
               manylinux_image: manylinux2014
+          - os: ubuntu-20.04
+            name: manylinux2014-gpu
+            cibw:
+              arch: x86_64
+              build: "cp36*"
+              manylinux_image: manylinux2014
           - os: windows-2019
             name: win_amd64
             architecture: x64
             cibw:
               build: "cp36-win_amd64 cp37-win_amd64 cp38-win_amd64 cp39-win_amd64"
     env:
+      QSIMCIRQ_PROJECT: "${{ matrix.name == 'manylinux2014-gpu' && 'qsimcirq-gpu' || 'qsimcirq'}}"
       CIBW_BUILD: "${{ matrix.cibw.build || '*' }}"
       CIBW_ARCHS: "${{ matrix.cibw.arch || 'auto' }}"
       CIBW_MANYLINUX_X86_64_IMAGE: "${{ matrix.cibw.manylinux_image }}"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index fba9db45..d3686ba6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,14 +1,13 @@
 cmake_minimum_required(VERSION 3.11)
 
-execute_process(COMMAND which nvcc OUTPUT_VARIABLE has_nvcc)
-if(has_nvcc STREQUAL "")
-    project(qsim)
-else()
+if(ENV{QSIMCIRQ_PROJECT} STREQUAL "qsimcirq-gpu")
     project(qsim LANGUAGES CXX CUDA)
     ADD_SUBDIRECTORY(pybind_interface/cuda)
     if(DEFINED ENV{CUQUANTUM_DIR})
         ADD_SUBDIRECTORY(pybind_interface/custatevec)
     endif()
+else()
+    project(qsim)
 endif()
 
 ADD_SUBDIRECTORY(pybind_interface/sse)
diff --git a/pybind_interface/decide/CMakeLists.txt b/pybind_interface/decide/CMakeLists.txt
index 643ca9ad..2e58016f 100644
--- a/pybind_interface/decide/CMakeLists.txt
+++ b/pybind_interface/decide/CMakeLists.txt
@@ -1,10 +1,9 @@
 cmake_minimum_required(VERSION 3.11)
 
-execute_process(COMMAND which nvcc OUTPUT_VARIABLE has_nvcc)
-if(has_nvcc STREQUAL "")
-  project(qsim)
-else()
+if(ENV{QSIMCIRQ_PROJECT} STREQUAL "qsimcirq-gpu")
   project(qsim LANGUAGES CXX CUDA)
+else()
+  project(qsim)
 endif()
 
 IF (WIN32)
@@ -32,9 +31,7 @@ if(NOT pybind11_POPULATED)
   add_subdirectory(${pybind11_SOURCE_DIR} ${pybind11_BINARY_DIR})
 endif()
 
-if(has_nvcc STREQUAL "")
-  pybind11_add_module(qsim_decide decide.cpp)
-else()
+if(ENV{QSIMCIRQ_PROJECT} STREQUAL "qsimcirq-gpu")
   find_package(PythonLibs 3.6 REQUIRED)
   find_package(CUDA REQUIRED)
 
@@ -53,4 +50,6 @@ else()
         SUFFIX "${PYTHON_MODULE_EXTENSION}"
   )
   set_source_files_properties(decide.cpp PROPERTIES LANGUAGE CUDA)
+else()
+  pybind11_add_module(qsim_decide decide.cpp)
 endif()
diff --git a/setup.py b/setup.py
index 121b4dad..37734c8c 100644
--- a/setup.py
+++ b/setup.py
@@ -9,6 +9,9 @@
 from distutils.version import LooseVersion
 
 
+project_name = os.environ.get('QSIMCIRQ_PROJECT', 'qsimcirq')
+
+
 class CMakeExtension(Extension):
     def __init__(self, name, sourcedir=""):
         Extension.__init__(self, name, sources=[])
@@ -87,27 +90,32 @@ def build_extension(self, ext):
 __version__ = ""
 exec(open("qsimcirq/_version.py").read())
 
+cmake_exts = [
+    CMakeExtension("qsimcirq/qsim_avx512"),
+    CMakeExtension("qsimcirq/qsim_avx2"),
+    CMakeExtension("qsimcirq/qsim_sse"),
+    CMakeExtension("qsimcirq/qsim_basic"),
+    CMakeExtension("qsimcirq/qsim_decide"),
+]
+if 'gpu' in project_name:
+    cmake_exts += [
+        CMakeExtension("qsimcirq/qsim_cuda"),
+        CMakeExtension("qsimcirq/qsim_custatevec"),
+    ]
+
 setup(
-    name="qsimcirq",
+    name=project_name,
     version=__version__,
     url="https://github.com/quantumlib/qsim",
-    author="Vamsi Krishna Devabathini",
-    author_email="devabathini92@gmail.com",
-    python_requires=">=3.3.0",
+    author="The qsim Developers",
+    author_email="qsim-qsimh-dev@googlegroups.com",
+    python_requires=">=3.6.0",
     install_requires=requirements,
     license="Apache 2",
     description=description,
     long_description=long_description,
     long_description_content_type="text/markdown",
-    ext_modules=[
-        CMakeExtension("qsimcirq/qsim_avx512"),
-        CMakeExtension("qsimcirq/qsim_avx2"),
-        CMakeExtension("qsimcirq/qsim_sse"),
-        CMakeExtension("qsimcirq/qsim_basic"),
-        CMakeExtension("qsimcirq/qsim_cuda"),
-        CMakeExtension("qsimcirq/qsim_custatevec"),
-        CMakeExtension("qsimcirq/qsim_decide"),
-    ],
+    ext_modules=cmake_exts,
     cmdclass=dict(build_ext=CMakeBuild),
     zip_safe=False,
     packages=["qsimcirq"],

From e798db092074980419ee6ae72501bdb8a2990ae3 Mon Sep 17 00:00:00 2001
From: Orion Martin <40585662+95-martin-orion@users.noreply.github.com>
Date: Wed, 26 Jan 2022 08:33:56 -0800
Subject: [PATCH 41/41] format

---
 setup.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 37734c8c..ad921e17 100644
--- a/setup.py
+++ b/setup.py
@@ -9,7 +9,7 @@
 from distutils.version import LooseVersion
 
 
-project_name = os.environ.get('QSIMCIRQ_PROJECT', 'qsimcirq')
+project_name = os.environ.get("QSIMCIRQ_PROJECT", "qsimcirq")
 
 
 class CMakeExtension(Extension):
@@ -97,7 +97,7 @@ def build_extension(self, ext):
     CMakeExtension("qsimcirq/qsim_basic"),
     CMakeExtension("qsimcirq/qsim_decide"),
 ]
-if 'gpu' in project_name:
+if "gpu" in project_name:
     cmake_exts += [
         CMakeExtension("qsimcirq/qsim_cuda"),
         CMakeExtension("qsimcirq/qsim_custatevec"),