Updates

ProcessPoolExecutor.ipynb

@@ -0,0 +1,183 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "8e4e62dd-fc46-48a2-9f28-b735a0ae9421",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import multiprocessing as mp\n",
+    "mp.set_start_method(\"fork\")\n",
+    "\n",
+    "from concurrent.futures import ProcessPoolExecutor, as_completed\n",
+    "import numpy as np\n",
+    "import time\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "19004296-83ee-4e34-a293-7b6b50382f2f",
+   "metadata": {},
+   "source": [
+    "### Create a simple function to process our parameters"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "8bbb3f6f-441e-449f-a0e6-868307c92870",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def process_parameters(params):\n",
+    "    return params + 1\n",
+    "\n",
+    "# Parameters right now are just a list\n",
+    "parameters = np.arange(0, 32)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5205cbb3-47d8-4c6a-8aa0-084cdcc45bc6",
+   "metadata": {},
+   "source": [
+    "### Create an executor to process all our work with number of workers equal to number of gpus"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1fa9d72a-e541-4ee4-84b6-021c7d379847",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "executor = ProcessPoolExecutor(max_workers=2)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d17f0acd-a3fc-45db-8fc8-868b9058bc91",
+   "metadata": {},
+   "source": [
+    "### Start and run a set of parameters on multiple processes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "52dabf62-9e87-48e6-bd1e-8f918fa34cb9",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "CPU times: user 3.77 ms, sys: 7.6 ms, total: 11.4 ms\n",
+      "Wall time: 11.1 ms\n"
+     ]
+    }
+   ],
+   "source": [
+    "%%time\n",
+    "futures = []\n",
+    "for pars in parameters:\n",
+    "    futures.append(executor.submit(process_parameters, pars))\n",
+    "\n",
+    "all_results = []\n",
+    "for f in futures:\n",
+    "    all_results.append(f.result())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "305410d7-227b-478f-946c-b6e2bfc33de2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_results[0:10]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "02c6b400-b39b-4038-9356-725c672758b3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# All the results are returned sorted\n",
+    "sorted(all_results) == all_results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "777fb518-c21a-4e5d-982b-3745ac53231d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "futures = []\n",
+    "for pars in parameters:\n",
+    "    futures.append(executor.submit(process_parameters, pars))\n",
+    "\n",
+    "all_results = []\n",
+    "# Using as completed will start a new processes as soon as the last has ended\n",
+    "# Out of order\n",
+    "for f in as_completed(futures):\n",
+    "    all_results.append(f.result())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "27529844-7c46-47ee-93d7-35ce6a86ff56",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "all_results[0:10]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fbda3584-863c-4c16-9d6f-a942ea9822a4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Results not ga\n",
+    "sorted(all_results) == all_results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "66a57619-0efb-4db2-b3f7-a15d0cca5ab6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}