Merge new Dask SDT runner tool

.gitignore

@@ -1,4 +1,3 @@
-
 # Created by https://www.gitignore.io/api/data,macos,python,pycharm,database,sublimetext,jupyternotebook
 # Edit at https://www.gitignore.io/?templates=data,macos,python,pycharm,database,sublimetext,jupyternotebook
 
@@ -321,3 +320,5 @@ tags
 [._]*.un~
 
 # End of https://www.toptal.com/developers/gitignore/api/vim
+/sdt_dask/results
+

conda_recipe/meta.yaml

@@ -38,6 +38,12 @@ requirements:
     - qss
     - tqdm
     - spcqe
+    - dask
+    - distributed
+    - dask-cloudprovider
+    - graphviz
+    - bokeh
+    - boto3
 
 test:
   imports:

notebooks/examples/redshift_database_example.ipynb

@@ -0,0 +1,299 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "cc12497b-b135-4234-be5f-c9b7d5b34ec7",
+   "metadata": {
+    "tags": []
+   },
+   "source": [
+    "# Query SunPower datasets \n",
+    "\n",
+    "Note that you need to request an API key by registering at https://pvdb.slacgismo.org and emailing [email protected] with your information and use case."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "65b1d0ac-ad2d-4046-a535-c5ce9c577a1e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from solardatatools.dataio import load_redshift_data\n",
+    "from solardatatools.data_handler import DataHandler\n",
+    "from solardatatools.time_axis_manipulation import make_time_series"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "5c6161aa-34d4-4f7e-aa1b-1a023d40ddd6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = {\n",
+    "    'siteid': 'TABJC1027159', #'TAAI01129193',\n",
+    "    'api_key': os.environ.get('REDSHIFT_API_KEY'),\n",
+    "    'sensor': 0\n",
+    "}\n",
+    "\n",
+    "df = load_redshift_data(**query)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "ce687cb2-1128-40a4-bd39-a3318d5b4f83",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>site</th>\n",
+       "      <th>meas_name</th>\n",
+       "      <th>ts</th>\n",
+       "      <th>sensor</th>\n",
+       "      <th>meas_val_f</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>TABJC1027159</td>\n",
+       "      <td>ac_power</td>\n",
+       "      <td>2016-03-28 20:40:00</td>\n",
+       "      <td>1913101452_SMA-SB-5000TL-US-22</td>\n",
+       "      <td>1.1394</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>TABJC1027159</td>\n",
+       "      <td>ac_power</td>\n",
+       "      <td>2016-03-28 20:45:00</td>\n",
+       "      <td>1913101452_SMA-SB-5000TL-US-22</td>\n",
+       "      <td>1.4464</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>TABJC1027159</td>\n",
+       "      <td>ac_power</td>\n",
+       "      <td>2016-03-28 20:50:00</td>\n",
+       "      <td>1913101452_SMA-SB-5000TL-US-22</td>\n",
+       "      <td>1.1930</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>TABJC1027159</td>\n",
+       "      <td>ac_power</td>\n",
+       "      <td>2016-03-28 20:55:00</td>\n",
+       "      <td>1913101452_SMA-SB-5000TL-US-22</td>\n",
+       "      <td>2.1952</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>TABJC1027159</td>\n",
+       "      <td>ac_power</td>\n",
+       "      <td>2016-03-28 21:00:00</td>\n",
+       "      <td>1913101452_SMA-SB-5000TL-US-22</td>\n",
+       "      <td>1.4514</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           site meas_name                   ts  \\\n",
+       "0  TABJC1027159  ac_power  2016-03-28 20:40:00   \n",
+       "1  TABJC1027159  ac_power  2016-03-28 20:45:00   \n",
+       "2  TABJC1027159  ac_power  2016-03-28 20:50:00   \n",
+       "3  TABJC1027159  ac_power  2016-03-28 20:55:00   \n",
+       "4  TABJC1027159  ac_power  2016-03-28 21:00:00   \n",
+       "\n",
+       "                           sensor  meas_val_f  \n",
+       "0  1913101452_SMA-SB-5000TL-US-22      1.1394  \n",
+       "1  1913101452_SMA-SB-5000TL-US-22      1.4464  \n",
+       "2  1913101452_SMA-SB-5000TL-US-22      1.1930  \n",
+       "3  1913101452_SMA-SB-5000TL-US-22      2.1952  \n",
+       "4  1913101452_SMA-SB-5000TL-US-22      1.4514  "
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "aaa3ac8d-0cd3-4b55-8e12-7d4d37676fd9",
+   "metadata": {},
+   "source": [
+    "# Create DataHandler"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "b3c10b3a-881a-4694-abea-3be877845ae1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dh = DataHandler(df, convert_to_ts=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "dae9926c-0896-4e06-91e9-0416161e4f9e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "total time: 22.85 seconds\n",
+      "--------------------------------\n",
+      "Breakdown\n",
+      "--------------------------------\n",
+      "Preprocessing              6.68s\n",
+      "Cleaning                   0.35s\n",
+      "Filtering/Summarizing      15.82s\n",
+      "    Data quality           0.22s\n",
+      "    Clear day detect       0.40s\n",
+      "    Clipping detect        7.37s\n",
+      "    Capacity change detect 7.83s\n",
+      "\n",
+      "\n",
+      "-----------------\n",
+      "DATA SET REPORT\n",
+      "-----------------\n",
+      "length               3.23 years\n",
+      "capacity estimate    3.79 kW\n",
+      "data sampling        5 minutes\n",
+      "quality score        0.96\n",
+      "clearness score      0.52\n",
+      "inverter clipping    False\n",
+      "clipped fraction     0.01\n",
+      "capacity changes     True\n",
+      "data quality warning True\n",
+      "time shift errors    False\n",
+      "time zone errors     False\n",
+      "            \n"
+     ]
+    }
+   ],
+   "source": [
+    "dh.run_pipeline()\n",
+    "dh.report()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bcdae3b1-b1f2-4487-ab1d-c6758caeb6ad",
+   "metadata": {},
+   "source": [
+    "### or manually adjust the conversion to timeseries"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "cdbac2e1-67d4-482b-83e3-588b0036b611",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df, _ = make_time_series(df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "ea2c2d15-8624-40a9-9cb4-2309ef83f8e6",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "total time: 23.33 seconds\n",
+      "--------------------------------\n",
+      "Breakdown\n",
+      "--------------------------------\n",
+      "Preprocessing              6.73s\n",
+      "Cleaning                   0.40s\n",
+      "Filtering/Summarizing      16.20s\n",
+      "    Data quality           0.23s\n",
+      "    Clear day detect       0.42s\n",
+      "    Clipping detect        6.90s\n",
+      "    Capacity change detect 8.66s\n",
+      "\n",
+      "\n",
+      "-----------------\n",
+      "DATA SET REPORT\n",
+      "-----------------\n",
+      "length               3.23 years\n",
+      "capacity estimate    3.79 kW\n",
+      "data sampling        5 minutes\n",
+      "quality score        0.96\n",
+      "clearness score      0.52\n",
+      "inverter clipping    False\n",
+      "clipped fraction     0.01\n",
+      "capacity changes     True\n",
+      "data quality warning True\n",
+      "time shift errors    False\n",
+      "time zone errors     False\n",
+      "            \n"
+     ]
+    }
+   ],
+   "source": [
+    "dh = DataHandler(df)#, convert_to_ts=True)\n",
+    "dh.run_pipeline()\n",
+    "dh.report()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pyproject.toml

@@ -34,7 +34,8 @@ dependencies = [
     "clarabel",
     "qss",
     "tqdm",
-    "spcqe"
+    "spcqe",
+    "boto3"
 ]
 
 classifiers = [
@@ -55,7 +56,7 @@ dynamic = ["version"]
 [tool.setuptools_scm]
 
 [tool.setuptools.packages.find]
-include = ["solardatatools*",  "pvsystemprofiler*", "statistical_clear_sky*"]
+include = ["solardatatools*", "pvsystemprofiler*", "statistical_clear_sky*", "sdt_dask*"]
 
 [project.optional-dependencies]
 docs = [
@@ -66,6 +67,14 @@ docs = [
 mosek = [
     "mosek"
 ]
+dask = [
+    "numpy==2.0", # to match provided sdt docker image
+    "dask==2024.5.2", # to match provided sdt docker image
+    "distributed==2024.5.2", # to match provided sdt docker image
+    "dask-cloudprovider[all]==2022.10.0",
+    "graphviz", # for local task graph visualization
+    "bokeh" # for local task graph visualization
+    ]
 
 [project.urls]
 Homepage = "https://github.com/slacgismo/solar-data-tools"

requirements.txt

@@ -16,3 +16,10 @@ clarabel
 qss
 tqdm
 spcqe
+# Packages below this line are for the SDT Dask tool feature
+dask
+distributed
+dask-cloudprovider[all]
+graphviz
+bokeh
+boto3