diff --git a/notebooks/examples/redshift_database_example.ipynb b/notebooks/examples/redshift_database_example.ipynb new file mode 100644 index 00000000..1a0f5080 --- /dev/null +++ b/notebooks/examples/redshift_database_example.ipynb @@ -0,0 +1,299 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cc12497b-b135-4234-be5f-c9b7d5b34ec7", + "metadata": { + "tags": [] + }, + "source": [ + "# Query SunPower datasets \n", + "\n", + "Note that you need to request an API key by registering at https://pvdb.slacgismo.org and emailing slacgismotutorials@gmail.com with your information and use case." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "65b1d0ac-ad2d-4046-a535-c5ce9c577a1e", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from solardatatools.dataio import load_redshift_data\n", + "from solardatatools.data_handler import DataHandler\n", + "from solardatatools.time_axis_manipulation import make_time_series" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "5c6161aa-34d4-4f7e-aa1b-1a023d40ddd6", + "metadata": {}, + "outputs": [], + "source": [ + "query = {\n", + " 'siteid': 'TABJC1027159', #'TAAI01129193',\n", + " 'api_key': os.environ.get('REDSHIFT_API_KEY'),\n", + " 'sensor': 0\n", + "}\n", + "\n", + "df = load_redshift_data(**query)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "ce687cb2-1128-40a4-bd39-a3318d5b4f83", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sitemeas_nametssensormeas_val_f
0TABJC1027159ac_power2016-03-28 20:40:001913101452_SMA-SB-5000TL-US-221.1394
1TABJC1027159ac_power2016-03-28 20:45:001913101452_SMA-SB-5000TL-US-221.4464
2TABJC1027159ac_power2016-03-28 20:50:001913101452_SMA-SB-5000TL-US-221.1930
3TABJC1027159ac_power2016-03-28 20:55:001913101452_SMA-SB-5000TL-US-222.1952
4TABJC1027159ac_power2016-03-28 21:00:001913101452_SMA-SB-5000TL-US-221.4514
\n", + "
" + ], + "text/plain": [ + " site meas_name ts \\\n", + "0 TABJC1027159 ac_power 2016-03-28 20:40:00 \n", + "1 TABJC1027159 ac_power 2016-03-28 20:45:00 \n", + "2 TABJC1027159 ac_power 2016-03-28 20:50:00 \n", + "3 TABJC1027159 ac_power 2016-03-28 20:55:00 \n", + "4 TABJC1027159 ac_power 2016-03-28 21:00:00 \n", + "\n", + " sensor meas_val_f \n", + "0 1913101452_SMA-SB-5000TL-US-22 1.1394 \n", + "1 1913101452_SMA-SB-5000TL-US-22 1.4464 \n", + "2 1913101452_SMA-SB-5000TL-US-22 1.1930 \n", + "3 1913101452_SMA-SB-5000TL-US-22 2.1952 \n", + "4 1913101452_SMA-SB-5000TL-US-22 1.4514 " + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "markdown", + "id": "aaa3ac8d-0cd3-4b55-8e12-7d4d37676fd9", + "metadata": {}, + "source": [ + "# Create DataHandler" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b3c10b3a-881a-4694-abea-3be877845ae1", + "metadata": {}, + "outputs": [], + "source": [ + "dh = DataHandler(df, convert_to_ts=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "dae9926c-0896-4e06-91e9-0416161e4f9e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total time: 22.85 seconds\n", + "--------------------------------\n", + "Breakdown\n", + "--------------------------------\n", + "Preprocessing 6.68s\n", + "Cleaning 0.35s\n", + "Filtering/Summarizing 15.82s\n", + " Data quality 0.22s\n", + " Clear day detect 0.40s\n", + " Clipping detect 7.37s\n", + " Capacity change detect 7.83s\n", + "\n", + "\n", + "-----------------\n", + "DATA SET REPORT\n", + "-----------------\n", + "length 3.23 years\n", + "capacity estimate 3.79 kW\n", + "data sampling 5 minutes\n", + "quality score 0.96\n", + "clearness score 0.52\n", + "inverter clipping False\n", + "clipped fraction 0.01\n", + "capacity changes True\n", + "data quality warning True\n", + "time shift errors False\n", + "time zone errors False\n", + " \n" + ] + } + ], + "source": [ + "dh.run_pipeline()\n", + "dh.report()" + ] + }, + { + "cell_type": "markdown", + "id": "bcdae3b1-b1f2-4487-ab1d-c6758caeb6ad", + "metadata": {}, + "source": [ + "### or manually adjust the conversion to timeseries" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "cdbac2e1-67d4-482b-83e3-588b0036b611", + "metadata": {}, + "outputs": [], + "source": [ + "df, _ = make_time_series(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ea2c2d15-8624-40a9-9cb4-2309ef83f8e6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total time: 23.33 seconds\n", + "--------------------------------\n", + "Breakdown\n", + "--------------------------------\n", + "Preprocessing 6.73s\n", + "Cleaning 0.40s\n", + "Filtering/Summarizing 16.20s\n", + " Data quality 0.23s\n", + " Clear day detect 0.42s\n", + " Clipping detect 6.90s\n", + " Capacity change detect 8.66s\n", + "\n", + "\n", + "-----------------\n", + "DATA SET REPORT\n", + "-----------------\n", + "length 3.23 years\n", + "capacity estimate 3.79 kW\n", + "data sampling 5 minutes\n", + "quality score 0.96\n", + "clearness score 0.52\n", + "inverter clipping False\n", + "clipped fraction 0.01\n", + "capacity changes True\n", + "data quality warning True\n", + "time shift errors False\n", + "time zone errors False\n", + " \n" + ] + } + ], + "source": [ + "dh = DataHandler(df)#, convert_to_ts=True)\n", + "dh.run_pipeline()\n", + "dh.report()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}