Skip to content

Commit

Permalink
redshift DB example
Browse files Browse the repository at this point in the history
  • Loading branch information
@pluflou
pluflou committed Feb 12, 2024
1 parent 5284398 commit 7c726a5
Showing 1 changed file with 299 additions and 0 deletions.
299 changes: 299 additions & 0 deletions notebooks/examples/redshift_database_example.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,299 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "cc12497b-b135-4234-be5f-c9b7d5b34ec7",
"metadata": {
"tags": []
},
"source": [
"# Query SunPower datasets \n",
"\n",
"Note that you need to request an API key by registering at https://pvdb.slacgismo.org and emailing [email protected] with your information and use case."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "65b1d0ac-ad2d-4046-a535-c5ce9c577a1e",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from solardatatools.dataio import load_redshift_data\n",
"from solardatatools.data_handler import DataHandler\n",
"from solardatatools.time_axis_manipulation import make_time_series"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "5c6161aa-34d4-4f7e-aa1b-1a023d40ddd6",
"metadata": {},
"outputs": [],
"source": [
"query = {\n",
" 'siteid': 'TABJC1027159', #'TAAI01129193',\n",
" 'api_key': os.environ.get('REDSHIFT_API_KEY'),\n",
" 'sensor': 0\n",
"}\n",
"\n",
"df = load_redshift_data(**query)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "ce687cb2-1128-40a4-bd39-a3318d5b4f83",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>site</th>\n",
" <th>meas_name</th>\n",
" <th>ts</th>\n",
" <th>sensor</th>\n",
" <th>meas_val_f</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>TABJC1027159</td>\n",
" <td>ac_power</td>\n",
" <td>2016-03-28 20:40:00</td>\n",
" <td>1913101452_SMA-SB-5000TL-US-22</td>\n",
" <td>1.1394</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>TABJC1027159</td>\n",
" <td>ac_power</td>\n",
" <td>2016-03-28 20:45:00</td>\n",
" <td>1913101452_SMA-SB-5000TL-US-22</td>\n",
" <td>1.4464</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>TABJC1027159</td>\n",
" <td>ac_power</td>\n",
" <td>2016-03-28 20:50:00</td>\n",
" <td>1913101452_SMA-SB-5000TL-US-22</td>\n",
" <td>1.1930</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>TABJC1027159</td>\n",
" <td>ac_power</td>\n",
" <td>2016-03-28 20:55:00</td>\n",
" <td>1913101452_SMA-SB-5000TL-US-22</td>\n",
" <td>2.1952</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>TABJC1027159</td>\n",
" <td>ac_power</td>\n",
" <td>2016-03-28 21:00:00</td>\n",
" <td>1913101452_SMA-SB-5000TL-US-22</td>\n",
" <td>1.4514</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" site meas_name ts \\\n",
"0 TABJC1027159 ac_power 2016-03-28 20:40:00 \n",
"1 TABJC1027159 ac_power 2016-03-28 20:45:00 \n",
"2 TABJC1027159 ac_power 2016-03-28 20:50:00 \n",
"3 TABJC1027159 ac_power 2016-03-28 20:55:00 \n",
"4 TABJC1027159 ac_power 2016-03-28 21:00:00 \n",
"\n",
" sensor meas_val_f \n",
"0 1913101452_SMA-SB-5000TL-US-22 1.1394 \n",
"1 1913101452_SMA-SB-5000TL-US-22 1.4464 \n",
"2 1913101452_SMA-SB-5000TL-US-22 1.1930 \n",
"3 1913101452_SMA-SB-5000TL-US-22 2.1952 \n",
"4 1913101452_SMA-SB-5000TL-US-22 1.4514 "
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.head()"
]
},
{
"cell_type": "markdown",
"id": "aaa3ac8d-0cd3-4b55-8e12-7d4d37676fd9",
"metadata": {},
"source": [
"# Create DataHandler"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "b3c10b3a-881a-4694-abea-3be877845ae1",
"metadata": {},
"outputs": [],
"source": [
"dh = DataHandler(df, convert_to_ts=True)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "dae9926c-0896-4e06-91e9-0416161e4f9e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"total time: 22.85 seconds\n",
"--------------------------------\n",
"Breakdown\n",
"--------------------------------\n",
"Preprocessing 6.68s\n",
"Cleaning 0.35s\n",
"Filtering/Summarizing 15.82s\n",
" Data quality 0.22s\n",
" Clear day detect 0.40s\n",
" Clipping detect 7.37s\n",
" Capacity change detect 7.83s\n",
"\n",
"\n",
"-----------------\n",
"DATA SET REPORT\n",
"-----------------\n",
"length 3.23 years\n",
"capacity estimate 3.79 kW\n",
"data sampling 5 minutes\n",
"quality score 0.96\n",
"clearness score 0.52\n",
"inverter clipping False\n",
"clipped fraction 0.01\n",
"capacity changes True\n",
"data quality warning True\n",
"time shift errors False\n",
"time zone errors False\n",
" \n"
]
}
],
"source": [
"dh.run_pipeline()\n",
"dh.report()"
]
},
{
"cell_type": "markdown",
"id": "bcdae3b1-b1f2-4487-ab1d-c6758caeb6ad",
"metadata": {},
"source": [
"### or manually adjust the conversion to timeseries"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "cdbac2e1-67d4-482b-83e3-588b0036b611",
"metadata": {},
"outputs": [],
"source": [
"df, _ = make_time_series(df)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "ea2c2d15-8624-40a9-9cb4-2309ef83f8e6",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"total time: 23.33 seconds\n",
"--------------------------------\n",
"Breakdown\n",
"--------------------------------\n",
"Preprocessing 6.73s\n",
"Cleaning 0.40s\n",
"Filtering/Summarizing 16.20s\n",
" Data quality 0.23s\n",
" Clear day detect 0.42s\n",
" Clipping detect 6.90s\n",
" Capacity change detect 8.66s\n",
"\n",
"\n",
"-----------------\n",
"DATA SET REPORT\n",
"-----------------\n",
"length 3.23 years\n",
"capacity estimate 3.79 kW\n",
"data sampling 5 minutes\n",
"quality score 0.96\n",
"clearness score 0.52\n",
"inverter clipping False\n",
"clipped fraction 0.01\n",
"capacity changes True\n",
"data quality warning True\n",
"time shift errors False\n",
"time zone errors False\n",
" \n"
]
}
],
"source": [
"dh = DataHandler(df)#, convert_to_ts=True)\n",
"dh.run_pipeline()\n",
"dh.report()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

0 comments on commit 7c726a5

Please sign in to comment.