diff --git a/examples/get_india_data.ipynb b/examples/get_india_data.ipynb
index 5945b42..70a5472 100644
--- a/examples/get_india_data.ipynb
+++ b/examples/get_india_data.ipynb
@@ -18,7 +18,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
@@ -29,7 +29,7 @@
"from matplotlib import pyplot as plt\n",
"\n",
"sys.path.append(\"..\")\n",
- "from pvoutput.grid_search import GridSearch\n",
+ "# from pvoutput.grid_search import GridSearch\n",
"from pvoutput.pvoutput import PVOutput"
]
},
@@ -44,12 +44,12 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
- "api_key = \"3f784ff6cfa27c44436f88da0c429b410687ad14\"\n",
- "system_id = \"68732\"\n",
+ "api_key = \"c5c7c5c232f68e2f6223bd45f536b17555ad04ba\"\n",
+ "system_id = \"93186\"\n",
"date = \"20230801\"\n",
"data_service_url = \"https://pvoutput.org\"\n",
"pv = PVOutput(api_key=api_key, system_id=system_id, data_service_url=data_service_url)\n"
@@ -66,7 +66,7 @@
},
{
"cell_type": "code",
- "execution_count": 184,
+ "execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@@ -75,60 +75,502 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 33,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "change is happening\n",
- " datetime cumulative_energy_gen_Wh instantaneous_power_gen_W \\\n",
- "0 2023-08-01 05:55:00 4.0 51.0 \n",
- "1 2023-08-01 06:00:00 30.0 315.0 \n",
- "2 2023-08-01 06:05:00 81.0 613.0 \n",
- "3 2023-08-01 06:10:00 164.0 996.0 \n",
- "4 2023-08-01 06:15:00 273.0 1303.0 \n",
- ".. ... ... ... \n",
- "334 2023-08-01 19:20:00 25960.0 0.0 \n",
- "335 2023-08-01 19:25:00 25960.0 0.0 \n",
- "336 2023-08-01 19:30:00 25960.0 0.0 \n",
- "337 2023-08-01 19:35:00 25960.0 0.0 \n",
- "338 2023-08-01 19:40:00 25960.0 0.0 \n",
- "\n",
- " temperature_C voltage system_id \n",
- "0 NaN NaN 85738 \n",
- "1 NaN NaN 85738 \n",
- "2 NaN NaN 85738 \n",
- "3 NaN NaN 85738 \n",
- "4 NaN NaN 85738 \n",
- ".. ... ... ... \n",
- "334 26.0 NaN 87410 \n",
- "335 26.0 NaN 87410 \n",
- "336 26.0 NaN 87410 \n",
- "337 26.0 NaN 87410 \n",
- "338 25.0 NaN 87410 \n",
- "\n",
- "[339 rows x 6 columns]\n"
+ "getting metadata for in for 20000 to 40000\n",
+ "getting metadata for in for 40000 to 60000\n",
+ "getting metadata for in for 60000 to 80000\n",
+ "getting metadata for in for 80000 to 100000\n",
+ " system_id \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 No \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 No \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 Low \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 No \n",
+ "\n",
+ " system_DC_capacity_W \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 20150120.0 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 20180505.0 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 20230105.0 \n",
+ "\n",
+ " postcode \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 18.498465 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 22.554795 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 18.496093 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 16.474609 \n",
+ "\n",
+ " num_panels \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 73.802870 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 72.963779 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 73.820738 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 80.701103 \n",
+ "\n",
+ " panel_capacity_W_each \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 5\\n35987 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5\\n59838 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 5\\n79612 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 15\\n99833 \n",
+ "\n",
+ " num_inverters \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 2000 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 4200 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 100000 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 3090 \n",
+ "\n",
+ " inverter_capacity_W \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 NaN \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 NaN \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 NaN \n",
+ "\n",
+ " orientation \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 8 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 14 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 400 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 6 \n",
+ "\n",
+ " array_tilt_degrees \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 250 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 300 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 250 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 515 \n",
+ "\n",
+ " shade \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 1 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 1 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 1 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 1 \n",
+ "\n",
+ " install_date \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 2000-01-01 00:00:00 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5000 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 20000 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 3000 \n",
+ "\n",
+ " latitude \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 S \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 S \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 N \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 N \n",
+ "\n",
+ " longitude \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 1.0 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 28.0 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 0.0 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 0.0 \n",
+ "\n",
+ " status_interval_minutes \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 No \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 Low \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 No \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 No \n",
+ "\n",
+ " secondary_num_panels \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 20150323.0 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 20170101.0 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 20230613.0 \n",
+ "\n",
+ " secondary_panel_capacity_W_each \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 12.970980 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 18.547855 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 8.462700 \n",
+ "\n",
+ " secondary_orientation \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 80.119504 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 73.788976 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 76.930325 \n",
+ "\n",
+ " secondary_array_tilt_degrees \n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 5 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 5 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 5 \n"
]
},
{
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/pvoutput/prcoess.py:93: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n",
- " one_pv_system_status[\"time\"] = pd.to_datetime(one_pv_system_status[\"time\"]).dt.strftime(\n",
- "/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/pvoutput/prcoess.py:93: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n",
- " one_pv_system_status[\"time\"] = pd.to_datetime(one_pv_system_status[\"time\"]).dt.strftime(\n"
- ]
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " | \n",
+ " system_id | \n",
+ " system_DC_capacity_W | \n",
+ " postcode | \n",
+ " num_panels | \n",
+ " panel_capacity_W_each | \n",
+ " num_inverters | \n",
+ " inverter_capacity_W | \n",
+ " orientation | \n",
+ " array_tilt_degrees | \n",
+ " shade | \n",
+ " install_date | \n",
+ " latitude | \n",
+ " longitude | \n",
+ " status_interval_minutes | \n",
+ " secondary_num_panels | \n",
+ " secondary_panel_capacity_W_each | \n",
+ " secondary_orientation | \n",
+ " secondary_array_tilt_degrees | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " 30432 | \n",
+ " 21600 | \n",
+ " NaN | \n",
+ " 72 | \n",
+ " 300 | \n",
+ " 6 | \n",
+ " 4000 | \n",
+ " S | \n",
+ " 1.0 | \n",
+ " No | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " 5\\n35948 | \n",
+ " 400 | \n",
+ " NaN | \n",
+ " 2 | \n",
+ " 200 | \n",
+ " 1 | \n",
+ " 600 | \n",
+ " S | \n",
+ " 15.5 | \n",
+ " No | \n",
+ " 20150120.0 | \n",
+ " 18.498465 | \n",
+ " 73.802870 | \n",
+ " 5\\n35987 | \n",
+ " 2000 | \n",
+ " NaN | \n",
+ " 8 | \n",
+ " 250 | \n",
+ " 1 | \n",
+ " 2000-01-01 00:00:00 | \n",
+ " S | \n",
+ " 1.0 | \n",
+ " No | \n",
+ " 20150323.0 | \n",
+ " 12.970980 | \n",
+ " 80.119504 | \n",
+ " 5 | \n",
+ "
\n",
+ " \n",
+ " 45628 | \n",
+ " 50 | \n",
+ " NaN | \n",
+ " 1 | \n",
+ " 50 | \n",
+ " 1 | \n",
+ " 50 | \n",
+ " N | \n",
+ " NaN | \n",
+ " No | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " 5\\n46732 | \n",
+ " 5300 | \n",
+ " NaN | \n",
+ " 20 | \n",
+ " 265 | \n",
+ " 1 | \n",
+ " 6000 | \n",
+ " S | \n",
+ " 15.0 | \n",
+ " No | \n",
+ " 20180505.0 | \n",
+ " 22.554795 | \n",
+ " 72.963779 | \n",
+ " 5\\n59838 | \n",
+ " 4200 | \n",
+ " NaN | \n",
+ " 14 | \n",
+ " 300 | \n",
+ " 1 | \n",
+ " 5000 | \n",
+ " S | \n",
+ " 28.0 | \n",
+ " Low | \n",
+ " 20170101.0 | \n",
+ " 18.547855 | \n",
+ " 73.788976 | \n",
+ " 5 | \n",
+ "
\n",
+ " \n",
+ " 60294 | \n",
+ " 5120 | \n",
+ " NaN | \n",
+ " 16 | \n",
+ " 320 | \n",
+ " 1 | \n",
+ " 5120 | \n",
+ " S | \n",
+ " 1.0 | \n",
+ " Low | \n",
+ " NaN | \n",
+ " 12.980483 | \n",
+ " 77.690743 | \n",
+ " 5\\n60602 | \n",
+ " 1950 | \n",
+ " NaN | \n",
+ " 6 | \n",
+ " 325 | \n",
+ " 1 | \n",
+ " 5000 | \n",
+ " S | \n",
+ " 20.0 | \n",
+ " Low | \n",
+ " NaN | \n",
+ " 18.496093 | \n",
+ " 73.820738 | \n",
+ " 5\\n79612 | \n",
+ " 100000 | \n",
+ " NaN | \n",
+ " 400 | \n",
+ " 250 | \n",
+ " 1 | \n",
+ " 20000 | \n",
+ " N | \n",
+ " 0.0 | \n",
+ " No | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " NaN | \n",
+ " 5 | \n",
+ "
\n",
+ " \n",
+ " 81408 | \n",
+ " 2370 | \n",
+ " NaN | \n",
+ " 6 | \n",
+ " 395 | \n",
+ " 1 | \n",
+ " 3000 | \n",
+ " SE | \n",
+ " 5.0 | \n",
+ " Medium | \n",
+ " 20210101 | \n",
+ " 9.714133 | \n",
+ " 76.323051 | \n",
+ " 5\\n82026 | \n",
+ " 3080 | \n",
+ " NaN | \n",
+ " 8 | \n",
+ " 385 | \n",
+ " 1 | \n",
+ " 3000 | \n",
+ " S | \n",
+ " 0.0 | \n",
+ " No | \n",
+ " 20230105.0 | \n",
+ " 16.474609 | \n",
+ " 80.701103 | \n",
+ " 15\\n99833 | \n",
+ " 3090 | \n",
+ " NaN | \n",
+ " 6 | \n",
+ " 515 | \n",
+ " 1 | \n",
+ " 3000 | \n",
+ " N | \n",
+ " 0.0 | \n",
+ " No | \n",
+ " 20230613.0 | \n",
+ " 8.462700 | \n",
+ " 76.930325 | \n",
+ " 5 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " system_id \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 No \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 No \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 Low \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 No \n",
+ "\n",
+ " system_DC_capacity_W \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 20150120.0 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 20180505.0 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 20230105.0 \n",
+ "\n",
+ " postcode \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 18.498465 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 22.554795 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 18.496093 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 16.474609 \n",
+ "\n",
+ " num_panels \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 73.802870 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 72.963779 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 73.820738 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 80.701103 \n",
+ "\n",
+ " panel_capacity_W_each \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 5\\n35987 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5\\n59838 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 5\\n79612 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 15\\n99833 \n",
+ "\n",
+ " num_inverters \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 2000 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 4200 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 100000 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 3090 \n",
+ "\n",
+ " inverter_capacity_W \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 NaN \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 NaN \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 NaN \n",
+ "\n",
+ " orientation \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 8 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 14 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 400 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 6 \n",
+ "\n",
+ " array_tilt_degrees \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 250 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 300 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 250 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 515 \n",
+ "\n",
+ " shade \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 1 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 1 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 1 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 1 \n",
+ "\n",
+ " install_date \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 2000-01-01 00:00:00 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5000 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 20000 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 3000 \n",
+ "\n",
+ " latitude \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 S \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 S \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 N \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 N \n",
+ "\n",
+ " longitude \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 1.0 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 28.0 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 0.0 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 0.0 \n",
+ "\n",
+ " status_interval_minutes \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 No \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 Low \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 No \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 No \n",
+ "\n",
+ " secondary_num_panels \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 20150323.0 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 20170101.0 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 20230613.0 \n",
+ "\n",
+ " secondary_panel_capacity_W_each \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 12.970980 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 18.547855 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 8.462700 \n",
+ "\n",
+ " secondary_orientation \\\n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 80.119504 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 73.788976 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 76.930325 \n",
+ "\n",
+ " secondary_array_tilt_degrees \n",
+ "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 5 \n",
+ "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5 \n",
+ "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 5 \n",
+ "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 5 "
+ ]
+ },
+ "execution_count": 33,
+ "metadata": {},
+ "output_type": "execute_result"
}
],
"source": [
- "# Get output for list of sites\n",
- "sites = [\"87410\", \"85738\"]\n",
- "df = pv.get_system_status(pv_system_ids=sites, date=date, use_data_service=True)\n",
- "df.to_csv(f\"/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/examples/pv_data/pv_data.csv\", index=False)\n",
- "print(df)\n"
+ "\n",
+ "# pvoutput.org has a limit of a maximum of 20000 range for system ids\n",
+ "# here we create a range of possible system ids to query or a range in which system ids might exist\n",
+ "def create_list_start(r1, r2, r3):\n",
+ " return [number for number in range(r1, r2, r3)]\n",
+ "def create_list_end(r1, r2, r3):\n",
+ " return [number for number in range(r1, r2, r3)]\n",
+ "start_id_range = create_list_start(0, 100000, 20000)\n",
+ "end_id_range = create_list_end(20000, 120000, 20000)\n",
+ "\n",
+ "def create_df(start_id_range=start_id_range, end_id_range=end_id_range):\n",
+ " frames = []\n",
+ " i = 0\n",
+ " while i < len(start_id_range)-1 and i < len(end_id_range)-1:\n",
+ " i += 1\n",
+ " df = pv.get_metadata_for_country(country_code=\"in\", start_id_range=start_id_range[i], end_id_range=end_id_range[i], use_data_service=True)\n",
+ " frames.append(df)\n",
+ " df= pd.concat(frames)\n",
+ " print(df)\n",
+ " df.to_csv(f\"/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/examples/pv_data/pv_data_india.csv\", index=False)\n",
+ " return df\n",
+ "\n",
+ "create_df(start_id_range=start_id_range, end_id_range=end_id_range)\n",
+ "\n",
+ "\n"
]
},
{
diff --git a/pvoutput/pvoutput.py b/pvoutput/pvoutput.py
index f4b1701..0d3c2fd 100644
--- a/pvoutput/pvoutput.py
+++ b/pvoutput/pvoutput.py
@@ -70,9 +70,9 @@ def __init__(
"""
self.api_key = api_key
self.system_id = system_id
- self.rate_limit_remaining = 600
- self.rate_limit_total = 600
- self.rate_limit_reset_time = 600
+ self.rate_limit_remaining = None
+ self.rate_limit_total = None
+ self.rate_limit_reset_time = None
self.data_service_url = data_service_url
# Set from config file if None
@@ -161,7 +161,9 @@ def search(
if lat is not None and lon is not None:
api_params["ll"] = "{:f},{:f}".format(lat, lon)
- pv_systems_text = self._api_query(service="search", api_params=api_params, **kwargs)
+ pv_systems_text = self._api_query(
+ service="search", api_params=api_params, **kwargs
+ )
pv_systems = pd.read_csv(
StringIO(pv_systems_text),
@@ -277,7 +279,7 @@ def get_status(
# add timezone
if timezone is not None:
- pv_system_status = pv_system_status.tz_localize(timezone).tz_convert("UTC")
+ pv_system_status = pv_system_status.tz_convert(timezone).tz_convert("UTC")
return pv_system_status
@@ -314,7 +316,9 @@ def get_system_status(
temperature_C,
voltage,
"""
- _LOG.info(f"system_ids {pv_system_ids}: Requesting batch system status for %s", date)
+ _LOG.info(
+ f"system_ids {pv_system_ids}: Requesting batch system status for %s", date
+ )
date = date_to_pvoutput_str(date)
_check_date(date)
@@ -332,7 +336,9 @@ def get_system_status(
)
except NoStatusFound:
- _LOG.info(f"system_id {all_pv_system_id}: No status found for date %s", date)
+ _LOG.info(
+ f"system_id {all_pv_system_id}: No status found for date %s", date
+ )
pv_system_status_text = "no status found"
# each pv system is on a new line
@@ -340,7 +346,6 @@ def get_system_status(
pv_system_status = []
for pv_system_status_text in pv_systems_status_text:
-
try:
one_pv_system_status = process_system_status(
pv_system_status_text=pv_system_status_text, date=date
@@ -360,7 +365,7 @@ def get_system_status(
if timezone is not None:
pv_system_status["datetime"] = (
pd.DatetimeIndex(pv_system_status["datetime"])
- .tz_localize(timezone)
+ .tz_convert(timezone)
.tz_convert("UTC")
)
@@ -416,10 +421,17 @@ def get_batch_status(
for retry in range(max_retries):
try:
pv_system_status_text = self._api_query(
- service="getbatchstatus", api_params=api_params, use_data_service=True, **kwargs
+ service="getbatchstatus",
+ api_params=api_params,
+ use_data_service=True,
+ **kwargs,
)
except NoStatusFound:
- _LOG.info("system_id %d: No status found for date_to %s", pv_system_id, date_to)
+ _LOG.info(
+ "system_id %d: No status found for date_to %s",
+ pv_system_id,
+ date_to,
+ )
pv_system_status_text = ""
break
@@ -431,7 +443,8 @@ def get_batch_status(
time.sleep(1)
else:
_print_and_log(
- "Call get_batch_status again in a minute to see if" " results are ready."
+ "Call get_batch_status again in a minute to see if"
+ " results are ready."
)
else:
break
@@ -517,6 +530,85 @@ def get_metadata(self, pv_system_id: int, **kwargs) -> pd.Series:
pv_metadata.name = pv_system_id
return pv_metadata
+ def get_metadata_for_country(
+ self, country_code: str, start_id_range: int, end_id_range: int, **kwargs
+ ) -> pd.DataFrame:
+ """Get metadata for a single PV system.
+
+ Args:
+ country_code: str,
+ start_id_range: int,
+ end_id_range: int,
+
+ Returns:
+ pd.Dataframe. Index is:
+ system_id,
+ system_DC_capacity_W,
+ postcode,
+ num_panels,
+ panel_capacity_W_each,
+ num_inverters,
+ inverter_capacity_W,
+ orientation,
+ array_tilt_degrees,
+ shade,
+ install_date,
+ latitude,
+ longitude,
+ status_interval_minutes,
+ secondary_num_panels,
+ secondary_panel_capacity_W_each,
+ secondary_orientation,
+ secondary_array_tilt_degrees
+ """
+ pv_metadata_text = self._api_query(
+ service="getcountrysystem",
+ api_params={
+ "c": country_code, # Provide data about secondary array, if present.
+ "from": start_id_range,
+ "to": end_id_range,
+ },
+ **kwargs,
+ )
+
+ _LOG.debug(
+ f"getting metadata for {country_code} for {start_id_range} to {end_id_range}"
+ )
+ print(
+ f"getting metadata for {country_code} for {start_id_range} to {end_id_range}"
+ )
+
+ pv_metadata_for_country = pd.read_csv(
+ StringIO(pv_metadata_text),
+ lineterminator=";",
+ names=[
+ "system_id",
+ "system_DC_capacity_W",
+ "postcode",
+ "num_panels",
+ "panel_capacity_W_each",
+ "num_inverters",
+ "inverter_capacity_W",
+ "orientation",
+ "array_tilt_degrees",
+ "shade",
+ "install_date",
+ "latitude",
+ "longitude",
+ "status_interval_minutes",
+ "secondary_num_panels",
+ "secondary_panel_capacity_W_each",
+ "secondary_orientation",
+ "secondary_array_tilt_degrees",
+ ],
+ parse_dates=["install_date"],
+ nrows=1,
+ )
+ pv_metadata_for_country["system_id"] = pv_metadata_for_country["system_id"]
+ # pv_metadata_for_country["system_id"] = [start_id_range, end_id_range]
+ # pv_metadata_for_country.name = [start_id_range, end_id_range]
+ return pv_metadata_for_country
+
def get_statistic(
self,
pv_system_id: int,
@@ -596,8 +688,12 @@ def get_statistic(
else:
pv_metadata.index = [pv_system_id]
- pv_metadata["query_date_from"] = pd.Timestamp(date_from) if date_from else pd.NaT
- pv_metadata["query_date_to"] = pd.Timestamp(date_to) if date_to else pd.Timestamp.now()
+ pv_metadata["query_date_from"] = (
+ pd.Timestamp(date_from) if date_from else pd.NaT
+ )
+ pv_metadata["query_date_to"] = (
+ pd.Timestamp(date_to) if date_to else pd.Timestamp.now()
+ )
return pv_metadata
def _get_statistic_with_cache(
@@ -644,7 +740,9 @@ def _get_fresh_statistic():
return stats
try:
- stats = pd.read_hdf(store_filename, key="statistics", where="index=pv_system_id")
+ stats = pd.read_hdf(
+ store_filename, key="statistics", where="index=pv_system_id"
+ )
except (FileNotFoundError, KeyError):
return _get_fresh_statistic()
@@ -710,7 +808,9 @@ def download_multiple_systems_to_disk(
n = len(system_ids)
for i, pv_system_id in enumerate(system_ids):
_LOG.info("**********************")
- msg = "system_id {:d}: {:d} of {:d} ({:%})".format(pv_system_id, i + 1, n, (i + 1) / n)
+ msg = "system_id {:d}: {:d} of {:d} ({:%})".format(
+ pv_system_id, i + 1, n, (i + 1) / n
+ )
_LOG.info(msg)
print("\r", msg, end="", flush=True)
@@ -722,7 +822,10 @@ def download_multiple_systems_to_disk(
# How much data is actually available?
date_ranges_to_download = self._filter_date_range(
- output_filename, pv_system_id, date_ranges_to_download, min_data_availability
+ output_filename,
+ pv_system_id,
+ date_ranges_to_download,
+ min_data_availability,
)
if not date_ranges_to_download:
@@ -841,9 +944,13 @@ def _filter_date_range(
_LOG.info("system_id %d: Stats say there is no data!", system_id)
return []
- timeseries_date_range = DateRange(stats["actual_date_from"], stats["actual_date_to"])
+ timeseries_date_range = DateRange(
+ stats["actual_date_from"], stats["actual_date_to"]
+ )
- data_availability = stats["num_outputs"] / (timeseries_date_range.total_days() + 1)
+ data_availability = stats["num_outputs"] / (
+ timeseries_date_range.total_days() + 1
+ )
if data_availability < min_data_availability:
_LOG.info(
@@ -861,7 +968,11 @@ def _filter_date_range(
return new_date_ranges
def _download_multiple_using_get_batch_status(
- self, output_filename, pv_system_id, date_ranges_to_download, timezone: Optional[str] = None
+ self,
+ output_filename,
+ pv_system_id,
+ date_ranges_to_download,
+ timezone: Optional[str] = None,
):
years = merge_date_ranges_to_years(date_ranges_to_download)
dates_to = [year.end_date for year in years]
@@ -875,7 +986,11 @@ def _download_multiple_using_get_batch_status(
sort_and_de_dupe_pv_system(store, pv_system_id)
def _download_multiple_using_get_status(
- self, output_filename, pv_system_id, date_ranges_to_download, timezone: Optional[str] = None
+ self,
+ output_filename,
+ pv_system_id,
+ date_ranges_to_download,
+ timezone: Optional[str] = None,
):
for date_range in date_ranges_to_download:
dates = date_range.date_range()
@@ -904,7 +1019,9 @@ def _download_multiple_worker(
timeseries = self.get_batch_status(pv_system_id, date_to=date_to_load)
if timeseries.empty:
_LOG.info(
- "system_id %d: Got empty timeseries back for %s", pv_system_id, date_to_load
+ "system_id %d: Got empty timeseries back for %s",
+ pv_system_id,
+ date_to_load,
)
if use_get_status:
_append_missing_date_range(
@@ -924,8 +1041,8 @@ def _download_multiple_worker(
)
else:
total_rows += len(timeseries)
- _LOG.info(f'Adding timezone {timezone} to {total_rows} rows')
- timeseries = timeseries.tz_localize(timezone)
+ _LOG.info(f"Adding timezone {timezone} to {total_rows} rows")
+ timeseries = timeseries.tz_convert(timezone)
_LOG.info(
"system_id: %d: %d rows retrieved: %s to %s",
pv_system_id,
@@ -974,7 +1091,9 @@ def _api_query(
RateLimitExceeded
"""
get_response_func = (
- self._get_data_service_response if use_data_service else self._get_api_response
+ self._get_data_service_response
+ if use_data_service
+ else self._get_api_response
)
try:
@@ -986,13 +1105,16 @@ def _api_query(
try:
return self._process_api_response(response)
except RateLimitExceeded:
- msg = "PVOutput.org API rate limit exceeded!" " Rate limit will be reset at {}".format(
- self.rate_limit_reset_time
+ msg = (
+ "PVOutput.org API rate limit exceeded!"
+ " Rate limit will be reset at {}".format(self.rate_limit_reset_time)
)
_print_and_log(msg)
if wait_if_rate_limit_exceeded:
self.wait_for_rate_limit_reset()
- return self._api_query(service, api_params, wait_if_rate_limit_exceeded=False)
+ return self._api_query(
+ service, api_params, wait_if_rate_limit_exceeded=False
+ )
raise RateLimitExceeded(response, msg)
@@ -1007,16 +1129,18 @@ def _get_api_response(self, service: str, api_params: Dict) -> requests.Response
self._check_api_params()
# Create request headers
headers = {
- "X-Rate-Limit": "5",
+ "X-Rate-Limit": "1",
"X-Pvoutput-Apikey": self.api_key,
"X-Pvoutput-SystemId": self.system_id,
}
- api_url = urljoin(BASE_URL, "/data/r2/{}.jsp".format(service))
+ api_url = urljoin(BASE_URL, "service/r2/{}.jsp".format(service))
return _get_response(api_url, api_params, headers)
- def _get_data_service_response(self, service: str, api_params: Dict) -> requests.Response:
+ def _get_data_service_response(
+ self, service: str, api_params: Dict
+ ) -> requests.Response:
"""
Get the data service response from pvoutput.org
@@ -1033,7 +1157,7 @@ def _get_data_service_response(self, service: str, api_params: Dict) -> requests
api_params["key"] = self.api_key
api_params["sid"] = self.system_id
- api_url = urljoin(self.data_service_url, "/data/r2/{}.jsp".format(service))
+ api_url = urljoin(self.data_service_url, "data/r2/{}.jsp".format(service))
return _get_response(api_url, api_params, headers)
@@ -1048,8 +1172,9 @@ def _set_rate_limit_params(self, headers):
header_value = int(headers[header_key])
setattr(self, param_name, header_value)
- self.rate_limit_reset_time = pd.Timestamp.utcfromtimestamp(self.rate_limit_reset_time)
- print("change is happening")
+ self.rate_limit_reset_time = pd.Timestamp.utcfromtimestamp(
+ self.rate_limit_reset_time
+ )
self.rate_limit_reset_time = self.rate_limit_reset_time.tz_convert("utc")
_LOG.debug("%s", self.rate_limit_info())
@@ -1123,7 +1248,9 @@ def wait_for_rate_limit_reset(self, do_sleeping: bool = True) -> int:
# retry_time_local = retry_time_utc.tz_convert(tz=datetime.now(tzlocal()).tzname())
retry_time_local = retry_time_utc
_print_and_log(
- "Waiting {:.0f} seconds. Will retry at {} UTC".format(secs_to_wait, retry_time_local)
+ "Waiting {:.0f} seconds. Will retry at {} UTC".format(
+ secs_to_wait, retry_time_local
+ )
)
if do_sleeping:
time.sleep(secs_to_wait)
@@ -1192,9 +1319,12 @@ def check_pv_system_status(pv_system_status: pd.DataFrame, requested_date: date)
def _append_missing_date_range(
- output_filename, pv_system_id, missing_start_date, missing_end_date, datetime_of_api_request
+ output_filename,
+ pv_system_id,
+ missing_start_date,
+ missing_end_date,
+ datetime_of_api_request,
):
-
data = {
"missing_start_date_PV_localtime": pd.Timestamp(missing_start_date),
"missing_end_date_PV_localtime": pd.Timestamp(missing_end_date),
@@ -1209,14 +1339,25 @@ def _append_missing_date_range(
missing_end_date,
)
with pd.HDFStore(output_filename, mode="a", complevel=9) as store:
- store.append(key="missing_dates", value=new_missing_date_range, data_columns=True)
+ store.append(
+ key="missing_dates", value=new_missing_date_range, data_columns=True
+ )
-def _record_gaps(output_filename, pv_system_id, date_to, timeseries, datetime_of_api_request):
+def _record_gaps(
+ output_filename, pv_system_id, date_to, timeseries, datetime_of_api_request
+):
dates_of_data = (
- timeseries["instantaneous_power_gen_W"].dropna().resample("D").mean().dropna().index.date
+ timeseries["instantaneous_power_gen_W"]
+ .dropna()
+ .resample("D")
+ .mean()
+ .dropna()
+ .index.date
)
- dates_requested = pd.date_range(date_to - timedelta(days=365), date_to, freq="D").date
+ dates_requested = pd.date_range(
+ date_to - timedelta(days=365), date_to, freq="D"
+ ).date
missing_dates = set(dates_requested) - set(dates_of_data)
missing_date_ranges = _convert_consecutive_dates_to_date_ranges(list(missing_dates))
_LOG.info(
@@ -1259,7 +1400,10 @@ def _convert_consecutive_dates_to_date_ranges(missing_dates):
end_date = missing_dates[-1]
new_missing.append(
- {"missing_start_date_PV_localtime": start_date, "missing_end_date_PV_localtime": end_date}
+ {
+ "missing_start_date_PV_localtime": start_date,
+ "missing_end_date_PV_localtime": end_date,
+ }
)
return pd.DataFrame(new_missing)