diff --git a/examples/get_india_data.ipynb b/examples/get_india_data.ipynb index 5945b42..70a5472 100644 --- a/examples/get_india_data.ipynb +++ b/examples/get_india_data.ipynb @@ -18,7 +18,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -29,7 +29,7 @@ "from matplotlib import pyplot as plt\n", "\n", "sys.path.append(\"..\")\n", - "from pvoutput.grid_search import GridSearch\n", + "# from pvoutput.grid_search import GridSearch\n", "from pvoutput.pvoutput import PVOutput" ] }, @@ -44,12 +44,12 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ - "api_key = \"3f784ff6cfa27c44436f88da0c429b410687ad14\"\n", - "system_id = \"68732\"\n", + "api_key = \"c5c7c5c232f68e2f6223bd45f536b17555ad04ba\"\n", + "system_id = \"93186\"\n", "date = \"20230801\"\n", "data_service_url = \"https://pvoutput.org\"\n", "pv = PVOutput(api_key=api_key, system_id=system_id, data_service_url=data_service_url)\n" @@ -66,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 184, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -75,60 +75,502 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 33, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "change is happening\n", - " datetime cumulative_energy_gen_Wh instantaneous_power_gen_W \\\n", - "0 2023-08-01 05:55:00 4.0 51.0 \n", - "1 2023-08-01 06:00:00 30.0 315.0 \n", - "2 2023-08-01 06:05:00 81.0 613.0 \n", - "3 2023-08-01 06:10:00 164.0 996.0 \n", - "4 2023-08-01 06:15:00 273.0 1303.0 \n", - ".. ... ... ... \n", - "334 2023-08-01 19:20:00 25960.0 0.0 \n", - "335 2023-08-01 19:25:00 25960.0 0.0 \n", - "336 2023-08-01 19:30:00 25960.0 0.0 \n", - "337 2023-08-01 19:35:00 25960.0 0.0 \n", - "338 2023-08-01 19:40:00 25960.0 0.0 \n", - "\n", - " temperature_C voltage system_id \n", - "0 NaN NaN 85738 \n", - "1 NaN NaN 85738 \n", - "2 NaN NaN 85738 \n", - "3 NaN NaN 85738 \n", - "4 NaN NaN 85738 \n", - ".. ... ... ... \n", - "334 26.0 NaN 87410 \n", - "335 26.0 NaN 87410 \n", - "336 26.0 NaN 87410 \n", - "337 26.0 NaN 87410 \n", - "338 25.0 NaN 87410 \n", - "\n", - "[339 rows x 6 columns]\n" + "getting metadata for in for 20000 to 40000\n", + "getting metadata for in for 40000 to 60000\n", + "getting metadata for in for 60000 to 80000\n", + "getting metadata for in for 80000 to 100000\n", + " system_id \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 No \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 No \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 Low \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 No \n", + "\n", + " system_DC_capacity_W \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 20150120.0 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 20180505.0 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 20230105.0 \n", + "\n", + " postcode \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 18.498465 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 22.554795 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 18.496093 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 16.474609 \n", + "\n", + " num_panels \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 73.802870 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 72.963779 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 73.820738 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 80.701103 \n", + "\n", + " panel_capacity_W_each \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 5\\n35987 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5\\n59838 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 5\\n79612 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 15\\n99833 \n", + "\n", + " num_inverters \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 2000 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 4200 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 100000 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 3090 \n", + "\n", + " inverter_capacity_W \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 NaN \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 NaN \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 NaN \n", + "\n", + " orientation \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 8 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 14 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 400 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 6 \n", + "\n", + " array_tilt_degrees \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 250 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 300 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 250 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 515 \n", + "\n", + " shade \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 1 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 1 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 1 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 1 \n", + "\n", + " install_date \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 2000-01-01 00:00:00 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5000 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 20000 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 3000 \n", + "\n", + " latitude \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 S \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 S \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 N \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 N \n", + "\n", + " longitude \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 1.0 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 28.0 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 0.0 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 0.0 \n", + "\n", + " status_interval_minutes \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 No \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 Low \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 No \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 No \n", + "\n", + " secondary_num_panels \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 20150323.0 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 20170101.0 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 20230613.0 \n", + "\n", + " secondary_panel_capacity_W_each \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 12.970980 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 18.547855 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 8.462700 \n", + "\n", + " secondary_orientation \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 80.119504 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 73.788976 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 76.930325 \n", + "\n", + " secondary_array_tilt_degrees \n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 5 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 5 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 5 \n" ] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/pvoutput/prcoess.py:93: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n", - " one_pv_system_status[\"time\"] = pd.to_datetime(one_pv_system_status[\"time\"]).dt.strftime(\n", - "/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/pvoutput/prcoess.py:93: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n", - " one_pv_system_status[\"time\"] = pd.to_datetime(one_pv_system_status[\"time\"]).dt.strftime(\n" - ] + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
system_idsystem_DC_capacity_Wpostcodenum_panelspanel_capacity_W_eachnum_invertersinverter_capacity_Worientationarray_tilt_degreesshadeinstall_datelatitudelongitudestatus_interval_minutessecondary_num_panelssecondary_panel_capacity_W_eachsecondary_orientationsecondary_array_tilt_degrees
3043221600NaN7230064000S1.0NoNaNNaNNaN5\\n35948400NaN22001600S15.5No20150120.018.49846573.8028705\\n359872000NaN825012000-01-01 00:00:00S1.0No20150323.012.97098080.1195045
4562850NaN150150NNaNNoNaNNaNNaN5\\n467325300NaN2026516000S15.0No20180505.022.55479572.9637795\\n598384200NaN1430015000S28.0Low20170101.018.54785573.7889765
602945120NaN1632015120S1.0LowNaN12.98048377.6907435\\n606021950NaN632515000S20.0LowNaN18.49609373.8207385\\n79612100000NaN400250120000N0.0NoNaNNaNNaN5
814082370NaN639513000SE5.0Medium202101019.71413376.3230515\\n820263080NaN838513000S0.0No20230105.016.47460980.70110315\\n998333090NaN651513000N0.0No20230613.08.46270076.9303255
\n", + "
" + ], + "text/plain": [ + " system_id \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 No \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 No \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 Low \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 No \n", + "\n", + " system_DC_capacity_W \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 20150120.0 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 20180505.0 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 20230105.0 \n", + "\n", + " postcode \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 18.498465 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 22.554795 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 18.496093 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 16.474609 \n", + "\n", + " num_panels \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 73.802870 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 72.963779 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 73.820738 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 80.701103 \n", + "\n", + " panel_capacity_W_each \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 5\\n35987 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5\\n59838 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 5\\n79612 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 15\\n99833 \n", + "\n", + " num_inverters \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 2000 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 4200 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 100000 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 3090 \n", + "\n", + " inverter_capacity_W \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 NaN \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 NaN \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 NaN \n", + "\n", + " orientation \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 8 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 14 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 400 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 6 \n", + "\n", + " array_tilt_degrees \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 250 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 300 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 250 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 515 \n", + "\n", + " shade \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 1 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 1 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 1 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 1 \n", + "\n", + " install_date \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 2000-01-01 00:00:00 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5000 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 20000 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 3000 \n", + "\n", + " latitude \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 S \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 S \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 N \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 N \n", + "\n", + " longitude \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 1.0 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 28.0 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 0.0 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 0.0 \n", + "\n", + " status_interval_minutes \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 No \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 Low \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 No \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 No \n", + "\n", + " secondary_num_panels \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 20150323.0 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 20170101.0 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 20230613.0 \n", + "\n", + " secondary_panel_capacity_W_each \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 12.970980 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 18.547855 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 8.462700 \n", + "\n", + " secondary_orientation \\\n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 80.119504 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 73.788976 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 NaN \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 76.930325 \n", + "\n", + " secondary_array_tilt_degrees \n", + "30432 21600 NaN 72 300 6 4000 S 1.0 No NaN NaN NaN 5\\n35948 400 NaN 2 200 1 600 S 15.5 5 \n", + "45628 50 NaN 1 50 1 50 N NaN No NaN NaN NaN 5\\n46732 5300 NaN 20 265 1 6000 S 15.0 5 \n", + "60294 5120 NaN 16 320 1 5120 S 1.0 Low NaN 12.980483 77.690743 5\\n60602 1950 NaN 6 325 1 5000 S 20.0 5 \n", + "81408 2370 NaN 6 395 1 3000 SE 5.0 Medium 20210101 9.714133 76.323051 5\\n82026 3080 NaN 8 385 1 3000 S 0.0 5 " + ] + }, + "execution_count": 33, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "# Get output for list of sites\n", - "sites = [\"87410\", \"85738\"]\n", - "df = pv.get_system_status(pv_system_ids=sites, date=date, use_data_service=True)\n", - "df.to_csv(f\"/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/examples/pv_data/pv_data.csv\", index=False)\n", - "print(df)\n" + "\n", + "# pvoutput.org has a limit of a maximum of 20000 range for system ids\n", + "# here we create a range of possible system ids to query or a range in which system ids might exist\n", + "def create_list_start(r1, r2, r3):\n", + " return [number for number in range(r1, r2, r3)]\n", + "def create_list_end(r1, r2, r3):\n", + " return [number for number in range(r1, r2, r3)]\n", + "start_id_range = create_list_start(0, 100000, 20000)\n", + "end_id_range = create_list_end(20000, 120000, 20000)\n", + "\n", + "def create_df(start_id_range=start_id_range, end_id_range=end_id_range):\n", + " frames = []\n", + " i = 0\n", + " while i < len(start_id_range)-1 and i < len(end_id_range)-1:\n", + " i += 1\n", + " df = pv.get_metadata_for_country(country_code=\"in\", start_id_range=start_id_range[i], end_id_range=end_id_range[i], use_data_service=True)\n", + " frames.append(df)\n", + " df= pd.concat(frames)\n", + " print(df)\n", + " df.to_csv(f\"/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/examples/pv_data/pv_data_india.csv\", index=False)\n", + " return df\n", + "\n", + "create_df(start_id_range=start_id_range, end_id_range=end_id_range)\n", + "\n", + "\n" ] }, { diff --git a/pvoutput/pvoutput.py b/pvoutput/pvoutput.py index f4b1701..0d3c2fd 100644 --- a/pvoutput/pvoutput.py +++ b/pvoutput/pvoutput.py @@ -70,9 +70,9 @@ def __init__( """ self.api_key = api_key self.system_id = system_id - self.rate_limit_remaining = 600 - self.rate_limit_total = 600 - self.rate_limit_reset_time = 600 + self.rate_limit_remaining = None + self.rate_limit_total = None + self.rate_limit_reset_time = None self.data_service_url = data_service_url # Set from config file if None @@ -161,7 +161,9 @@ def search( if lat is not None and lon is not None: api_params["ll"] = "{:f},{:f}".format(lat, lon) - pv_systems_text = self._api_query(service="search", api_params=api_params, **kwargs) + pv_systems_text = self._api_query( + service="search", api_params=api_params, **kwargs + ) pv_systems = pd.read_csv( StringIO(pv_systems_text), @@ -277,7 +279,7 @@ def get_status( # add timezone if timezone is not None: - pv_system_status = pv_system_status.tz_localize(timezone).tz_convert("UTC") + pv_system_status = pv_system_status.tz_convert(timezone).tz_convert("UTC") return pv_system_status @@ -314,7 +316,9 @@ def get_system_status( temperature_C, voltage, """ - _LOG.info(f"system_ids {pv_system_ids}: Requesting batch system status for %s", date) + _LOG.info( + f"system_ids {pv_system_ids}: Requesting batch system status for %s", date + ) date = date_to_pvoutput_str(date) _check_date(date) @@ -332,7 +336,9 @@ def get_system_status( ) except NoStatusFound: - _LOG.info(f"system_id {all_pv_system_id}: No status found for date %s", date) + _LOG.info( + f"system_id {all_pv_system_id}: No status found for date %s", date + ) pv_system_status_text = "no status found" # each pv system is on a new line @@ -340,7 +346,6 @@ def get_system_status( pv_system_status = [] for pv_system_status_text in pv_systems_status_text: - try: one_pv_system_status = process_system_status( pv_system_status_text=pv_system_status_text, date=date @@ -360,7 +365,7 @@ def get_system_status( if timezone is not None: pv_system_status["datetime"] = ( pd.DatetimeIndex(pv_system_status["datetime"]) - .tz_localize(timezone) + .tz_convert(timezone) .tz_convert("UTC") ) @@ -416,10 +421,17 @@ def get_batch_status( for retry in range(max_retries): try: pv_system_status_text = self._api_query( - service="getbatchstatus", api_params=api_params, use_data_service=True, **kwargs + service="getbatchstatus", + api_params=api_params, + use_data_service=True, + **kwargs, ) except NoStatusFound: - _LOG.info("system_id %d: No status found for date_to %s", pv_system_id, date_to) + _LOG.info( + "system_id %d: No status found for date_to %s", + pv_system_id, + date_to, + ) pv_system_status_text = "" break @@ -431,7 +443,8 @@ def get_batch_status( time.sleep(1) else: _print_and_log( - "Call get_batch_status again in a minute to see if" " results are ready." + "Call get_batch_status again in a minute to see if" + " results are ready." ) else: break @@ -517,6 +530,85 @@ def get_metadata(self, pv_system_id: int, **kwargs) -> pd.Series: pv_metadata.name = pv_system_id return pv_metadata + def get_metadata_for_country( + self, country_code: str, start_id_range: int, end_id_range: int, **kwargs + ) -> pd.DataFrame: + """Get metadata for a single PV system. + + Args: + country_code: str, + start_id_range: int, + end_id_range: int, + + Returns: + pd.Dataframe. Index is: + system_id, + system_DC_capacity_W, + postcode, + num_panels, + panel_capacity_W_each, + num_inverters, + inverter_capacity_W, + orientation, + array_tilt_degrees, + shade, + install_date, + latitude, + longitude, + status_interval_minutes, + secondary_num_panels, + secondary_panel_capacity_W_each, + secondary_orientation, + secondary_array_tilt_degrees + """ + pv_metadata_text = self._api_query( + service="getcountrysystem", + api_params={ + "c": country_code, # Provide data about secondary array, if present. + "from": start_id_range, + "to": end_id_range, + }, + **kwargs, + ) + + _LOG.debug( + f"getting metadata for {country_code} for {start_id_range} to {end_id_range}" + ) + print( + f"getting metadata for {country_code} for {start_id_range} to {end_id_range}" + ) + + pv_metadata_for_country = pd.read_csv( + StringIO(pv_metadata_text), + lineterminator=";", + names=[ + "system_id", + "system_DC_capacity_W", + "postcode", + "num_panels", + "panel_capacity_W_each", + "num_inverters", + "inverter_capacity_W", + "orientation", + "array_tilt_degrees", + "shade", + "install_date", + "latitude", + "longitude", + "status_interval_minutes", + "secondary_num_panels", + "secondary_panel_capacity_W_each", + "secondary_orientation", + "secondary_array_tilt_degrees", + ], + parse_dates=["install_date"], + nrows=1, + ) + pv_metadata_for_country["system_id"] = pv_metadata_for_country["system_id"] + # pv_metadata_for_country["system_id"] = [start_id_range, end_id_range] + # pv_metadata_for_country.name = [start_id_range, end_id_range] + return pv_metadata_for_country + def get_statistic( self, pv_system_id: int, @@ -596,8 +688,12 @@ def get_statistic( else: pv_metadata.index = [pv_system_id] - pv_metadata["query_date_from"] = pd.Timestamp(date_from) if date_from else pd.NaT - pv_metadata["query_date_to"] = pd.Timestamp(date_to) if date_to else pd.Timestamp.now() + pv_metadata["query_date_from"] = ( + pd.Timestamp(date_from) if date_from else pd.NaT + ) + pv_metadata["query_date_to"] = ( + pd.Timestamp(date_to) if date_to else pd.Timestamp.now() + ) return pv_metadata def _get_statistic_with_cache( @@ -644,7 +740,9 @@ def _get_fresh_statistic(): return stats try: - stats = pd.read_hdf(store_filename, key="statistics", where="index=pv_system_id") + stats = pd.read_hdf( + store_filename, key="statistics", where="index=pv_system_id" + ) except (FileNotFoundError, KeyError): return _get_fresh_statistic() @@ -710,7 +808,9 @@ def download_multiple_systems_to_disk( n = len(system_ids) for i, pv_system_id in enumerate(system_ids): _LOG.info("**********************") - msg = "system_id {:d}: {:d} of {:d} ({:%})".format(pv_system_id, i + 1, n, (i + 1) / n) + msg = "system_id {:d}: {:d} of {:d} ({:%})".format( + pv_system_id, i + 1, n, (i + 1) / n + ) _LOG.info(msg) print("\r", msg, end="", flush=True) @@ -722,7 +822,10 @@ def download_multiple_systems_to_disk( # How much data is actually available? date_ranges_to_download = self._filter_date_range( - output_filename, pv_system_id, date_ranges_to_download, min_data_availability + output_filename, + pv_system_id, + date_ranges_to_download, + min_data_availability, ) if not date_ranges_to_download: @@ -841,9 +944,13 @@ def _filter_date_range( _LOG.info("system_id %d: Stats say there is no data!", system_id) return [] - timeseries_date_range = DateRange(stats["actual_date_from"], stats["actual_date_to"]) + timeseries_date_range = DateRange( + stats["actual_date_from"], stats["actual_date_to"] + ) - data_availability = stats["num_outputs"] / (timeseries_date_range.total_days() + 1) + data_availability = stats["num_outputs"] / ( + timeseries_date_range.total_days() + 1 + ) if data_availability < min_data_availability: _LOG.info( @@ -861,7 +968,11 @@ def _filter_date_range( return new_date_ranges def _download_multiple_using_get_batch_status( - self, output_filename, pv_system_id, date_ranges_to_download, timezone: Optional[str] = None + self, + output_filename, + pv_system_id, + date_ranges_to_download, + timezone: Optional[str] = None, ): years = merge_date_ranges_to_years(date_ranges_to_download) dates_to = [year.end_date for year in years] @@ -875,7 +986,11 @@ def _download_multiple_using_get_batch_status( sort_and_de_dupe_pv_system(store, pv_system_id) def _download_multiple_using_get_status( - self, output_filename, pv_system_id, date_ranges_to_download, timezone: Optional[str] = None + self, + output_filename, + pv_system_id, + date_ranges_to_download, + timezone: Optional[str] = None, ): for date_range in date_ranges_to_download: dates = date_range.date_range() @@ -904,7 +1019,9 @@ def _download_multiple_worker( timeseries = self.get_batch_status(pv_system_id, date_to=date_to_load) if timeseries.empty: _LOG.info( - "system_id %d: Got empty timeseries back for %s", pv_system_id, date_to_load + "system_id %d: Got empty timeseries back for %s", + pv_system_id, + date_to_load, ) if use_get_status: _append_missing_date_range( @@ -924,8 +1041,8 @@ def _download_multiple_worker( ) else: total_rows += len(timeseries) - _LOG.info(f'Adding timezone {timezone} to {total_rows} rows') - timeseries = timeseries.tz_localize(timezone) + _LOG.info(f"Adding timezone {timezone} to {total_rows} rows") + timeseries = timeseries.tz_convert(timezone) _LOG.info( "system_id: %d: %d rows retrieved: %s to %s", pv_system_id, @@ -974,7 +1091,9 @@ def _api_query( RateLimitExceeded """ get_response_func = ( - self._get_data_service_response if use_data_service else self._get_api_response + self._get_data_service_response + if use_data_service + else self._get_api_response ) try: @@ -986,13 +1105,16 @@ def _api_query( try: return self._process_api_response(response) except RateLimitExceeded: - msg = "PVOutput.org API rate limit exceeded!" " Rate limit will be reset at {}".format( - self.rate_limit_reset_time + msg = ( + "PVOutput.org API rate limit exceeded!" + " Rate limit will be reset at {}".format(self.rate_limit_reset_time) ) _print_and_log(msg) if wait_if_rate_limit_exceeded: self.wait_for_rate_limit_reset() - return self._api_query(service, api_params, wait_if_rate_limit_exceeded=False) + return self._api_query( + service, api_params, wait_if_rate_limit_exceeded=False + ) raise RateLimitExceeded(response, msg) @@ -1007,16 +1129,18 @@ def _get_api_response(self, service: str, api_params: Dict) -> requests.Response self._check_api_params() # Create request headers headers = { - "X-Rate-Limit": "5", + "X-Rate-Limit": "1", "X-Pvoutput-Apikey": self.api_key, "X-Pvoutput-SystemId": self.system_id, } - api_url = urljoin(BASE_URL, "/data/r2/{}.jsp".format(service)) + api_url = urljoin(BASE_URL, "service/r2/{}.jsp".format(service)) return _get_response(api_url, api_params, headers) - def _get_data_service_response(self, service: str, api_params: Dict) -> requests.Response: + def _get_data_service_response( + self, service: str, api_params: Dict + ) -> requests.Response: """ Get the data service response from pvoutput.org @@ -1033,7 +1157,7 @@ def _get_data_service_response(self, service: str, api_params: Dict) -> requests api_params["key"] = self.api_key api_params["sid"] = self.system_id - api_url = urljoin(self.data_service_url, "/data/r2/{}.jsp".format(service)) + api_url = urljoin(self.data_service_url, "data/r2/{}.jsp".format(service)) return _get_response(api_url, api_params, headers) @@ -1048,8 +1172,9 @@ def _set_rate_limit_params(self, headers): header_value = int(headers[header_key]) setattr(self, param_name, header_value) - self.rate_limit_reset_time = pd.Timestamp.utcfromtimestamp(self.rate_limit_reset_time) - print("change is happening") + self.rate_limit_reset_time = pd.Timestamp.utcfromtimestamp( + self.rate_limit_reset_time + ) self.rate_limit_reset_time = self.rate_limit_reset_time.tz_convert("utc") _LOG.debug("%s", self.rate_limit_info()) @@ -1123,7 +1248,9 @@ def wait_for_rate_limit_reset(self, do_sleeping: bool = True) -> int: # retry_time_local = retry_time_utc.tz_convert(tz=datetime.now(tzlocal()).tzname()) retry_time_local = retry_time_utc _print_and_log( - "Waiting {:.0f} seconds. Will retry at {} UTC".format(secs_to_wait, retry_time_local) + "Waiting {:.0f} seconds. Will retry at {} UTC".format( + secs_to_wait, retry_time_local + ) ) if do_sleeping: time.sleep(secs_to_wait) @@ -1192,9 +1319,12 @@ def check_pv_system_status(pv_system_status: pd.DataFrame, requested_date: date) def _append_missing_date_range( - output_filename, pv_system_id, missing_start_date, missing_end_date, datetime_of_api_request + output_filename, + pv_system_id, + missing_start_date, + missing_end_date, + datetime_of_api_request, ): - data = { "missing_start_date_PV_localtime": pd.Timestamp(missing_start_date), "missing_end_date_PV_localtime": pd.Timestamp(missing_end_date), @@ -1209,14 +1339,25 @@ def _append_missing_date_range( missing_end_date, ) with pd.HDFStore(output_filename, mode="a", complevel=9) as store: - store.append(key="missing_dates", value=new_missing_date_range, data_columns=True) + store.append( + key="missing_dates", value=new_missing_date_range, data_columns=True + ) -def _record_gaps(output_filename, pv_system_id, date_to, timeseries, datetime_of_api_request): +def _record_gaps( + output_filename, pv_system_id, date_to, timeseries, datetime_of_api_request +): dates_of_data = ( - timeseries["instantaneous_power_gen_W"].dropna().resample("D").mean().dropna().index.date + timeseries["instantaneous_power_gen_W"] + .dropna() + .resample("D") + .mean() + .dropna() + .index.date ) - dates_requested = pd.date_range(date_to - timedelta(days=365), date_to, freq="D").date + dates_requested = pd.date_range( + date_to - timedelta(days=365), date_to, freq="D" + ).date missing_dates = set(dates_requested) - set(dates_of_data) missing_date_ranges = _convert_consecutive_dates_to_date_ranges(list(missing_dates)) _LOG.info( @@ -1259,7 +1400,10 @@ def _convert_consecutive_dates_to_date_ranges(missing_dates): end_date = missing_dates[-1] new_missing.append( - {"missing_start_date_PV_localtime": start_date, "missing_end_date_PV_localtime": end_date} + { + "missing_start_date_PV_localtime": start_date, + "missing_end_date_PV_localtime": end_date, + } ) return pd.DataFrame(new_missing)