Skip to content

Commit

Permalink
india notebook pulling data
Browse files Browse the repository at this point in the history
  • Loading branch information
@rachel-labri-tipton
rachel-labri-tipton committed Oct 25, 2023
1 parent 5d704b3 commit 84773df
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 62 deletions.
118 changes: 63 additions & 55 deletions examples/get_india_data.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
},
{
"cell_type": "code",
"execution_count": 26,
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -30,7 +30,7 @@
"\n",
"sys.path.append(\"..\")\n",
"from pvoutput.grid_search import GridSearch\n",
"from pvoutput import PVOutput"
"from pvoutput.pvoutput import PVOutput"
]
},
{
Expand All @@ -44,25 +44,15 @@
},
{
"cell_type": "code",
"execution_count": 28,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"api_key = \"3f784ff6cfa27c44436f88da0c429b410687ad14\"\n",
"system_id = \"68732\"\n",
"date = \"20230801\"\n",
"site_id = \"\"\n",
"data_service_url = \"https://pvoutput.org/data/r2/getsystemstatus.jsp\"\n",
"# pv = PVOutput(api_key, system_id, data_service_url=data_service_url)"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"CACHE_DIR = \"/tmp\""
"data_service_url = \"https://pvoutput.org\"\n",
"pv = PVOutput(api_key=api_key, system_id=system_id, data_service_url=data_service_url)\n"
]
},
{
Expand All @@ -76,59 +66,77 @@
},
{
"cell_type": "code",
"execution_count": 38,
"execution_count": 184,
"metadata": {},
"outputs": [
{
"ename": "SyntaxError",
"evalue": "invalid syntax (2245105558.py, line 3)",
"output_type": "error",
"traceback": [
"\u001b[0;36m Cell \u001b[0;32mIn[38], line 3\u001b[0;36m\u001b[0m\n\u001b[0;31m curl -H \"X-Pvoutput-Apikey: 3f784ff6cfa27c44436f88da0c429b410687ad14\" -H \"X-Pvoutput-SystemId: 68732\" https://pvoutput.org/data/r2/getsystemstatus.jsp\\?dt\\=20230801\\&sid1\\=87410\u001b[0m\n\u001b[0m ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
]
}
],
"outputs": [],
"source": [
"# Get the data from PVOutput\n",
"# sites = [85738, 56151, 87410, 78186, 100451, 90559, 60294]\n",
"request = curl -H \"X-Pvoutput-Apikey: 3f784ff6cfa27c44436f88da0c429b410687ad14\" -H \"X-Pvoutput-SystemId: 68732\" https://pvoutput.org/data/r2/getsystemstatus.jsp\\?dt\\=20230801\\&sid1\\=87410\n",
"dfs = []\n",
"url = \"https://pvoutput.org/data/r2/getsystemstatus.jsp\\?dt\\=20230801\\&sid1\\=87410\"\n",
"headers = {api_key, system_id}\n",
"r = requests.get(url, headers=headers)\n"
"CACHE_DIR = \"/tmp\""
]
},
{
"cell_type": "code",
"execution_count": 35,
"execution_count": 10,
"metadata": {},
"outputs": [
{
"ename": "JSONDecodeError",
"evalue": "Expecting value: line 1 column 1 (char 0)",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mJSONDecodeError\u001b[0m Traceback (most recent call last)",
"File \u001b[0;32m~/anaconda3/envs/pvoutput-venv/lib/python3.10/site-packages/requests/models.py:971\u001b[0m, in \u001b[0;36mResponse.json\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 970\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 971\u001b[0m \u001b[39mreturn\u001b[39;00m complexjson\u001b[39m.\u001b[39;49mloads(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtext, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 972\u001b[0m \u001b[39mexcept\u001b[39;00m JSONDecodeError \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 973\u001b[0m \u001b[39m# Catch JSON-related errors and raise as requests.JSONDecodeError\u001b[39;00m\n\u001b[1;32m 974\u001b[0m \u001b[39m# This aliases json.JSONDecodeError and simplejson.JSONDecodeError\u001b[39;00m\n",
"File \u001b[0;32m~/anaconda3/envs/pvoutput-venv/lib/python3.10/json/__init__.py:346\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m 343\u001b[0m \u001b[39mif\u001b[39;00m (\u001b[39mcls\u001b[39m \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m object_hook \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m\n\u001b[1;32m 344\u001b[0m parse_int \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m parse_float \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m\n\u001b[1;32m 345\u001b[0m parse_constant \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m object_pairs_hook \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m kw):\n\u001b[0;32m--> 346\u001b[0m \u001b[39mreturn\u001b[39;00m _default_decoder\u001b[39m.\u001b[39;49mdecode(s)\n\u001b[1;32m 347\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mcls\u001b[39m \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n",
"File \u001b[0;32m~/anaconda3/envs/pvoutput-venv/lib/python3.10/json/decoder.py:337\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m 333\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Return the Python representation of ``s`` (a ``str`` instance\u001b[39;00m\n\u001b[1;32m 334\u001b[0m \u001b[39mcontaining a JSON document).\u001b[39;00m\n\u001b[1;32m 335\u001b[0m \n\u001b[1;32m 336\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 337\u001b[0m obj, end \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mraw_decode(s, idx\u001b[39m=\u001b[39;49m_w(s, \u001b[39m0\u001b[39;49m)\u001b[39m.\u001b[39;49mend())\n\u001b[1;32m 338\u001b[0m end \u001b[39m=\u001b[39m _w(s, end)\u001b[39m.\u001b[39mend()\n",
"File \u001b[0;32m~/anaconda3/envs/pvoutput-venv/lib/python3.10/json/decoder.py:355\u001b[0m, in \u001b[0;36mJSONDecoder.raw_decode\u001b[0;34m(self, s, idx)\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mStopIteration\u001b[39;00m \u001b[39mas\u001b[39;00m err:\n\u001b[0;32m--> 355\u001b[0m \u001b[39mraise\u001b[39;00m JSONDecodeError(\u001b[39m\"\u001b[39m\u001b[39mExpecting value\u001b[39m\u001b[39m\"\u001b[39m, s, err\u001b[39m.\u001b[39mvalue) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 356\u001b[0m \u001b[39mreturn\u001b[39;00m obj, end\n",
"\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)",
"\nDuring handling of the above exception, another exception occurred:\n",
"\u001b[0;31mJSONDecodeError\u001b[0m Traceback (most recent call last)",
"\u001b[1;32m/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/examples/get_india_data.ipynb Cell 9\u001b[0m line \u001b[0;36m3\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/examples/get_india_data.ipynb#X13sZmlsZQ%3D%3D?line=0'>1</a>\u001b[0m r\u001b[39m.\u001b[39mstatus_code\n\u001b[1;32m <a href='vscode-notebook-cell:/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/examples/get_india_data.ipynb#X13sZmlsZQ%3D%3D?line=1'>2</a>\u001b[0m r\u001b[39m.\u001b[39mtext\n\u001b[0;32m----> <a href='vscode-notebook-cell:/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/examples/get_india_data.ipynb#X13sZmlsZQ%3D%3D?line=2'>3</a>\u001b[0m r\u001b[39m.\u001b[39;49mjson()\n",
"File \u001b[0;32m~/anaconda3/envs/pvoutput-venv/lib/python3.10/site-packages/requests/models.py:975\u001b[0m, in \u001b[0;36mResponse.json\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 971\u001b[0m \u001b[39mreturn\u001b[39;00m complexjson\u001b[39m.\u001b[39mloads(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtext, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m 972\u001b[0m \u001b[39mexcept\u001b[39;00m JSONDecodeError \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 973\u001b[0m \u001b[39m# Catch JSON-related errors and raise as requests.JSONDecodeError\u001b[39;00m\n\u001b[1;32m 974\u001b[0m \u001b[39m# This aliases json.JSONDecodeError and simplejson.JSONDecodeError\u001b[39;00m\n\u001b[0;32m--> 975\u001b[0m \u001b[39mraise\u001b[39;00m RequestsJSONDecodeError(e\u001b[39m.\u001b[39mmsg, e\u001b[39m.\u001b[39mdoc, e\u001b[39m.\u001b[39mpos)\n",
"\u001b[0;31mJSONDecodeError\u001b[0m: Expecting value: line 1 column 1 (char 0)"
"name": "stdout",
"output_type": "stream",
"text": [
"change is happening\n",
" datetime cumulative_energy_gen_Wh instantaneous_power_gen_W \\\n",
"0 2023-08-01 05:55:00 4.0 51.0 \n",
"1 2023-08-01 06:00:00 30.0 315.0 \n",
"2 2023-08-01 06:05:00 81.0 613.0 \n",
"3 2023-08-01 06:10:00 164.0 996.0 \n",
"4 2023-08-01 06:15:00 273.0 1303.0 \n",
".. ... ... ... \n",
"334 2023-08-01 19:20:00 25960.0 0.0 \n",
"335 2023-08-01 19:25:00 25960.0 0.0 \n",
"336 2023-08-01 19:30:00 25960.0 0.0 \n",
"337 2023-08-01 19:35:00 25960.0 0.0 \n",
"338 2023-08-01 19:40:00 25960.0 0.0 \n",
"\n",
" temperature_C voltage system_id \n",
"0 NaN NaN 85738 \n",
"1 NaN NaN 85738 \n",
"2 NaN NaN 85738 \n",
"3 NaN NaN 85738 \n",
"4 NaN NaN 85738 \n",
".. ... ... ... \n",
"334 26.0 NaN 87410 \n",
"335 26.0 NaN 87410 \n",
"336 26.0 NaN 87410 \n",
"337 26.0 NaN 87410 \n",
"338 25.0 NaN 87410 \n",
"\n",
"[339 rows x 6 columns]\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/pvoutput/prcoess.py:93: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n",
" one_pv_system_status[\"time\"] = pd.to_datetime(one_pv_system_status[\"time\"]).dt.strftime(\n",
"/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/pvoutput/prcoess.py:93: UserWarning: Could not infer format, so each element will be parsed individually, falling back to `dateutil`. To ensure parsing is consistent and as-expected, please specify a format.\n",
" one_pv_system_status[\"time\"] = pd.to_datetime(one_pv_system_status[\"time\"]).dt.strftime(\n"
]
}
],
"source": [
"r.status_code\n",
"r.text\n",
"r.json()\n",
"\n"
"# Get output for list of sites\n",
"sites = [\"87410\", \"85738\"]\n",
"df = pv.get_system_status(pv_system_ids=sites, date=date, use_data_service=True)\n",
"df.to_csv(f\"/Users/dragonflyiterations/Desktop/open-climate-fix/pvoutput/pvoutput/examples/pv_data/pv_data.csv\", index=False)\n",
"print(df)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand All @@ -147,7 +155,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
"version": "3.10.6"
}
},
"nbformat": 4,
Expand Down
15 changes: 8 additions & 7 deletions pvoutput/pvoutput.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,9 +70,9 @@ def __init__(
"""
self.api_key = api_key
self.system_id = system_id
self.rate_limit_remaining = None
self.rate_limit_total = None
self.rate_limit_reset_time = None
self.rate_limit_remaining = 600
self.rate_limit_total = 600
self.rate_limit_reset_time = 600
self.data_service_url = data_service_url

# Set from config file if None
Expand Down Expand Up @@ -1007,12 +1007,12 @@ def _get_api_response(self, service: str, api_params: Dict) -> requests.Response
self._check_api_params()
# Create request headers
headers = {
"X-Rate-Limit": "1",
"X-Rate-Limit": "5",
"X-Pvoutput-Apikey": self.api_key,
"X-Pvoutput-SystemId": self.system_id,
}

api_url = urljoin(BASE_URL, "service/r2/{}.jsp".format(service))
api_url = urljoin(BASE_URL, "/data/r2/{}.jsp".format(service))

return _get_response(api_url, api_params, headers)

Expand All @@ -1033,7 +1033,7 @@ def _get_data_service_response(self, service: str, api_params: Dict) -> requests
api_params["key"] = self.api_key
api_params["sid"] = self.system_id

api_url = urljoin(self.data_service_url, "data/r2/{}.jsp".format(service))
api_url = urljoin(self.data_service_url, "/data/r2/{}.jsp".format(service))

return _get_response(api_url, api_params, headers)

Expand All @@ -1049,7 +1049,8 @@ def _set_rate_limit_params(self, headers):
setattr(self, param_name, header_value)

self.rate_limit_reset_time = pd.Timestamp.utcfromtimestamp(self.rate_limit_reset_time)
self.rate_limit_reset_time = self.rate_limit_reset_time.tz_localize("utc")
print("change is happening")
self.rate_limit_reset_time = self.rate_limit_reset_time.tz_convert("utc")

_LOG.debug("%s", self.rate_limit_info())

Expand Down

0 comments on commit 84773df

Please sign in to comment.