Skip to content

Commit

Permalink
Add extreme QCRAD QC limits (#190)
Browse files Browse the repository at this point in the history 
* Add extreme BSRN limits

* Add test

* Change BSRN to QCRAD

* Update tests

* Update citations

* Add whatsnew

* Fix linter

* Fix typo

* Update docs/whatsnew/0.2.2.rst

* Fix typo

Co-authored-by: Cliff Hansen <[email protected]>

* Update limits docstring from review

Co-authored-by: Kevin Anderson <[email protected]>

* Implement review feedback

* Remove <>_ from link

* Add bsrn comment to consistency tests

---------

Co-authored-by: Cliff Hansen <[email protected]>
Co-authored-by: Kevin Anderson <[email protected]>
  • Loading branch information
@AdamRJensen @cwhanse @kandersolar
3 people authored Oct 29, 2024
1 parent bf9eff6 commit 12f16d6
Show file tree
Hide file tree
Showing 3 changed files with 139 additions and 65 deletions.
7 changes: 6 additions & 1 deletion docs/whatsnew/0.2.2.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,16 @@

Enhancements
~~~~~~~~~~~~
* Added extreme limits option to
:py:func:`~pvanalytics.quality.irradiance.check_ghi_limits_qcrad`,
:py:func:`~pvanalytics.quality.irradiance.check_dhi_limits_qcrad`,
:py:func:`~pvanalytics.quality.irradiance.check_dni_limits_qcrad`, and
:py:func:`~pvanalytics.quality.irradiance.check_irradiance_limits_qcrad`.
(:pull:`190`)
* Added optional keyword `outside_domain` to
:py:func:`~pvanalytics.quality.irradiance.check_irradiance_consistency_qcrad`.
(:pull:`214`)


Bug Fixes
~~~~~~~~~

Expand Down
127 changes: 79 additions & 48 deletions pvanalytics/quality/irradiance.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,18 @@
from pvanalytics import util


QCRAD_LIMITS = {'ghi_ub': {'mult': 1.5, 'exp': 1.2, 'min': 100},
'dhi_ub': {'mult': 0.95, 'exp': 1.2, 'min': 50},
'dni_ub': {'mult': 1.0, 'exp': 0.0, 'min': 0},
'ghi_lb': -4, 'dhi_lb': -4, 'dni_lb': -4}
# QCRAD limits are often also referred to as BSRN limits
QCRAD_LIMITS_PHYSICAL = { # Physically Possible Limits
'ghi_ub': {'mult': 1.5, 'exp': 1.2, 'min': 100},
'dhi_ub': {'mult': 0.95, 'exp': 1.2, 'min': 50},
'dni_ub': {'mult': 1.0, 'exp': 0.0, 'min': 0},
'ghi_lb': -4, 'dhi_lb': -4, 'dni_lb': -4}

QCRAD_LIMITS_EXTREME = { # Extremely Rare Limits
'ghi_ub': {'mult': 1.2, 'exp': 1.2, 'min': 50},
'dhi_ub': {'mult': 0.75, 'exp': 1.2, 'min': 30},
'dni_ub': {'mult': 0.95, 'exp': 0.2, 'min': 10},
'ghi_lb': -2, 'dhi_lb': -2, 'dni_lb': -2}

QCRAD_CONSISTENCY = {
'ghi_ratio': {
Expand Down Expand Up @@ -42,8 +50,8 @@ def _qcrad_ub(dni_extra, sza, lim):
return lim['mult'] * dni_extra * cosd_sza**lim['exp'] + lim['min']


def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None):
r"""Test for physical limits on GHI using the QCRad criteria.
def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits='physical'):
r"""Test for lower and upper limits on GHI using the QCRad criteria.
Test is applied to each GHI value. A GHI value passes if value >
lower bound and value < upper bound. Lower bounds are constant for
Expand All @@ -60,10 +68,11 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None):
Solar zenith angle in degrees
dni_extra : Series
Extraterrestrial normal irradiance in :math:`W/m^2`
limits : dict, default QCRAD_LIMITS
Must have keys 'ghi_ub' and 'ghi_lb'. For 'ghi_ub' value is a
dict with keys {'mult', 'exp', 'min'} and float values. For
'ghi_lb' value is a float.
limits : {'physical', 'extreme'} or dict, default 'physical'
If string, must be either 'physical' or 'extreme', corresponding to the
QCRAD QC limits. If dict, must have keys 'ghi_ub' and 'ghi_lb'. For
'ghi_ub' value is a dict with keys {'mult', 'exp', 'min'} and float
values. For 'ghi_lb' value is a float.
Returns
-------
Expand All @@ -79,17 +88,20 @@ def check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra, limits=None):
for more information.
"""
if not limits:
limits = QCRAD_LIMITS
if limits == 'physical':
limits = QCRAD_LIMITS_PHYSICAL
elif limits == 'extreme':
limits = QCRAD_LIMITS_EXTREME

ghi_ub = _qcrad_ub(dni_extra, solar_zenith, limits['ghi_ub'])

ghi_limit_flag = quality.util.check_limits(ghi, limits['ghi_lb'], ghi_ub)

return ghi_limit_flag


def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None):
r"""Test for physical limits on DHI using the QCRad criteria.
def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits='physical'):
r"""Test for lower and upper limits on DHI using the QCRad criteria.
Test is applied to each DHI value. A DHI value passes if value >
lower bound and value < upper bound. Lower bounds are constant for
Expand All @@ -106,10 +118,11 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None):
Solar zenith angle in degrees
dni_extra : Series
Extraterrestrial normal irradiance in :math:`W/m^2`
limits : dict, default QCRAD_LIMITS
Must have keys 'dhi_ub' and 'dhi_lb'. For 'dhi_ub' value is a
dict with keys {'mult', 'exp', 'min'} and float values. For
'dhi_lb' value is a float.
limits : {'physical', 'extreme'} or dict, default 'physical'
If string, must be either 'physical' or 'extreme', corresponding to the
QCRAD QC limits. If dict, must have keys 'dhi_ub' and 'dhi_lb'. For
'dhi_ub' value is a dict with keys {'mult', 'exp', 'min'} and float
values. For 'dhi_lb' value is a float.
Returns
-------
Expand All @@ -125,8 +138,10 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None):
for more information.
"""
if not limits:
limits = QCRAD_LIMITS
if limits == 'physical':
limits = QCRAD_LIMITS_PHYSICAL
elif limits == 'extreme':
limits = QCRAD_LIMITS_EXTREME

dhi_ub = _qcrad_ub(dni_extra, solar_zenith, limits['dhi_ub'])

Expand All @@ -135,8 +150,8 @@ def check_dhi_limits_qcrad(dhi, solar_zenith, dni_extra, limits=None):
return dhi_limit_flag


def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None):
r"""Test for physical limits on DNI using the QCRad criteria.
def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits='physical'):
r"""Test for lower and upper limits on DNI using the QCRad criteria.
Test is applied to each DNI value. A DNI value passes if value >
lower bound and value < upper bound. Lower bounds are constant for
Expand All @@ -153,10 +168,11 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None):
Solar zenith angle in degrees
dni_extra : Series
Extraterrestrial normal irradiance in :math:`W/m^2`
limits : dict, default QCRAD_LIMITS
Must have keys 'dni_ub' and 'dni_lb'. For 'dni_ub' value is a
dict with keys {'mult', 'exp', 'min'} and float values. For
'dni_lb' value is a float.
limits : {'physical', 'extreme'} or dict, default 'physical'
If string, must be either 'physical' or 'extreme', corresponding to the
QCRAD QC limits. If dict, must have keys 'dni_ub' and 'dni_lb'. For
'dni_ub' value is a dict with keys {'mult', 'exp', 'min'} and float
values. For 'dni_lb' value is a float.
Returns
-------
Expand All @@ -172,8 +188,10 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None):
for more information.
"""
if not limits:
limits = QCRAD_LIMITS
if limits == 'physical':
limits = QCRAD_LIMITS_PHYSICAL
elif limits == 'extreme':
limits = QCRAD_LIMITS_EXTREME

dni_ub = _qcrad_ub(dni_extra, solar_zenith, limits['dni_ub'])

Expand All @@ -183,10 +201,10 @@ def check_dni_limits_qcrad(dni, solar_zenith, dni_extra, limits=None):


def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None,
dni=None, limits=None):
dni=None, limits='physical'):
r"""Test for physical limits on GHI, DHI or DNI using the QCRad criteria.
Criteria from [1]_ are used to determine physically plausible
Criteria from [1]_ and [2]_ are used to determine physically plausible
lower and upper bounds. Each value is tested and a value passes if
value > lower bound and value < upper bound. Lower bounds are
constant for all tests. Upper bounds are calculated as
Expand All @@ -209,10 +227,13 @@ def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None,
Diffuse horizontal irradiance in :math:`W/m^2`
dni : Series or None, default None
Direct normal irradiance in :math:`W/m^2`
limits : dict, default QCRAD_LIMITS
for keys 'ghi_ub', 'dhi_ub', 'dni_ub', value is a dict with
keys {'mult', 'exp', 'min'} and float values. For keys
'ghi_lb', 'dhi_lb', 'dni_lb', value is a float.
limits : {'physical', 'extreme'} or dict, default 'physical'
If string, must be either 'physical' or 'extreme', corresponding to the
QCRAD QC limits. If dict, must have keys:
* 'ghi_ub', 'dhi_ub', 'dni_ub': dicts with keys
{'mult', 'exp', 'min'} and float values.
* 'ghi_lb', 'dhi_lb', 'dni_lb': float values.
Returns
-------
Expand All @@ -233,13 +254,19 @@ def check_irradiance_limits_qcrad(solar_zenith, dni_extra, ghi=None, dhi=None,
References
----------
.. [1] C. N. Long and Y. Shi, An Automated Quality Assessment and Control
Algorithm for Surface Radiation Measurements, The Open Atmospheric
Science Journal 2, pp. 23-37, 2008.
"""
if not limits:
limits = QCRAD_LIMITS
.. [1] C. N. Long and Y. Shi, "An Automated Quality Assessment and Control
Algorithm for Surface Radiation Measurements," The Open Atmospheric
Science Journal, vol. 2, no. 1. Bentham Science Publishers Ltd.,
pp. 23–37, Apr. 18, 2008. :doi:`10.2174/1874282300802010023`.
.. [2] C. N. Long and E. G. Dutton, "BSRN Global Network recommended QC
tests, V2.0," Baseline Surface Radiation Network (BSRN),
Accessed: Oct. 24, 2024. [Online.] Available:
`<https://bsrn.awi.de/fileadmin/user_upload/bsrn.awi.de/Publications/BSRN_recommended_QC_tests_V2.pdf>`_
""" # noqa: E501
if limits == 'physical':
limits = QCRAD_LIMITS_PHYSICAL
elif limits == 'extreme':
limits = QCRAD_LIMITS_EXTREME

if ghi is not None:
ghi_limit_flag = check_ghi_limits_qcrad(ghi, solar_zenith, dni_extra,
Expand Down Expand Up @@ -289,8 +316,8 @@ def check_irradiance_consistency_qcrad(solar_zenith, ghi, dhi, dni,
param=None, outside_domain=False):
r"""Check consistency of GHI, DHI and DNI using QCRad criteria.
Uses criteria given in [1]_ to validate the ratio of irradiance
components.
Uses criteria given in [1]_ to validate the ratio of irradiance components.
These tests are equivalent to the BSRN comparison tests [2]_.
.. warning:: Not valid for night time or low irradiance. When the input
data fall outside the test domain, the returned value is set by the
Expand Down Expand Up @@ -342,11 +369,15 @@ def check_irradiance_consistency_qcrad(solar_zenith, ghi, dhi, dni,
References
----------
.. [1] C. N. Long and Y. Shi, An Automated Quality Assessment and Control
Algorithm for Surface Radiation Measurements, The Open Atmospheric
Science Journal 2, pp. 23-37, 2008.
"""
.. [1] C. N. Long and Y. Shi, "An Automated Quality Assessment and Control
Algorithm for Surface Radiation Measurements," The Open Atmospheric
Science Journal, vol. 2, no. 1. Bentham Science Publishers Ltd.,
pp. 23–37, Apr. 18, 2008. :doi:`10.2174/1874282300802010023`.
.. [2] C. N. Long and E. G. Dutton, "BSRN Global Network recommended QC
tests, V2.0," Baseline Surface Radiation Network (BSRN),
Accessed: Oct. 24, 2024. [Online.] Available:
https://bsrn.awi.de/fileadmin/user_upload/bsrn.awi.de/Publications/BSRN_recommended_QC_tests_V2.pdf
""" # noqa: E501
if not param:
param = QCRAD_CONSISTENCY

Expand Down
70 changes: 54 additions & 16 deletions pvanalytics/tests/quality/test_irradiance.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,28 +42,31 @@ def irradiance_qcrad():
output = pd.DataFrame(
columns=['ghi', 'dhi', 'dni', 'solar_zenith', 'dni_extra',
'ghi_limit_flag', 'dhi_limit_flag', 'dni_limit_flag',
'ghi_extreme_limit_flag', 'dhi_extreme_limit_flag',
'dni_extreme_limit_flag',
'consistent_components', 'diffuse_ratio_limit',
'consistent_components_outside_domain',
'diffuse_ratio_limit_outside_domain',
],
data=np.array([[-100, 100, 100, 30, 1370, 0, 1, 1, 0, 0, 0, 1],
[100, -100, 100, 30, 1370, 1, 0, 1, 0, 0, 1, 0],
[100, 100, -100, 30, 1370, 1, 1, 0, 0, 1, 1, 1],
[1000, 100, 900, 0, 1370, 1, 1, 1, 1, 1, 1, 1],
[1000, 200, 800, 15, 1370, 1, 1, 1, 1, 1, 1, 1],
[1000, 200, 800, 60, 1370, 0, 1, 1, 0, 1, 0, 1],
[1000, 300, 850, 80, 1370, 0, 0, 1, 0, 1, 0, 1],
[1000, 500, 800, 90, 1370, 0, 0, 1, 0, 1, 0, 1],
[500, 100, 1100, 0, 1370, 1, 1, 1, 0, 1, 0, 1],
[1000, 300, 1200, 0, 1370, 1, 1, 1, 0, 1, 0, 1],
[500, 600, 100, 60, 1370, 1, 1, 1, 0, 0, 0, 0],
[500, 600, 400, 80, 1370, 0, 0, 1, 0, 0, 0, 0],
[500, 500, 300, 80, 1370, 0, 0, 1, 1, 1, 1, 1],
[0, 0, 0, 93, 1370, 1, 1, 1, 0, 0, 1, 1],
[100, 100, 0, 95, 1370, 0, 0, 1, 0, 0, 1, 1],
data=np.array([[-100, 100, 100, 30, 1370, 0, 1, 1, 0, 1, 1, 0, 0, 0, 1], # noqa: E501
[100, -100, 100, 30, 1370, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0], # noqa: E501
[100, 100, -100, 30, 1370, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1], # noqa: E501
[1000, 100, 900, 0, 1370, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
[1000, 200, 800, 15, 1370, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], # noqa: E501
[1000, 200, 800, 60, 1370, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1], # noqa: E501
[1000, 300, 850, 80, 1370, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1], # noqa: E501
[1000, 500, 800, 90, 1370, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1], # noqa: E501
[500, 100, 1100, 0, 1370, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1],
[1000, 300, 1200, 0, 1370, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1], # noqa: E501
[500, 600, 100, 60, 1370, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0],
[500, 600, 400, 80, 1370, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0],
[500, 500, 300, 80, 1370, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1],
[0, 0, 0, 93, 1370, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1],
[100, 100, 0, 95, 1370, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1]
]))
dtypes = ['float64', 'float64', 'float64', 'float64', 'float64',
'bool', 'bool', 'bool', 'bool', 'bool', 'bool', 'bool']
'bool', 'bool', 'bool', 'bool', 'bool', 'bool', 'bool', 'bool',
'bool', 'bool']
for (col, typ) in zip(output.columns, dtypes):
output[col] = output[col].astype(typ)
return output
Expand All @@ -88,6 +91,13 @@ def test_check_ghi_limits_qcrad(irradiance_qcrad):
expected['dni_extra'])
assert_series_equal(ghi_out, ghi_out_expected, check_names=False)

ghi_out_extreme = irradiance.check_ghi_limits_qcrad(
expected['ghi'], expected['solar_zenith'], expected['dni_extra'],
limits='extreme')
ghi_extreme_out_expected = expected['ghi_extreme_limit_flag']
assert_series_equal(
ghi_out_extreme, ghi_extreme_out_expected, check_names=False)


def test_check_dhi_limits_qcrad(irradiance_qcrad):
"""Test that QCRad identifies out of bounds DHI values.
Expand All @@ -108,6 +118,13 @@ def test_check_dhi_limits_qcrad(irradiance_qcrad):
expected['dni_extra'])
assert_series_equal(dhi_out, dhi_out_expected, check_names=False)

dhi_out_extreme = irradiance.check_dhi_limits_qcrad(
expected['dhi'], expected['solar_zenith'], expected['dni_extra'],
limits='extreme')
dhi_extreme_out_expected = expected['dhi_extreme_limit_flag']
assert_series_equal(
dhi_out_extreme, dhi_extreme_out_expected, check_names=False)


def test_check_dni_limits_qcrad(irradiance_qcrad):
"""Test that QCRad identifies out of bounds DNI values.
Expand All @@ -128,6 +145,13 @@ def test_check_dni_limits_qcrad(irradiance_qcrad):
expected['dni_extra'])
assert_series_equal(dni_out, dni_out_expected, check_names=False)

dni_out_extreme = irradiance.check_dni_limits_qcrad(
expected['dni'], expected['solar_zenith'], expected['dni_extra'],
limits='extreme')
dni_extreme_out_expected = expected['dni_extreme_limit_flag']
assert_series_equal(
dni_out_extreme, dni_extreme_out_expected, check_names=False)


def test_check_irradiance_limits_qcrad(irradiance_qcrad):
"""Test different input combinations to check_irradiance_limits_qcrad.
Expand Down Expand Up @@ -162,6 +186,20 @@ def test_check_irradiance_limits_qcrad(irradiance_qcrad):
assert_series_equal(dni_out, dni_out_expected, check_names=False)


def test_check_irradiance_limits_qcrad_extreme(irradiance_qcrad):
"""Test different input combinations to check_irradiance_limits_qcrad.
"""
expected = irradiance_qcrad

ghi_out, dhi_out, dni_out = irradiance.check_irradiance_limits_qcrad(
expected['solar_zenith'], expected['dni_extra'], ghi=expected['ghi'],
dhi=expected['dhi'], dni=expected['dni'], limits='extreme')

assert_series_equal(ghi_out, expected['ghi_extreme_limit_flag'], check_names=False) # noqa: E501
assert_series_equal(dhi_out, expected['dhi_extreme_limit_flag'], check_names=False) # noqa: E501
assert_series_equal(dni_out, expected['dni_extreme_limit_flag'], check_names=False) # noqa: E501


def test_check_irradiance_consistency_qcrad(irradiance_qcrad):
"""Test that QCRad identifies consistent irradiance measurements.
Expand Down

0 comments on commit 12f16d6

Please sign in to comment.