Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix issue #19: when input data has no non-co2 variables #22

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Fix issue #19: when input data has no non-co2 variables #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e1e2992
Fix no non-co2 error: minimum ad-hoc if-statement
jkikstra Dec 12, 2022
84452c2
Add suggested minimum input test-data (co2 and co2|eip)
jkikstra Dec 12, 2022
c90119d
Satisfy linters
jkikstra Dec 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 79 additions & 67 deletions src/climate_assessment/checks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -555,78 +555,90 @@ def check_negatives(
"""
Check for negative emissions and remove any timeseries which has negative non-CO2 values.
"""
# set small non-negative non-CO2 values to zero
df_co2 = df.filter(variable=f"{prefix}Emissions|CO2*").timeseries()
df_nonco2 = df.filter(variable=f"{prefix}Emissions|CO2*", keep=False).timeseries()
df_nonco2 = df_nonco2.where(
(df_nonco2 > 0) | (df_nonco2 < negativethreshold) | df_nonco2.isnull(), other=0
)
df = pyam.IamDataFrame(pd.concat([df_co2, df_nonco2]))

# TODO: only checking for negatives in these variables, not in cfcs/minor gases
emissions_noco2 = [
"Emissions|BC",
"Emissions|PFC|C2F6",
"Emissions|PFC|C6F14",
"Emissions|PFC|CF4",
"Emissions|CO",
"Emissions|CH4",
"Emissions|F-Gases",
"Emissions|HFC",
"Emissions|HFC|HFC125",
"Emissions|HFC|HFC134a",
"Emissions|HFC|HFC143a",
"Emissions|HFC|HFC227ea",
"Emissions|HFC|HFC23",
# 'Emissions|HFC|HFC245ca', # not in historical dataset (RCMIP)
# "Emissions|HFC|HFC245fa", # all nan in historical dataset (RCMIP)
"Emissions|HFC|HFC32",
"Emissions|HFC|HFC43-10",
"Emissions|N2O",
"Emissions|NH3",
"Emissions|NOx",
"Emissions|OC",
"Emissions|PFC",
"Emissions|SF6",
"Emissions|Sulfur",
"Emissions|VOC",
]
df_nonco2 = df.filter(
variable=[f"{prefix}{s}" for s in emissions_noco2]
).timeseries()

# remove any timeseries which still have negative non-CO2 values
negative_nonco2 = (df_nonco2 < 0).any(axis=1).groupby(["model", "scenario"]).sum()
negative_nonco2.name = "negative_nonco2_count"
# make the negative non CO2 count line up with meta and fill anything which
# isn't there (i.e. provides CO2 only) with 0
negative_nonco2 = negative_nonco2.align(df.meta)[0].fillna(0)

df.set_meta(negative_nonco2)
df_no_negatives = df.filter(negative_nonco2_count=0.0)
df_negatives = df.filter(negative_nonco2_count=0.0, keep=False)

if filename:
_write_file(
outdir,
df_negatives,
"{}_excluded_scenarios_unexpectednegatives.csv".format(filename),
if df.filter(variable=f"{prefix}Emissions|CO2*", keep=False).empty:
# if there are only CO2 emissions in this emissions set, return input directly
return df
else:
# if there are non-CO2 emissions, perform a couple of checks and small modifications

# set small non-negative non-CO2 values to zero
df_co2 = df.filter(variable=f"{prefix}Emissions|CO2*").timeseries()
df_nonco2 = df.filter(
variable=f"{prefix}Emissions|CO2*", keep=False
).timeseries()
df_nonco2 = df_nonco2.where(
(df_nonco2 > 0) | (df_nonco2 < negativethreshold) | df_nonco2.isnull(),
other=0,
)
df = pyam.IamDataFrame(pd.concat([df_co2, df_nonco2]))

# only checking for negatives in these variables, not in cfcs/minor gases (which are always infilled for now)
emissions_noco2 = [
"Emissions|BC",
"Emissions|PFC|C2F6",
"Emissions|PFC|C6F14",
"Emissions|PFC|CF4",
"Emissions|CO",
"Emissions|CH4",
"Emissions|F-Gases",
"Emissions|HFC",
"Emissions|HFC|HFC125",
"Emissions|HFC|HFC134a",
"Emissions|HFC|HFC143a",
"Emissions|HFC|HFC227ea",
"Emissions|HFC|HFC23",
# 'Emissions|HFC|HFC245ca', # not in historical dataset (RCMIP)
# "Emissions|HFC|HFC245fa", # all nan in historical dataset (RCMIP)
"Emissions|HFC|HFC32",
"Emissions|HFC|HFC43-10",
"Emissions|N2O",
"Emissions|NH3",
"Emissions|NOx",
"Emissions|OC",
"Emissions|PFC",
"Emissions|SF6",
"Emissions|Sulfur",
"Emissions|VOC",
]
df_nonco2 = df.filter(
variable=[f"{prefix}{s}" for s in emissions_noco2]
).timeseries()

# the case of skipping the entire scenario - remove from df to be passed on to harmonization.
for model, scen in df_negatives.index:
LOGGER.info(
"\n==================================\n"
+ "Unexpected (non-CO2) negative emissions found in "
+ "scenario "
+ scen
+ " produced by "
+ model
+ "!\n"
+ "=================================="
# remove any timeseries which still have negative non-CO2 values
negative_nonco2 = (
(df_nonco2 < 0).any(axis=1).groupby(["model", "scenario"]).sum()
)
negative_nonco2.name = "negative_nonco2_count"
# make the negative non CO2 count line up with meta and fill anything which
# isn't there (i.e. provides CO2 only) with 0
negative_nonco2 = negative_nonco2.align(df.meta)[0].fillna(0)

df.set_meta(negative_nonco2)
df_no_negatives = df.filter(negative_nonco2_count=0.0)
df_negatives = df.filter(negative_nonco2_count=0.0, keep=False)

if filename:
_write_file(
outdir,
df_negatives,
"{}_excluded_scenarios_unexpectednegatives.csv".format(filename),
)

# the case of skipping the entire scenario - remove from df to be passed on to harmonization.
for model, scen in df_negatives.index:
LOGGER.info(
"\n==================================\n"
+ "Unexpected (non-CO2) negative emissions found in "
+ "scenario "
+ scen
+ " produced by "
+ model
+ "!\n"
+ "=================================="
)

return df_no_negatives
return df_no_negatives


def remove_rows_with_zero_in_harmonization_year(
Expand Down
3 changes: 3 additions & 0 deletions tests/test-data/ar6_minimum_emissions.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Model,Scenario,Region,Variable,Unit,2010,2011,2012,2013,2014,2015,2016,2017,2018,2019,2020,2021,2022,2023,2024,2025,2026,2027,2028,2029,2030,2031,2032,2033,2034,2035,2036,2037,2038,2039,2040,2041,2042,2043,2044,2045,2046,2047,2048,2049,2050,2051,2052,2053,2054,2055,2056,2057,2058,2059,2060,2061,2062,2063,2064,2065,2066,2067,2068,2069,2070,2071,2072,2073,2074,2075,2076,2077,2078,2079,2080,2081,2082,2083,2084,2085,2086,2087,2088,2089,2090,2091,2092,2093,2094,2095,2096,2097,2098,2099,2100
GCAM 5.3,NGFS2_Current Policies,World,Emissions|CO2|Energy and Industrial Processes,Mt CO2/yr,,,,,,35250.5742,,,,,36560.72139,,,,,38949.65036,,,,,40623.76657,,,,,43413.74439,,,,,44501.04156,,,,,45744.10764,,,,,46760.51837,,,,,48042.89848,,,,,48986.12525,,,,,50068.46561,,,,,50663.63855,,,,,51315.83815,,,,,52282.91282,,,,,52451.34883,,,,,52655.79184,,,,,53304.09802,,,,,51998.82672
REMIND-MAgPIE 2.1-4.2,SusDev_SDP-PkBudg1000,World,Emissions|CO2,Mt CO2/yr,36877.9484,,,,,,,,,,39240.8915,,,,,30526.2728,,,,,23790.7544,,,,,17645.4877,,,,,13253.857,,,,,9532.7416,,,,,6081.4463,,,,,4064.6924,,,,,2299.2548,,,,,,,,,,-540.2351,,,,,,,,,,-2086.7302,,,,,,,,,,-2822.3177,,,,,,,,,,-3601.9426