Upt raw_fullacs to raw_acs, raw_acs to raw_spm_acs #57
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,3 @@ | ||
| from openfisca_us_data.datasets.acs.raw_acs import RawACS | ||
| from openfisca_us_data.datasets.acs.raw_spm_acs import RawSPMACS | ||
| from openfisca_us_data.datasets.acs.acs import ACS |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| from openfisca_us_data.utils import US, dataset | ||
| from openfisca_us_data.datasets.acs.raw_spm_acs import RawSPMACS | ||
| from pandas import DataFrame | ||
| import h5py | ||
|
|
||
|
|
@@ -19,10 +19,10 @@ def generate(year: int) -> None: | |
|
|
||
| # Prepare raw ACS tables | ||
| year = int(year) | ||
| if year not in RawSPMACS.years: | ||
| RawSPMACS.generate(year) | ||
|
|
||
| raw_data = RawSPMACS.load(year) | ||
| acs = h5py.File(ACS.file(year), mode="w") | ||
|
|
||
| person, spm_unit, household = [ | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,64 +2,37 @@ | |
| import requests | ||
| from io import BytesIO | ||
| import pandas as pd | ||
| from zipfile import ZipFile | ||
|
|
||
| @dataset | ||
| class RawACS: | ||
| name = "raw_acs" | ||
|
|
||
| def generate(year: int) -> None: | ||
| url = f"https://www2.census.gov/programs-surveys/acs/data/pums/{year}/1-Year/csv_pus.zip" | ||
| request = requests.get(url) | ||
| file = ZipFile(BytesIO(request.content)) | ||
| file.extractall(f'{year}_pus') | ||
|
There was a problem hiding this comment. @nikhilwoodruff feel free to suggest otherwise given these are large files (larger than others), but to be consistent with other def concat_zipped_csvs(url: str, prefix: str) -> pd.DataFrame:
# Creates a DataFrame with the two csvs inside a zip file from a URL.
zf = ZipFile(BytesIO(requests.get(url)))
a = pd.read_csv(zf.open(prefix + "a.csv"))
b = pd.read_csv(zf.open(prefix + "b.csv"))
res = pd.concat([a, b]).fillna(0)
res.columns = res.columns.str.lower()
return resThen called as: person_df = concat_zipped_csvs(
f"https://www2.census.gov/programs-surveys/acs/data/pums/{year}/1-Year/csv_pus.zip",
"psam_pus"
)And similarly for household. There was a problem hiding this comment. I think that's fine as a general approach for large files - though I did envisage something similar to |
||
|
|
||
| url2 = f'https://www2.census.gov/programs-surveys/acs/data/pums/{year}/1-Year/csv_hus.zip' | ||
| request = requests.get(url2) | ||
| file = ZipFile(BytesIO(request.content)) | ||
| file.extractall(f'{year}_hus') | ||
|
|
||
| try: | ||
| with pd.HDFStore(RawACS.file(year)) as storage: | ||
| persona = pd.read_csv(f'{year}_pus/psam_pusa.csv') | ||
| personb = pd.read_csv(f'{year}_pus/psam_pusb.csv') | ||
| person_df = pd.concat(persona, personb).fillna(0) | ||
| person_df.columns = person_df.columns.str.upper() | ||
|
|
||
| householda = pd.read_csv(f'{year}_hus/psam_husa.csv') | ||
| householdb = pd.read_csv(f'{year}_hus/psam_husa.csv') | ||
| household_df = pd.concat(householda, householdb).fillna(0) | ||
| household_df.columns = household_df.columns.str.upper() | ||
|
|
||
| except Exception as e: | ||
| RawACS.remove(year) | ||
| raise ValueError( | ||
| f"Attempted to extract and save the CSV files, but encountered an error: {e}" | ||
| ) | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| from openfisca_us_data.utils import * | ||
| import requests | ||
| from io import BytesIO | ||
| import pandas as pd | ||
|
|
||
|
|
||
| @dataset | ||
| class RawSPMACS: | ||
| name = "raw_spm_acs" | ||
|
|
||
| def generate(year: int) -> None: | ||
| url = f"https://www2.census.gov/programs-surveys/supplemental-poverty-measure/datasets/spm/spm_{year}_pu.dta" | ||
| try: | ||
| with pd.HDFStore(RawSPMACS.file(year)) as storage: | ||
| person = pd.read_stata(url).fillna(0) | ||
| person.columns = person.columns.str.upper() | ||
| storage["person"] = person | ||
| storage["spm_unit"] = create_SPM_unit_table(person) | ||
| storage["household"] = create_household_table(person) | ||
| except Exception as e: | ||
| RawSPMACS.remove(year) | ||
| raise ValueError( | ||
| f"Attempted to extract and save the CSV files, but encountered an error: {e}" | ||
| ) | ||
|
|
||
|
|
||
| def create_SPM_unit_table(person: pd.DataFrame) -> pd.DataFrame: | ||
| SPM_UNIT_COLUMNS = [ | ||
| "CAPHOUSESUB", | ||
| "CAPWKCCXPNS", | ||
| "CHILDCAREXPNS", | ||
| "EITC", | ||
| "ENGVAL", | ||
| "EQUIVSCALE", | ||
| "FEDTAX", | ||
| "FEDTAXBC", | ||
| "FICA", | ||
| "GEOADJ", | ||
| "MEDXPNS", | ||
| "NUMADULTS", | ||
| "NUMKIDS", | ||
| "NUMPER", | ||
| "POOR", | ||
| "POVTHRESHOLD", | ||
| "RESOURCES", | ||
| "SCHLUNCH", | ||
| "SNAPSUB", | ||
| "STTAX", | ||
| "TENMORTSTATUS", | ||
| "TOTVAL", | ||
| "WCOHABIT", | ||
| "WICVAL", | ||
| "WKXPNS", | ||
| "WUI_LT15", | ||
| "ID", | ||
| ] | ||
| return ( | ||
| person[["SPM_" + column for column in SPM_UNIT_COLUMNS]] | ||
| .groupby(person.SPM_ID) | ||
| .first() | ||
| ) | ||
|
|
||
|
|
||
| def create_household_table(person: pd.DataFrame) -> pd.DataFrame: | ||
| return person[["SERIALNO", "ST", "PUMA"]].groupby(person.SERIALNO).first() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -13,6 +13,9 @@ def test_CPS_import(): | |
| def test_RawACS_import(): | ||
| from openfisca_us_data import RawACS | ||
|
|
||
| def test_RawSPMACS_import(): | ||
| from openfisca_us_data import RawSPMACS | ||
|
|
||
|
|
||
| def test_ACS_import(): | ||
| from openfisca_us_data import ACS | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.