Merge pull request #52 from PolicyEngine/docs-fixes

Add Jupyter Book documentation

.github/CONTRIBUTING.md

@@ -1,3 +1,7 @@
 ## Updating data
 
 If your changes present a non-bugfix change to one or more datasets which are cloud-hosted (CPS, ECPS and PUF), then please change both the filename and URL (in both the class definition file and in `storage/upload_completed_datasets.py`. This enables us to store historical versions of datasets separately and reproducibly.
+
+## Updating versioning
+
+Please add a versioning entry to `changelog_entry.yaml` (see previous PRs for examples), then run `make changelog` and commit the results ONCE in this PR.

.github/workflows/ci_cd.yaml

@@ -22,37 +22,8 @@ jobs:
           pip install black
       - name: Check formatting
         run: black . -l 79 --check
-  check-version:
-    name: Check version
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0 # Fetch all history for all tags and branches
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-      - name: Build changelog
-        run: pip install "yaml-changelog>=0.1.7" && make changelog
-      - name: Preview changelog update
-        run: ".github/get-changelog-diff.sh"
-      - name: Check version number has been properly updated
-        run: ".github/is-version-number-acceptable.sh"
-      - name: Update changelog
-        uses: EndBug/add-and-commit@v9
-        with:
-          add: "."
-          committer_name: Github Actions[bot]
-          author_name: Github Actions[bot]
-          message: Update PolicyEngine US data
-          github_token: ${{ secrets.POLICYENGINE_GITHUB }}
   test:
-    name: Build and Test
+    name: Build and test
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -71,8 +42,12 @@ jobs:
           POLICYENGINE_US_DATA_GITHUB_TOKEN: ${{ secrets.POLICYENGINE_US_DATA_GITHUB_TOKEN }}
       - name: Build datasets
         run: make data
+        env:
+          TEST_LITE: true
       - name: Run tests
         run: pytest
+      - name: Test documentation builds
+        run: make documentation
   publish-to-pypi:
     name: Publish to PyPI
     runs-on: ubuntu-latest
@@ -115,8 +90,31 @@ jobs:
         run: docker build . -f docker/policyengine_us_data.Dockerfile -t ghcr.io/policyengine/policyengine-us-data:latest
       - name: Push container
         run: docker push ghcr.io/policyengine/policyengine-us-data:latest
+  publish-docs:
+    name: Publish documentation
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all history for all tags and branches
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Install package
+        run: pip install -e ".[dev]"
+      - name: Build Jupyter Book
+        run: make documentation
+      - name: Deploy documentation
+        uses: JamesIves/github-pages-deploy-action@releases/v4
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BRANCH: gh-pages
+          FOLDER: docs/_build/html
   upload:
-    name: Upload Data 
+    name: Upload data 
     runs-on: ubuntu-latest
     needs: [lint, test]
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
@@ -141,12 +139,12 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: cps_2024.h5
-          path: policyengine_us_data/data_storage/cps_2024.h5
+          path: policyengine_us_data/storage/cps_2024.h5
       - name: Upload ECPS 2024
         uses: actions/upload-artifact@v4
         with:
           name: enhanced_cps_2024.h5
-          path: policyengine_us_data/data_storage/enhanced_cps_2024.h5
+          path: policyengine_us_data/storage/enhanced_cps_2024.h5
       - name: Upload data
         run: make upload
         env:

.gitignore

@@ -2,9 +2,9 @@
 **/__pycache__
 **/.DS_STORE
 **/*.h5
-*.ipynb
 **/*.csv
 !uprating_factors.csv
 !uprating_growth_factors.csv
 !healthcare_spending.csv
 !spm_threshold_agi.csv
+**/_build

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.3.1] - 2024-09-17 19:37:44
+
+### Added
+
+- Jupyter Book documentation.
+
 ## [1.3.0] - 2024-09-17 10:27:10
 
 ### Fixed
@@ -59,6 +65,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 
 
+[1.3.1]: https://github.com/PolicyEngine/policyengine-us-data/compare/1.3.0...1.3.1
 [1.3.0]: https://github.com/PolicyEngine/policyengine-us-data/compare/1.2.1...1.3.0
 [1.2.1]: https://github.com/PolicyEngine/policyengine-us-data/compare/1.2.0...1.2.1
 [1.2.0]: https://github.com/PolicyEngine/policyengine-us-data/compare/1.1.1...1.2.0

Makefile

@@ -19,25 +19,25 @@ changelog:
 	touch changelog_entry.yaml
 
 download:
-	python policyengine_us_data/data_storage/download_public_prerequisites.py
-	python policyengine_us_data/data_storage/download_private_prerequisites.py
+	python policyengine_us_data/storage/download_public_prerequisites.py
+	python policyengine_us_data/storage/download_private_prerequisites.py
 
 upload:
-	python policyengine_us_data/data_storage/upload_completed_datasets.py
+	python policyengine_us_data/storage/upload_completed_datasets.py
 
 docker:
 	docker buildx build --platform linux/amd64 . -t policyengine-us-data:latest
-
+	
 documentation:
-	streamlit run docs/Home.py
+	jb clean docs && jb build docs
 
 data:
 	python policyengine_us_data/datasets/cps/cps.py
 	python policyengine_us_data/datasets/cps/enhanced_cps.py
 
 clean:
-	rm -f policyengine_us_data/data_storage/puf_2015.csv
-	rm -f policyengine_us_data/data_storage/demographics_2015.csv
+	rm -f policyengine_us_data/storage/puf_2015.csv
+	rm -f policyengine_us_data/storage/demographics_2015.csv
 
 build:
 	python -m build

changelog.yaml

@@ -40,3 +40,8 @@
     fixed:
     - Moved heavy dependencies to dev.
   date: 2024-09-17 10:27:10
+- bump: patch
+  changes:
+    added:
+    - Jupyter Book documentation.
+  date: 2024-09-17 19:37:44

docs/_config.yml

@@ -0,0 +1,22 @@
+title: PolicyEngine US data
+author: PolicyEngine
+copyright: "2024"
+logo: logo.png
+
+execute:
+  execute_notebooks: off
+
+repository:
+  url: https://github.com/policyengine/policyengine-us-data
+  branch: master
+  path_to_book: docs
+
+sphinx:
+  config:
+    html_js_files:
+    - https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.7/require.min.js
+    html_theme: furo
+    pygments_style: default
+    html_css_files:
+      - style.css
+    nb_remove_code_source: true

docs/_static/style.css

@@ -0,0 +1,2 @@
+@import url('https://fonts.googleapis.com/css2?family=Roboto+Serif:[email protected]&family=Roboto:wght@300&display=swap');
+

docs/_toc.yml

@@ -0,0 +1,4 @@
+format: jb-book
+root: intro
+chapters:
+- file: validation.ipynb

docs/intro.md

@@ -0,0 +1,14 @@
+# Introduction
+
+PolicyEngine-US-Data is a package that creates representative microdata for the US, 
+designed for input in the PolicyEngine tax-benefit microsimulation model. This tool 
+allows users to explore the data sources, validation processes, and enhancements 
+made to ensure accurate and reliable microsimulation results.
+
+PolicyEngine is a tool with a clear purpose: for given assumptions about US government policy and US households, predicting what US households will look like in the next few years. To do that, we need both of two things:
+
+* An accurate model of the effects of policy rules on households.
+* An accurate representation of the current US household sector *now*.
+
+This repository is dedicated to the second of those. In this documentation, we'll explain how we do that, but we'll also use our model (the first bullet) to see what we end up with when we combine the two, and measure up against other organisations doing the same thing.
+