From 921998dd4e43697a68e7c08f5423b47026c1ccca Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Thu, 19 Sep 2024 14:31:09 -0400 Subject: [PATCH 01/22] Add automated pytest with GitHub Actions and fix egrid test Add pytest with conda Allow workflow to run on pytest branch Fix script location Fix conda environment sourcing Add conda version script Add conda init Test setup miniconda Add conda creation Test Fix conda environment name Add execute permissions Fix pytest command Activate emcommon Fix footprint calculations Attempt to fix footprint test Attempt to fix footprint test Add MODE for footprint test Reset test footprint Sync with Jack --- .github/workflows/pytest.yml | 87 ++++++++ bin/export_versions.sh | 4 + bin/run_pytest.sh | 0 test/metrics/test_egrid.py | 87 ++++---- test/metrics/test_footprint_calculations.py | 222 ++++++++++++++------ 5 files changed, 294 insertions(+), 106 deletions(-) create mode 100644 .github/workflows/pytest.yml create mode 100644 bin/export_versions.sh mode change 100644 => 100755 bin/run_pytest.sh diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml new file mode 100644 index 0000000..3b47121 --- /dev/null +++ b/.github/workflows/pytest.yml @@ -0,0 +1,87 @@ +# taken from https://github.com/e-mission/e-mission-server/blob/master/.github/workflows/test-with-manual-install.yml + +name: pytest + +# Controls when the action will run. Triggers the workflow on push or pull request +# events but only for the master branch +on: + push: + branches: + - master + - pytest + - gis-based-mode-detection + pull_request: + branches: + - master + - pytest + - gis-based-mode-detection + schedule: + # * is a special character in YAML so you have to quote this string + - cron: '5 4 * * 0' + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # The type of runner that the job will run on + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + - name: Install and start MongoDB + uses: supercharge/mongodb-github-action@1.3.0 + with: + mongodb-version: 4.4.0 + + - name: Check existing version of miniconda + shell: bash -l {0} + run: conda info -a + + - uses: conda-incubator/setup-miniconda@v2 + with: + miniconda-version: "latest" + channels: bioconda, conda-forge, defaults + use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! + auto-update-conda: true + auto-activate-base: true + + - name: Install emcommon conda environment + shell: bash -l {0} + run: | + # echo Test + conda env update -n emcommon -f bin/environment.yml + + - name: Activate the conda environment + shell: bash -l {0} + run: | + conda activate emcommon + + - name: Show current conda environment + shell: bash -l {0} + run: | + conda activate emcommon && conda info --envs + + # - name: Setup the emission environment for testing + # shell: bash -l {0} + # run: | + # conda --version + # which conda + # source setup/activate_conda.sh + # conda --version + # source setup/setup_tests.sh + + - name: Switch to emission and run the tests + shell: bash -l {0} + run: | + conda activate emcommon + PYTHONPATH=./src pytest test --asyncio-mode=auto + + # - name: Teardown the test environment + # shell: bash -l {0} + # run: source setup/teardown_tests.sh diff --git a/bin/export_versions.sh b/bin/export_versions.sh new file mode 100644 index 0000000..95d4dda --- /dev/null +++ b/bin/export_versions.sh @@ -0,0 +1,4 @@ +# taken from https://github.com/e-mission/e-mission-server/blob/master/setup/export_versions.sh + +export EXP_CONDA_VER=23.5.2 +export EXP_CONDA_VER_SUFFIX=0 diff --git a/bin/run_pytest.sh b/bin/run_pytest.sh old mode 100644 new mode 100755 diff --git a/test/metrics/test_egrid.py b/test/metrics/test_egrid.py index b7716b8..faefea4 100644 --- a/test/metrics/test_egrid.py +++ b/test/metrics/test_egrid.py @@ -1,6 +1,5 @@ -import unittest - import emcommon.metrics.footprint.egrid as emcmfe +from ..__testing import jest_test, jest_describe, expectEqual, expectAlmostEqual KG_PER_LB = 0.453592 @@ -13,43 +12,47 @@ } -class TestEgrid(unittest.IsolatedAsyncioTestCase): - async def test_egrid_intensity_cincinnati_2022(self): - # Cincinnati, OH (RFCW region) - coords = [-84.52, 39.13] - (kg_per_kwh, metadata) = await emcmfe.get_egrid_intensity_for_coords(2022, coords) - - expected_kg_per_kwh = EGRID_EXPECTED_LBS_PER_KWH["RFCW"] * KG_PER_LB - expected_metadata = { - "data_sources": ["egrid2022"], - "data_source_urls": [ - "https://www.epa.gov/system/files/documents/2024-01/egrid2022_data.xlsx", - ], - "is_provisional": False, - "requested_year": 2022, - "requested_coords": coords, - "egrid_region": "RFCW", - } - self.assertAlmostEqual(kg_per_kwh, expected_kg_per_kwh, places=2) - for key in expected_metadata: - self.assertEqual(metadata[key], expected_metadata[key]) - - async def test_egrid_intensity_eagle_point_2023(self): - # Eagle Point, OR (NWPP region) - coords = [-122.83, 42.29] - (kg_per_kwh, metadata) = await emcmfe.get_egrid_intensity_for_coords(2023, coords) - - expected_kg_per_kwh = EGRID_EXPECTED_LBS_PER_KWH["NWPP"] * KG_PER_LB - expected_metadata = { - "data_sources": ["egrid2022"], - "data_source_urls": [ - "https://www.epa.gov/system/files/documents/2024-01/egrid2022_data.xlsx", - ], - "is_provisional": True, # provisional; 2023 was requested but 2022 was used - "requested_year": 2023, - "requested_coords": coords, - "egrid_region": "NWPP", - } - self.assertAlmostEqual(kg_per_kwh, expected_kg_per_kwh, places=2) - for key in expected_metadata: - self.assertEqual(metadata[key], expected_metadata[key]) +@jest_test +async def test_egrid_intensity_cincinnati_2022(): + # Cincinnati, OH (RFCW region) + coords = [-84.52, 39.13] + (kg_per_kwh, metadata) = await emcmfe.get_egrid_intensity(2022, coords) + + expected_kg_per_kwh = EGRID_EXPECTED_LBS_PER_KWH["RFCW"] * KG_PER_LB + expected_metadata = { + "data_sources": ["egrid2022"], + "data_source_urls": [ + "https://www.epa.gov/system/files/documents/2024-01/egrid2022_data.xlsx", + ], + "is_provisional": False, + "requested_year": 2022, + "requested_coords": coords, + "egrid_region": "RFCW", + } + expectAlmostEqual(kg_per_kwh, expected_kg_per_kwh, delta=0.01) + for key in expected_metadata.keys(): + expectEqual(metadata[key], expected_metadata[key]) + + +@jest_test +async def test_egrid_intensity_eagle_point_2023(): + # Eagle Point, OR (NWPP region) + coords = [-122.83, 42.29] + (kg_per_kwh, metadata) = await emcmfe.get_egrid_intensity(2023, coords) + + expected_kg_per_kwh = EGRID_EXPECTED_LBS_PER_KWH["NWPP"] * KG_PER_LB + expected_metadata = { + "data_sources": ["egrid2022"], + "data_source_urls": [ + "https://www.epa.gov/system/files/documents/2024-01/egrid2022_data.xlsx", + ], + "is_provisional": True, # provisional; 2023 was requested but 2022 was used + "requested_year": 2023, + "requested_coords": coords, + "egrid_region": "NWPP", + } + expectAlmostEqual(kg_per_kwh, expected_kg_per_kwh, delta=0.01) + for key in expected_metadata.keys(): + expectEqual(metadata[key], expected_metadata[key]) + +jest_describe("test_egrid") diff --git a/test/metrics/test_footprint_calculations.py b/test/metrics/test_footprint_calculations.py index bb9d547..1d62b6c 100644 --- a/test/metrics/test_footprint_calculations.py +++ b/test/metrics/test_footprint_calculations.py @@ -1,69 +1,163 @@ -import unittest import emcommon.metrics.footprint.footprint_calculations as emcmff import emcommon.metrics.footprint.util as emcmfu +from ..__testing import jest_test, jest_describe, expectEqual, expectAlmostEqual -class TestFootprintCalculations(unittest.IsolatedAsyncioTestCase): - - async def test_car_default_footprint(self): - """ - 1 km in a default CAR should consume 0.87 kWh and emit 0.23 kg CO2e. - """ - fake_trip = {'distance': 1000} - fake_mode = {'base_mode': 'CAR', 'passengers': 1} - - (footprint, metadata) = await emcmff.calc_footprint_for_trip(fake_trip, fake_mode) - - expected_footprint = {'kwh': 0.87, 'kg_co2': 0.23} - for key in expected_footprint: - self.assertAlmostEqual(footprint[key], expected_footprint[key], places=2) - - # with 2 passengers, the footprint should be halved - fake_mode['passengers'] = 2 - - (footprint, metadata) = await emcmff.calc_footprint_for_trip(fake_trip, fake_mode) - - expected_footprint = {'kwh': 0.87 / 2, 'kg_co2': 0.23 / 2} - for key in expected_footprint: - self.assertAlmostEqual(footprint[key], expected_footprint[key], places=2) - - async def test_car_custom_footprint(self): - """ - 1 km in a custom CAR (wh/km = 100) should consume 0.1 kWh and ... - """ - fake_trip = {'distance': 1000} - fake_mode = {'base_mode': 'CAR', 'passengers': 1, - 'footprint': {'gasoline': {'wh_per_km': 100}}} - - (footprint, metadata) = await emcmff.calc_footprint_for_trip(fake_trip, fake_mode) - - expected_footprint = {'kwh': 0.1, 'kg_co2': 0.1 * emcmfu.FUELS_KG_CO2_PER_KWH['gasoline']} - for key in expected_footprint: - self.assertAlmostEqual(footprint[key], expected_footprint[key], places=2) - - async def test_nyc_bus_footprint(self): - """ - 10 km in a NYC BUS should consume 6.47 kWh and emit 1.72 kg CO2e. - """ - fake_trip = { - 'distance': 10000, - 'start_fmt_time': '2022-01-01', - 'start_loc': {'coordinates': [-74.006, 40.7128]} - } - fake_mode = {'base_mode': 'BUS'} - - (footprint, metadata) = await emcmff.calc_footprint_for_trip(fake_trip, fake_mode) - - expected_footprint = {'kwh': 6.47, 'kg_co2': 1.72} - expected_metadata = { - "data_sources": ["ntd2022", "egrid2022"], - "is_provisional": False, - "requested_year": 2022, - "ntd_uace_code": "63217", - "ntd_modes": ["MB", "RB", "CB"], - } - for key in expected_footprint: - self.assertAlmostEqual(footprint[key], expected_footprint[key], places=2) - for key in expected_metadata: - self.assertEqual(metadata[key], expected_metadata[key]) +@jest_test +async def test_car_footprint(): + """ + Test kWh and kg CO2e for a 1 km trip in the default CAR. + """ + fake_trip = { + "_id": 'fake_trip_id', + "distance": 1000, + "start_fmt_time": "0000-00-00T00:00:00", + "start_loc": {"coordinates": [0, 0]}, + "user_input": {"mode_confirm": "default_car"} + } + fake_label_options = { + "MODE": [ + {"value": "default_car", "base_mode": "CAR"} + ] + } + + (footprint, metadata) = await emcmff.calc_footprint_for_trip(fake_trip, fake_label_options) + + expected_footprint = {'kwh': 0.899, 'kg_co2': 0.291} + for key in expected_footprint.keys(): + expectAlmostEqual(footprint[key], expected_footprint[key], delta=0.001) + + # with 2 passengers, the footprint should be halved + fake_label_options['MODE'][0]['passengers'] = 2 + + (footprint, metadata) = await emcmff.calc_footprint_for_trip(fake_trip, fake_label_options) + + expected_footprint = {'kwh': 0.899 / 2, 'kg_co2': 0.291 / 2} + for key in expected_footprint.keys(): + expectAlmostEqual(footprint[key], expected_footprint[key], delta=0.001) + + +@jest_test +async def test_ebike_footprint(): + """ + Test kWh and kg CO2e for a 1 km trip in the default E_BIKE. + """ + fake_trip = { + "_id": "fake_trip_id", + "distance": 1000, + "start_fmt_time": "0000-00-00T00:00:00", + "start_loc": {"coordinates": [0, 0]}, + "user_input": {"mode_confirm": "ebike"} + } + fake_label_options = { + "MODE": [ + {"value": "ebike", "base_mode": "E_BIKE"} + ] + } + + (footprint, metadata) = await emcmff.calc_footprint_for_trip(fake_trip, fake_label_options) + + expected_footprint = {'kwh': 0.014, 'kg_co2': 0.006} + for key in expected_footprint.keys(): + expectAlmostEqual(footprint[key], expected_footprint[key], delta=0.001) + + +@jest_test +async def test_custom_car_footprint(): + """ + Test kWh and kg CO2e for a 10 km trip in a custom CAR (wh/km = 100). + """ + fake_trip = { + "_id": "fake_trip_id", + "distance": 1000, + "start_fmt_time": "0000-00-00T00:00:00", + "start_loc": {"coordinates": [0, 0]}, + "user_input": {"mode_confirm": "custom_car"} + } + fake_label_options = { + "MODE": [ + {"value": "custom_car", "passengers": 1, "footprint": {"gasoline": {"wh_per_km": 100}}} + ] + } + + (footprint, metadata) = await emcmff.calc_footprint_for_trip(fake_trip, fake_label_options) + + expected_footprint = { + 'kwh': 0.1, + 'kg_co2': (0.1 / 1000) * emcmfu.FUELS_KG_CO2_PER_MWH['gasoline'] + } + for key in expected_footprint.keys(): + expectAlmostEqual(footprint[key], expected_footprint[key], delta=0.01) + + +@jest_test +async def test_nyc_bus_footprint(): + """ + Test kWh and kg CO2e for a 10 km bus trip in NYC. + """ + fake_trip = { + "_id": "fake_trip_id", + "distance": 10000, + "start_fmt_time": "2022-01-01", + "start_loc": {"coordinates": [-74.006, 40.7128]}, + "user_input": {"mode_confirm": "bus"}, + } + fake_label_options = { + "MODE": [ + {"value": "bus", "base_mode": "BUS"} + ] + } + + (footprint, metadata) = await emcmff.calc_footprint_for_trip(fake_trip, fake_label_options) + + expected_footprint = {'kwh': 12.93, 'kg_co2': 2.80} + expected_metadata = { + "data_sources": ["ntd2022", "egrid2022"], + "is_provisional": False, + "requested_year": 2022, + "ntd_uace_code": "63217", + "ntd_modes": ["MB", "RB", "CB"], + } + for key in expected_footprint.keys(): + expectAlmostEqual(footprint[key], expected_footprint[key], delta=0.01) + for key in expected_metadata.keys(): + expectEqual(metadata[key], expected_metadata[key]) + + +@jest_test +async def test_impact_of_ebike_replacing_car(): + """ + Test kWh and kg CO2e saved for a 1 km trip where E_BIKE replaced CAR. + """ + fake_trip = { + "_id": "fake_trip_id", + "distance": 1000, + "start_fmt_time": "0000-00-00T00:00:00", + "start_loc": {"coordinates": [0, 0]}, + "user_input": {"mode_confirm": "ebike", "replaced_mode_confirm": "car"} + } + fake_label_options = { + "MODE": [ + {"value": "car", "base_mode": "CAR"}, + {"value": "ebike", "base_mode": "E_BIKE"} + ] + } + + (mode_footprint, metadata) = await emcmff.calc_footprint_for_trip( + fake_trip, + fake_label_options + ) + (replaced_mode_footprint, metadata) = await emcmff.calc_footprint_for_trip( + fake_trip, + fake_label_options, + 'replaced_mode' + ) + kwh_saved = replaced_mode_footprint['kwh'] - mode_footprint['kwh'] + kg_co2_saved = replaced_mode_footprint['kg_co2'] - mode_footprint['kg_co2'] + + expectAlmostEqual(kwh_saved, 0.885, delta=0.001) + expectAlmostEqual(kg_co2_saved, 0.285, delta=0.001) + + +jest_describe("test_footprint_calculations") From 22a9c183a6195ff7cf639c661b110601ee2e32dd Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Thu, 19 Sep 2024 16:21:09 -0400 Subject: [PATCH 02/22] Remove redundant GitHub Actions jobs --- .github/workflows/pytest.yml | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3b47121..c5b1322 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -39,10 +39,6 @@ jobs: with: mongodb-version: 4.4.0 - - name: Check existing version of miniconda - shell: bash -l {0} - run: conda info -a - - uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" @@ -51,37 +47,23 @@ jobs: auto-update-conda: true auto-activate-base: true + - name: Check existing version of miniconda + shell: bash -l {0} + run: conda info -a + - name: Install emcommon conda environment shell: bash -l {0} run: | # echo Test conda env update -n emcommon -f bin/environment.yml - - name: Activate the conda environment - shell: bash -l {0} - run: | - conda activate emcommon - - name: Show current conda environment shell: bash -l {0} run: | conda activate emcommon && conda info --envs - # - name: Setup the emission environment for testing - # shell: bash -l {0} - # run: | - # conda --version - # which conda - # source setup/activate_conda.sh - # conda --version - # source setup/setup_tests.sh - - - name: Switch to emission and run the tests + - name: Switch to emcommon and run the tests shell: bash -l {0} run: | conda activate emcommon PYTHONPATH=./src pytest test --asyncio-mode=auto - - # - name: Teardown the test environment - # shell: bash -l {0} - # run: source setup/teardown_tests.sh From 8633c2c9b3a73659a9a46f5521f60aeef8378b36 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Thu, 19 Sep 2024 23:57:02 -0400 Subject: [PATCH 03/22] Add jest job --- .github/workflows/pytest.yml | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c5b1322..7f20727 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -6,23 +6,11 @@ name: pytest # events but only for the master branch on: push: - branches: - - master - - pytest - - gis-based-mode-detection pull_request: - branches: - - master - - pytest - - gis-based-mode-detection - schedule: - # * is a special character in YAML so you have to quote this string - - cron: '5 4 * * 0' # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: - # This workflow contains a single job called "build" - build: + test-pytest: # The type of runner that the job will run on runs-on: ${{ matrix.os }} strategy: @@ -67,3 +55,20 @@ jobs: run: | conda activate emcommon PYTHONPATH=./src pytest test --asyncio-mode=auto + + test-jest: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: '14' + + - name: Install dependencies + run: npm install + + - name: Run jest tests + run: npm test From 8d0bcc9ef4be269a2aba0573127e34e890098862 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 00:21:50 -0400 Subject: [PATCH 04/22] Fix npm test --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 7f20727..417658b 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -55,7 +55,7 @@ jobs: run: | conda activate emcommon PYTHONPATH=./src pytest test --asyncio-mode=auto - + test-jest: runs-on: ubuntu-latest @@ -71,4 +71,4 @@ jobs: run: npm install - name: Run jest tests - run: npm test + run: npm test test From fd71135802968b69ce27f65b1c3ce028935202b5 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 00:26:16 -0400 Subject: [PATCH 05/22] Fix jest --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 417658b..d38b3ef 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -71,4 +71,4 @@ jobs: run: npm install - name: Run jest tests - run: npm test test + run: npx jest From cd8250ab6046223cb13611e0daca648c278ecec0 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 00:30:08 -0400 Subject: [PATCH 06/22] Fix babel and jest --- .babelrc | 3 +++ package.json | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 .babelrc diff --git a/.babelrc b/.babelrc new file mode 100644 index 0000000..a1b6dc2 --- /dev/null +++ b/.babelrc @@ -0,0 +1,3 @@ +{ + "presets": ["@babel/preset-env"] + } diff --git a/package.json b/package.json index 6e7e99c..47e27ee 100644 --- a/package.json +++ b/package.json @@ -5,12 +5,17 @@ "description": "A common library to share code between projects in the e-mission-platform. Written in Python and exposed as both Python and JavaScript.", "main": "./emcommon_js/index.js", "types": "./emcommon_js/dist/index.d.ts", + "scripts": { + "test": "jest" + }, "jest": { "testMatch": [ "**/test_*.js" ], "globals": { "window": {} }, - "transform": {} + "transform": { + "^.+\\.jsx?$": "babel-jest" + } }, "prettier": { "printWidth": 100, @@ -25,8 +30,18 @@ "dependencies": { "color": "^3.1.2" }, + "babel": { + "presets": [ + "@babel/preset-env" + ] + }, "devDependencies": { "@types/jest": "^29.5.12", + "@babel/preset-env": "^7.14.5", + "@babel/preset-react": "^7.14.5", + "@babel/preset-typescript": "^7.14.5", + "@babel/plugin-transform-runtime": "^7.14.5", + "babel-jest": "^27.0.0", "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", "mapshaper": "^0.6.99", From 8ce82d2e1e2aa7a8c451d2ff4143939120dc9836 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 00:31:41 -0400 Subject: [PATCH 07/22] Remove .babelrc --- .babelrc | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .babelrc diff --git a/.babelrc b/.babelrc deleted file mode 100644 index a1b6dc2..0000000 --- a/.babelrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "presets": ["@babel/preset-env"] - } From 65820a674891a484f75c4c6332b2ef41cad9cf18 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 02:35:42 -0400 Subject: [PATCH 08/22] Try to see the exception in github actions --- .../emcommon.metrics.footprint.util.js | 52 ++++++++++--------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/emcommon_js/emcommon.metrics.footprint.util.js b/emcommon_js/emcommon.metrics.footprint.util.js index 95215e5..ce217a0 100644 --- a/emcommon_js/emcommon.metrics.footprint.util.js +++ b/emcommon_js/emcommon.metrics.footprint.util.js @@ -88,29 +88,33 @@ export var get_egrid_region = async function (coords, year) { return null; }; export var get_uace_by_coords = async function (coords, year) { - var census_year = year - __mod__ (year, 10); - var url = ('https://geocoding.geo.census.gov/geocoder/geographies/coordinates?' + 'x={}&y={}'.format (coords [0], coords [1])) + '&benchmark=Public_AR_Current&vintage=Census{}_Current&layers=87&format=json'.format (census_year); - try { - var data = await fetch_url (url); - } - catch (__except0__) { - if (isinstance (__except0__, Exception)) { - var e = __except0__; - Log.error ('Failed to geocode {} in year {}: {}'.format (coords, year, e)); - return null; - } - else Log.error ('Failed to geocode {} in year {}'.format (coords, year)); - return null; - } - for (var g in data ['result'] ['geographies']) { - for (var entry of data ['result'] ['geographies'] [g]) { - if (__in__ ('UA', entry)) { - return entry ['UA']; - } - } - } - Log.warn ('Urban Area not in geocoding response for coords {} in year {}: {}'.format (coords, year, url)); - return null; + var census_year = year - __mod__ (year, 10); + var url = ('https://geocoding.geo.census.gov/geocoder/geographies/coordinates?' + 'x={}&y={}'.format (coords [0], coords [1])) + '&benchmark=Public_AR_Current&vintage=Census{}_Current&layers=87&format=json'.format (census_year); + try { + var data = await fetch_url (url); + } + catch (__except0__) { + if (isinstance (__except0__, Exception)) { + var e = __except0__; + Log.error ('Failed to geocode {} in year {}: {}'.format (coords, year, e)); + console.error('Exception:', e); // Print the exception + return null; + } + else { + Log.error ('Failed to geocode {} in year {}'.format (coords, year)); + console.error('Exception:', __except0__); // Print the exception + return null; + } + } + for (var g in data ['result'] ['geographies']) { + for (var entry of data ['result'] ['geographies'] [g]) { + if (__in__ ('UA', entry)) { + return entry ['UA']; + } + } + } + Log.warn ('Urban Area not in geocoding response for coords {} in year {}: {}'.format (coords, year, url)); + return null; }; export var get_intensities_data = async function (year, dataset) { if (year < 2018) { @@ -178,4 +182,4 @@ export var merge_metadatas = function (meta_a, meta_b) { } }; -//# sourceMappingURL=emcommon.metrics.footprint.util.map \ No newline at end of file +//# sourceMappingURL=emcommon.metrics.footprint.util.map From 9ff55814a4681a103bdcce5b46fed5dee1a7f109 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 04:18:30 -0400 Subject: [PATCH 09/22] Debug no fetch --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index d38b3ef..0fa5c96 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -65,7 +65,7 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v2 with: - node-version: '14' + node-version: '22' - name: Install dependencies run: npm install From 4676309403f1267ad2ab68a0c6686977a3369e41 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 04:22:01 -0400 Subject: [PATCH 10/22] Reset util.js --- .../workflows/{pytest.yml => unit-tests.yml} | 0 .../emcommon.metrics.footprint.util.js | 50 +++++++++---------- 2 files changed, 23 insertions(+), 27 deletions(-) rename .github/workflows/{pytest.yml => unit-tests.yml} (100%) diff --git a/.github/workflows/pytest.yml b/.github/workflows/unit-tests.yml similarity index 100% rename from .github/workflows/pytest.yml rename to .github/workflows/unit-tests.yml diff --git a/emcommon_js/emcommon.metrics.footprint.util.js b/emcommon_js/emcommon.metrics.footprint.util.js index ce217a0..db9e34a 100644 --- a/emcommon_js/emcommon.metrics.footprint.util.js +++ b/emcommon_js/emcommon.metrics.footprint.util.js @@ -88,33 +88,29 @@ export var get_egrid_region = async function (coords, year) { return null; }; export var get_uace_by_coords = async function (coords, year) { - var census_year = year - __mod__ (year, 10); - var url = ('https://geocoding.geo.census.gov/geocoder/geographies/coordinates?' + 'x={}&y={}'.format (coords [0], coords [1])) + '&benchmark=Public_AR_Current&vintage=Census{}_Current&layers=87&format=json'.format (census_year); - try { - var data = await fetch_url (url); - } - catch (__except0__) { - if (isinstance (__except0__, Exception)) { - var e = __except0__; - Log.error ('Failed to geocode {} in year {}: {}'.format (coords, year, e)); - console.error('Exception:', e); // Print the exception - return null; - } - else { - Log.error ('Failed to geocode {} in year {}'.format (coords, year)); - console.error('Exception:', __except0__); // Print the exception - return null; - } - } - for (var g in data ['result'] ['geographies']) { - for (var entry of data ['result'] ['geographies'] [g]) { - if (__in__ ('UA', entry)) { - return entry ['UA']; - } - } - } - Log.warn ('Urban Area not in geocoding response for coords {} in year {}: {}'.format (coords, year, url)); - return null; + var census_year = year - __mod__ (year, 10); + var url = ('https://geocoding.geo.census.gov/geocoder/geographies/coordinates?' + 'x={}&y={}'.format (coords [0], coords [1])) + '&benchmark=Public_AR_Current&vintage=Census{}_Current&layers=87&format=json'.format (census_year); + try { + var data = await fetch_url (url); + } + catch (__except0__) { + if (isinstance (__except0__, Exception)) { + var e = __except0__; + Log.error ('Failed to geocode {} in year {}: {}'.format (coords, year, e)); + return null; + } + else Log.error ('Failed to geocode {} in year {}'.format (coords, year)); + return null; + } + for (var g in data ['result'] ['geographies']) { + for (var entry of data ['result'] ['geographies'] [g]) { + if (__in__ ('UA', entry)) { + return entry ['UA']; + } + } + } + Log.warn ('Urban Area not in geocoding response for coords {} in year {}: {}'.format (coords, year, url)); + return null; }; export var get_intensities_data = async function (year, dataset) { if (year < 2018) { From 2c1698d558a1ac8aa824caf187f2ab5c840644f1 Mon Sep 17 00:00:00 2001 From: jpfleischer <70083705+jpfleischer@users.noreply.github.com> Date: Fri, 20 Sep 2024 04:32:02 -0400 Subject: [PATCH 11/22] Update emcommon.metrics.footprint.util.js From 85ad3caca88596d1eedb2408c9e4191b69cb5ebb Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 04:33:23 -0400 Subject: [PATCH 12/22] Fix end of line --- emcommon_js/emcommon.metrics.footprint.util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emcommon_js/emcommon.metrics.footprint.util.js b/emcommon_js/emcommon.metrics.footprint.util.js index db9e34a..95215e5 100644 --- a/emcommon_js/emcommon.metrics.footprint.util.js +++ b/emcommon_js/emcommon.metrics.footprint.util.js @@ -178,4 +178,4 @@ export var merge_metadatas = function (meta_a, meta_b) { } }; -//# sourceMappingURL=emcommon.metrics.footprint.util.map +//# sourceMappingURL=emcommon.metrics.footprint.util.map \ No newline at end of file From 400431a3e9fdc35945e8d501415e222a4910da6f Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 04:36:46 -0400 Subject: [PATCH 13/22] Simplify workflow jobs --- .github/workflows/unit-tests.yml | 32 ++++---------------------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 0fa5c96..dd32079 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -22,39 +22,15 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 - - name: Install and start MongoDB - uses: supercharge/mongodb-github-action@1.3.0 - with: - mongodb-version: 4.4.0 - - - uses: conda-incubator/setup-miniconda@v2 - with: - miniconda-version: "latest" - channels: bioconda, conda-forge, defaults - use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly! - auto-update-conda: true - auto-activate-base: true - - - name: Check existing version of miniconda - shell: bash -l {0} - run: conda info -a - - - name: Install emcommon conda environment - shell: bash -l {0} - run: | - # echo Test - conda env update -n emcommon -f bin/environment.yml - - - name: Show current conda environment + - name: Run setup shell: bash -l {0} run: | - conda activate emcommon && conda info --envs + . bin/setup.sh - - name: Switch to emcommon and run the tests + - name: Run pytest shell: bash -l {0} run: | - conda activate emcommon - PYTHONPATH=./src pytest test --asyncio-mode=auto + . bin/run_pytest.sh test-jest: runs-on: ubuntu-latest From f0c6af91f12a65181b4ed787d2c799339ff332e3 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 04:41:38 -0400 Subject: [PATCH 14/22] Add miniconda --- .github/workflows/unit-tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index dd32079..a90f12e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -22,6 +22,14 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v2 + - name: Set up Miniconda + uses: conda-incubator/setup-miniconda@v3 + with: + miniconda-version: "latest" + auto-update-conda: true + auto-activate-base: true + + - name: Run setup shell: bash -l {0} run: | From e880ad879bf2fb889c1c8b6e35574084e30a6e29 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 04:45:06 -0400 Subject: [PATCH 15/22] Debug --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index a90f12e..096ffbc 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -33,7 +33,7 @@ jobs: - name: Run setup shell: bash -l {0} run: | - . bin/setup.sh + set -x && . bin/setup.sh - name: Run pytest shell: bash -l {0} From 1f175008bac9c2d90c7d387a20a8a2f5f7741b6d Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 04:55:23 -0400 Subject: [PATCH 16/22] Add architecture to platform --- bin/setup.sh | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/bin/setup.sh b/bin/setup.sh index d451d47..2996612 100644 --- a/bin/setup.sh +++ b/bin/setup.sh @@ -1,5 +1,36 @@ #!/bin/bash +# Check if conda exists +if ! command -v conda &> /dev/null; then + echo "conda could not be found" + return 1 +fi + +# Determine platform and architecture +ARCH=$(uname -m) +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + if [[ "$ARCH" == "x86_64" ]]; then + TARGET_PLATFORM="Linux-x86_64" + elif [[ "$ARCH" == "aarch64" ]]; then + TARGET_PLATFORM="Linux-aarch64" + else + echo "Unsupported architecture $ARCH on Linux" + exit 1 + fi +elif [[ "$OSTYPE" == "darwin"* ]]; then + if [[ "$ARCH" == "x86_64" ]]; then + TARGET_PLATFORM="MacOSX-x86_64" + elif [[ "$ARCH" == "arm64" ]]; then + TARGET_PLATFORM="MacOSX-arm64" + else + echo "Unsupported architecture $ARCH on macOS" + exit 1 + fi +else + echo "Unsupported platform $OSTYPE" + exit 1 +fi + # copy the 'setup' directory from the e-mission-server repo git clone -n --depth=1 --filter=tree:0 https://github.com/e-mission/e-mission-server.git cd e-mission-server @@ -9,16 +40,6 @@ cp -r setup ../ cd .. rm -rf e-mission-server -# determine platform -if [[ "$OSTYPE" == "linux-gnu"* ]]; then - TARGET_PLATFORM="linux" -elif [[ "$OSTYPE" == "darwin"* ]]; then - TARGET_PLATFORM="mac" -else - echo "Unsupported platform $OSTYPE" - exit 1 -fi - # set up conda using the e-mission-server scripts . setup/setup_conda.sh $TARGET_PLATFORM . setup/activate_conda.sh From 60ff612b075aa4ef9595ba3bdc905d9e37b14789 Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 04:57:40 -0400 Subject: [PATCH 17/22] Elaborate on prerequisites for README --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4dc3ab1..d682a13 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ This repository uses the [Transcrypt](https://www.transcrypt.org/) library to co ## Setup -``` +Anaconda is required. + +```bash . bin/setup.sh ``` @@ -21,7 +23,7 @@ Re-run this if you change the dependencies in `environment.yml` or `package.json ## To contribute 1. Make your changes to Python code under the `src` directory. -1. Run `bash bin/compile_to_js.sh` to build the JavaScript. This will produce output JS files in the `emcommon_js` directory. +1. Run `bash bin/compile_to_js.sh` to build the JavaScript. This will produce output JS files in the `emcommon_js` directory. However, `pip install transcrypt` is required. 1. Commit changes from both the `src` and `emcommon_js` directories to your branch. ## Tips for writing code to work in both Python and JavaScript From 462e5d644e2eb99627a2da1e573b3ac7e2d9606f Mon Sep 17 00:00:00 2001 From: jpfleischer Date: Fri, 20 Sep 2024 04:58:57 -0400 Subject: [PATCH 18/22] Simplify workflow yaml --- .github/workflows/unit-tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 096ffbc..51b464d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -3,7 +3,6 @@ name: pytest # Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch on: push: pull_request: @@ -29,7 +28,6 @@ jobs: auto-update-conda: true auto-activate-base: true - - name: Run setup shell: bash -l {0} run: | From 27d999917386708617b2553a15cc763fa7c319f2 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 23 Oct 2024 12:23:17 -0400 Subject: [PATCH 19/22] remove unneeded files/packages export_versions.sh is in the 'setup' folder of e-mission-server and will be downloaded dynamically via bin/setup.sh; does not need to be duplicated here We do not need babel to run the jest tests because Transcrypt outputs jest-compliant JS --- bin/export_versions.sh | 4 ---- package.json | 17 +---------------- 2 files changed, 1 insertion(+), 20 deletions(-) delete mode 100644 bin/export_versions.sh diff --git a/bin/export_versions.sh b/bin/export_versions.sh deleted file mode 100644 index 95d4dda..0000000 --- a/bin/export_versions.sh +++ /dev/null @@ -1,4 +0,0 @@ -# taken from https://github.com/e-mission/e-mission-server/blob/master/setup/export_versions.sh - -export EXP_CONDA_VER=23.5.2 -export EXP_CONDA_VER_SUFFIX=0 diff --git a/package.json b/package.json index 387f553..8c023d8 100644 --- a/package.json +++ b/package.json @@ -5,17 +5,12 @@ "description": "A common library to share code between projects in the e-mission-platform. Written in Python and exposed as both Python and JavaScript.", "main": "./emcommon_js/index.js", "types": "./emcommon_js/dist/index.d.ts", - "scripts": { - "test": "jest" - }, "jest": { "testMatch": [ "**/test_*.js" ], "globals": { "window": {} }, - "transform": { - "^.+\\.jsx?$": "babel-jest" - } + "transform": {} }, "prettier": { "printWidth": 100, @@ -30,18 +25,8 @@ "dependencies": { "color": "^3.1.2" }, - "babel": { - "presets": [ - "@babel/preset-env" - ] - }, "devDependencies": { "@types/jest": "^29.5.12", - "@babel/preset-env": "^7.14.5", - "@babel/preset-react": "^7.14.5", - "@babel/preset-typescript": "^7.14.5", - "@babel/plugin-transform-runtime": "^7.14.5", - "babel-jest": "^27.0.0", "jest": "^29.7.0", "jest-environment-jsdom": "^29.7.0", "mapshaper": "^0.6.99", From 4a05a62d17be2bd680bfa2c615a268dd68837633 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 23 Oct 2024 12:27:05 -0400 Subject: [PATCH 20/22] Revert "remove unit-tests.yml experiment" This reverts commit a9fdffd816c860b657c7e46377184d4f790238b1. --- .github/workflows/unit-tests.yml | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000..3691eca --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,48 @@ +# taken from https://github.com/e-mission/e-mission-server/blob/master/.github/workflows/test-with-manual-install.yml + +name: unit-tests + +# Controls when the action will run. Triggers the workflow on push or pull request +on: + push: + pull_request: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + test-pytest: + # The type of runner that the job will run on + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Run setup.sh + shell: bash -l {0} + run: | + source bin/setup.sh + + - name: Run pytest + shell: bash -l {0} + run: | + source bin/run_pytest.sh + + test-jest: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Node.js + uses: actions/setup-node@v2 + with: + node-version: '20' + + - name: Run setup.sh + shell: bash -l {0} + run: | + source bin/setup.sh + + - name: Run jest tests + shell: bash -l {0} + run: | + source bin/run_jest.sh From 4316185d9b6a67e648b83a75520accf5c771d1b5 Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 23 Oct 2024 12:30:35 -0400 Subject: [PATCH 21/22] update README, remove unneeded notes, add GH Actions section --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d682a13..b1f891d 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,6 @@ This repository uses the [Transcrypt](https://www.transcrypt.org/) library to co ## Setup -Anaconda is required. - ```bash . bin/setup.sh ``` @@ -23,7 +21,7 @@ Re-run this if you change the dependencies in `environment.yml` or `package.json ## To contribute 1. Make your changes to Python code under the `src` directory. -1. Run `bash bin/compile_to_js.sh` to build the JavaScript. This will produce output JS files in the `emcommon_js` directory. However, `pip install transcrypt` is required. +1. Run `bash bin/compile_to_js.sh` to build the JavaScript. This will produce output JS files in the `emcommon_js` directory. 1. Commit changes from both the `src` and `emcommon_js` directories to your branch. ## Tips for writing code to work in both Python and JavaScript @@ -128,3 +126,7 @@ There may be testing scenarios that must significantly diverge between Python an ```bash . bin/run_jest.sh ``` + +### GitHub Actions + +The unit tests also run via an Actions workflow, which executes both `run_pytest.sh` and `run_jest.sh` on each commit or PR to `master`. From 9ef8349e588d422d531f60b852ea84cd00fe4c8b Mon Sep 17 00:00:00 2001 From: Jack Greenlee Date: Wed, 23 Oct 2024 12:39:04 -0400 Subject: [PATCH 22/22] make all bin/*.sh scripts executable --- bin/compile_to_js.sh | 0 bin/run_jest.sh | 0 bin/setup.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/compile_to_js.sh mode change 100644 => 100755 bin/run_jest.sh mode change 100644 => 100755 bin/setup.sh diff --git a/bin/compile_to_js.sh b/bin/compile_to_js.sh old mode 100644 new mode 100755 diff --git a/bin/run_jest.sh b/bin/run_jest.sh old mode 100644 new mode 100755 diff --git a/bin/setup.sh b/bin/setup.sh old mode 100644 new mode 100755