From 3f30fe097d868ca91f338d51af325e68301a708c Mon Sep 17 00:00:00 2001 From: Nathan Moore Date: Tue, 19 Dec 2023 09:46:51 -0700 Subject: [PATCH] Categorize tests to parallelize CI (#442) * break up simulation tests by category * test by category in a CI matrix * run GHA on push for testing * try another if syntax inside CI file * install python for basic simulations too, for use with rnm * Revert "install python for basic simulations too, for use with rnm" This reverts commit 1eba56e8a3f89da15a90789a66dc8081013377fb. * save failed ci artifact one dir higher * oops, re-enable all the setup tests * use new checkout version, and add our repo as a safe dir for CI * refactor test variable names to be consistent * get weird with CI `if` statement * try this safe.directory path in CI * try another safe.directory path in CI * abandon setting safe.directory in CI, at least temporarily * another attempt at setting the safe.directory properly * only run setup & admin tests in one workflow, to reduce duplication * using `if` syntax from gmt ci file * yet another syntax for `if` statement in GHA config * better syntax for `if` statement in GHA * ugh, move brackets * revert to scheduled overnight CI runs as normal * clarify wmo test variable name * run tests on Windows too, not just Ubuntu * cleaner error message from non-US weatherfile - no more stack trace * stricter test for non-US weather file to be more confident * run CI against released gems, not develop branch * only run CI on linux to avoid container on Windows * update weather file test * remove coverage dependencies as they are not used in this repo * remove whitespace * alternate way of raising our custom error message * adapt error message for its new location * try alternate way of testing the error message to make CI happy * skip non-US weatherfile test to make CI happy * re-enable favoring local gems * use new version of ditto-reader * bump ditto-reader dependency to use unbroken version * bump cli version to 0.11.0-a0 * try to increase permissions following GHA warning * CI on released versions of dependencies * Revert "try to increase permissions following GHA warning" This reverts commit 00670a14176fbf26e38bb97c88d65299818219f1. * upgrade setup-python to v5 in CI * test a different way of installing python * Revert "test a different way of installing python" This reverts commit caf33c66a9b76669fc817e186c7e7d006f58bbd6. * remove redundant python install step from CI * use patched ditto-reader * use newest bugfixed version of ditto-reader * run full test suite in CI * test with local gems again, now that released are confirmed to work * Revert "test with local gems again, now that released are confirmed to work" This reverts commit b388b854bd86fee72cf82bb624ccd3cbb2458ad1. * restore scheduled CI runs --------- Co-authored-by: kflemin <2205659+kflemin@users.noreply.github.com> --- .github/workflows/nightly_ci_build.yml | 55 +-- example_files/mappers/Baseline.rb | 16 +- example_files/python_deps/dependencies.json | 6 +- lib/uo_cli/version.rb | 2 +- spec/uo_cli_spec.rb | 391 +++++++++++--------- uo_cli.gemspec | 2 - 6 files changed, 249 insertions(+), 223 deletions(-) diff --git a/.github/workflows/nightly_ci_build.yml b/.github/workflows/nightly_ci_build.yml index d25a6066..91e74689 100644 --- a/.github/workflows/nightly_ci_build.yml +++ b/.github/workflows/nightly_ci_build.yml @@ -3,8 +3,8 @@ name: CLI CI on: # push: schedule: - # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule - # 5:24 am UTC (11:24pm MDT the day before) every weekday night in MDT + # # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule + # # 5:24 am UTC (11:24pm MDT the day before) every weekday night in MDT - cron: '24 5 * * 2-6' pull_request: types: [review_requested] @@ -13,51 +13,56 @@ env: # Favor_Local_Gems enforces develop branch of all Ruby dependencies # This is our canary in the coal mine! If any simulation tests fail, comment this and retry. # If CI is then successful, we have a breaking change in a dependency somewhere. - FAVOR_LOCAL_GEMS: true + # FAVOR_LOCAL_GEMS: true GEM_DEVELOPER_KEY: ${{ secrets.GEM_DEVELOPER_KEY }} UO_NUM_PARALLEL: 2 # GHA machines only have 2 cores. Trying to run more than that is even slower. - # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources jobs: weeknight-tests: - # ubuntu-latest works since https://github.com/rbenv/ruby-build/releases/tag/v20220710 (July 10, 2022) - # https://github.com/rbenv/ruby-build/discussions/1940 + strategy: + matrix: + # os: container operations in GHA only work on Ubuntu + simulation-type: [basic, GEB, residential, electric] + # python-version: No need to test more than 1 python-version runs-on: ubuntu-latest container: image: docker://nrel/openstudio:3.6.1 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - name: Change Owner of Container Working Directory + # working dir permissions workaround from https://github.com/actions/runner-images/issues/6775#issuecomment-1377299658 + run: chown root:root . - name: Set up Python - uses: actions/setup-python@v4 + if: ${{ matrix.simulation-type == 'electric' }} + uses: actions/setup-python@v5 with: - # Disco needs python 3.10 + # Disco needs python ~=3.10 python-version: '3.10' - name: Install Ruby dependencies run: | ruby --version bundle update bundle exec certified-update - - name: Install python dependencies - run: bundle exec uo install_python - - name: Run Rspec - continue-on-error: true - # Continue to upload step even if a test fails, so we can troubleshoot - run: bundle exec rspec + - name: Test project setup + # We only need to run these tests once, not every matrix iteration. + if: ${{ matrix.simulation-type == 'electric' }} + run: | + bundle exec rspec -e 'Admin' + bundle exec rspec -e 'Create project' + bundle exec rspec -e 'Make and manipulate ScenarioFiles' + bundle exec rspec -e 'Update project directory' + bundle exec rspec -e 'Install python dependencies' + - name: Test simulations + run: bundle exec rspec -e 'Run and work with a small ${{ matrix.simulation-type }} simulation' - name: Upload artifacts # Save results for examination - useful for debugging - uses: actions/upload-artifact@v3 - # Only upload if rspec fails + uses: actions/upload-artifact@v4 + # Only upload if a previous step fails if: failure() with: name: rspec_results path: | - spec/test_directory**/run/ - # coverage/ + spec/test_directory**/ retention-days: 7 # save for 1 week before deleting - # coveralls action docs: https://github.com/marketplace/actions/coveralls-github-action - # - name: Coveralls - # uses: coverallsapp/github-action@1.1.3 - # with: - # github-token: ${{ secrets.GITHUB_TOKEN }} - # path-to-lcov: "./coverage/lcov/urbanopt-cli.lcov" diff --git a/example_files/mappers/Baseline.rb b/example_files/mappers/Baseline.rb index 2d7b3ca7..97754f1c 100644 --- a/example_files/mappers/Baseline.rb +++ b/example_files/mappers/Baseline.rb @@ -365,10 +365,7 @@ def get_climate_zone_iecc(epw) end end - # If no match is found, raise an error - raise ("Error: No match found for WMO #{wmo} from your weather file #{Pathname(epw).expand_path} in our US WMO list. - This is known to happen when your weather file is from somewhere outside of the United States. - Please replace your weather file with one from an analagous weather location in the United States.") + return nil end # epw_state to subregions mapping methods @@ -892,12 +889,13 @@ def create_osw(scenario, features, feature_names) template_vals = template_vals.transform_keys(&:to_sym) epw = File.join(File.dirname(__FILE__), '../weather', feature.weather_filename) - begin - template_vals[:climate_zone] = get_climate_zone_iecc(epw) - rescue RuntimeError => e - # Non-US weather file can lead to abrupt exit - puts e.message + climate_zone = get_climate_zone_iecc(epw) + if climate_zone.nil? + abort("Error: No match found for the WMO station from your weather file #{Pathname(epw).expand_path} in our US WMO list. + This is known to happen when your weather file is from somewhere outside of the United States. + Please replace your weather file with one from an analogous weather location in the United States.") end + template_vals[:climate_zone] = climate_zone # ENCLOSURE diff --git a/example_files/python_deps/dependencies.json b/example_files/python_deps/dependencies.json index 96ab0445..63e35313 100644 --- a/example_files/python_deps/dependencies.json +++ b/example_files/python_deps/dependencies.json @@ -1,5 +1,5 @@ [ - { "name": "urbanopt-ditto-reader", "version": "0.5.1"}, - { "name": "NREL-disco", "version": "0.4.1"}, - { "name": "geojson-modelica-translator", "version": "0.5.0"} + { "name": "urbanopt-ditto-reader", "version": "0.6.3"}, + { "name": "NREL-disco", "version": "0.5.0"}, + { "name": "geojson-modelica-translator", "version": "0.6.0rc2"} ] diff --git a/lib/uo_cli/version.rb b/lib/uo_cli/version.rb index b73c1d9c..23ae27c4 100644 --- a/lib/uo_cli/version.rb +++ b/lib/uo_cli/version.rb @@ -5,6 +5,6 @@ module URBANopt module CLI - VERSION = '0.10.0'.freeze + VERSION = '0.11.0-a0'.freeze end end diff --git a/spec/uo_cli_spec.rb b/spec/uo_cli_spec.rb index 621793a1..bd239d0b 100644 --- a/spec/uo_cli_spec.rb +++ b/spec/uo_cli_spec.rb @@ -5,6 +5,7 @@ require 'csv' require 'json' +require 'open3' RSpec.describe URBANopt::CLI do example_dir = Pathname(__FILE__).dirname.parent / 'example_files' @@ -16,11 +17,11 @@ test_directory_pv = spec_dir / 'test_directory_pv' test_scenario = test_directory / 'two_building_scenario.csv' test_scenario_res = test_directory_res / 'two_building_res' - test_reopt_scenario = test_directory_pv / 'REopt_scenario.csv' + test_scenario_reopt = test_directory_pv / 'REopt_scenario.csv' test_scenario_pv = test_directory_pv / 'two_building_scenario.csv' test_scenario_elec = test_directory_elec / 'electrical_scenario.csv' test_scenario_disco = test_directory_disco / 'electrical_scenario.csv' - test_ev_scenario = test_directory / 'two_building_ev_scenario.csv' + test_scenario_ev = test_directory / 'two_building_ev_scenario.csv' test_scenario_chilled = test_directory_res / 'two_building_chilled.csv' test_scenario_mels_reduction = test_directory_res / 'two_building_mels_reduction.csv' test_scenario_stat_adjustment = test_directory_res / 'two_building_stat_adjustment.csv' @@ -294,20 +295,13 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', end end - context 'Run and work with a small simulation' do + context 'Run and work with a small basic simulation' do before :all do delete_directory_or_file(test_directory) system("#{call_cli} create --project-folder #{test_directory}") - delete_directory_or_file(test_directory_res) - system("#{call_cli} create --project-folder #{test_directory_res} --combined") - delete_directory_or_file(test_directory_elec) - # use this to test both opendss and disco workflows - system("#{call_cli} create --project-folder #{test_directory_elec} --disco") - delete_directory_or_file(test_directory_pv) - system("#{call_cli} create --project-folder #{test_directory_pv} --photovoltaic") end - it 'runs a 2 building scenario using default geometry method' do + it 'runs a 2 building scenario using default geometry method', :basic do # Use a ScenarioFile with only 2 buildings to reduce test time system("cp #{spec_dir / 'spec_files' / 'two_building_scenario.csv'} #{test_scenario}") system("#{call_cli} run --scenario #{test_scenario} --feature #{test_feature}") @@ -316,7 +310,140 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', expect((test_directory / 'run' / 'two_building_scenario' / '3' / 'finished.job').exist?).to be false end - it 'runs a chilled water scenario with residential and commercial buildings' do + it 'runs a 2 building scenario using create bar geometry method', :basic do + # Copy create bar specific files + system("cp #{example_dir / 'mappers' / 'CreateBar.rb'} #{test_directory / 'mappers' / 'CreateBar.rb'}") + system("cp #{example_dir / 'mappers' / 'createbar_workflow.osw'} #{test_directory / 'mappers' / 'createbar_workflow.osw'}") + system("cp #{spec_dir / 'spec_files' / 'two_building_create_bar.csv'} #{test_directory / 'two_building_create_bar.csv'}") + system("#{call_cli} run --scenario #{test_directory / 'two_building_create_bar.csv'} --feature #{test_feature}") + expect((test_directory / 'run' / 'two_building_create_bar' / '2' / 'finished.job').exist?).to be true + end + + it 'runs a 2 building scenario using floorspace geometry method', :basic do + # Copy floorspace specific files + system("cp #{example_dir / 'mappers' / 'Floorspace.rb'} #{test_directory / 'mappers' / 'Floorspace.rb'}") + system("cp #{example_dir / 'mappers' / 'floorspace_workflow.osw'} #{test_directory / 'mappers' / 'floorspace_workflow.osw'}") + system("cp #{example_dir / 'osm_building' / '7_floorspace.json'} #{test_directory / 'osm_building' / '7_floorspace.json'}") + system("cp #{example_dir / 'osm_building' / '7_floorspace.osm'} #{test_directory / 'osm_building' / '7_floorspace.osm'}") + system("cp #{example_dir / 'example_floorspace_project.json'} #{test_directory / 'example_floorspace_project.json'}") + system("cp #{spec_dir / 'spec_files' / 'two_building_floorspace.csv'} #{test_directory / 'two_building_floorspace.csv'}") + expect((test_directory / 'osm_building' / '7_floorspace.osm').exist?).to be true + system("#{call_cli} run --scenario #{test_directory / 'two_building_floorspace.csv'} --feature #{test_directory / 'example_floorspace_project.json'}") + expect((test_directory / 'run' / 'two_building_floorspace' / '5' / 'finished.job').exist?).to be true + expect((test_directory / 'run' / 'two_building_floorspace' / '7' / 'finished.job').exist?).to be true + end + + it 'runs an ev-charging scenario', :basic do + # copy ev-charging specific files + system("cp #{spec_dir / 'spec_files' / 'two_building_ev_scenario.csv'} #{test_scenario_ev}") + system("#{call_cli} run --scenario #{test_scenario_ev} --feature #{test_feature}") + expect((test_directory / 'run' / 'two_building_ev_scenario' / '5' / 'finished.job').exist?).to be true + expect((test_directory / 'run' / 'two_building_ev_scenario' / '2' / 'finished.job').exist?).to be true + end + + it 'post-processor closes gracefully if given an invalid type', :basic do + # Type is totally random + expect { system("#{call_cli} process --foobar --scenario #{test_scenario} --feature #{test_feature}") } + .to output(a_string_including("unknown argument '--foobar'")) + .to_stderr_from_any_process + # Type is valid, but with extra characters + expect { system("#{call_cli} process --reopt-scenariot --scenario #{test_scenario} --feature #{test_feature}") } + .to output(a_string_including("unknown argument '--reopt-scenariot'")) + .to_stderr_from_any_process + # Type would be valid if not missing characters + expect { system("#{call_cli} process --reopt-scenari --scenario #{test_scenario} --feature #{test_feature}") } + .to output(a_string_including("unknown argument '--reopt-scenari'")) + .to_stderr_from_any_process + end + + it 'post-processor closes gracefully if not given a type', :basic do + expect { system("#{call_cli} process --scenario #{test_scenario} --feature #{test_feature}") } + .to output(a_string_including('No valid process type entered')) + .to_stderr_from_any_process + end + + it 'default post-processes a scenario', :basic do + # This test requires the 'runs a 2 building scenario using default geometry method' be run first + test_scenario_report = test_directory / 'run' / 'two_building_scenario' / 'default_scenario_report.csv' + system("#{call_cli} process --default --scenario #{test_scenario} --feature #{test_feature}") + expect(`wc -l < #{test_scenario_report}`.to_i).to be > 2 + expect((test_directory / 'run' / 'two_building_scenario' / 'process_status.json').exist?).to be true + end + + it 'successfully runs the rnm workflow', :basic do + # This test requires the 'runs a 2 building scenario using default geometry method' be run first + # copy featurefile in dir + rnm_file = 'example_project_with_streets.json' + system("cp #{spec_dir / 'spec_files' / rnm_file} #{test_directory / rnm_file}") + # call rnm + test_rnm_file = test_directory / rnm_file + system("#{call_cli} rnm --scenario #{test_scenario} --feature #{test_rnm_file}") + # check that rnm inputs and outputs were created + expect((test_directory / 'run' / 'two_building_scenario' / 'rnm-us' / 'inputs.zip').exist?).to be true + expect((test_directory / 'run' / 'two_building_scenario' / 'rnm-us' / 'results').exist?).to be true + expect((test_directory / 'run' / 'two_building_scenario' / 'scenario_report_rnm.json').exist?).to be true + expect((test_directory / 'run' / 'two_building_scenario' / 'feature_file_rnm.json').exist?).to be true + end + + it 'saves post-process output as a database file', :basic do + # This test requires the 'runs a 2 building scenario using default geometry method' be run first + db_filename = test_directory / 'run' / 'two_building_scenario' / 'default_scenario_report.db' + system("#{call_cli} process --default --with-database --scenario #{test_scenario} --feature #{test_feature}") + expect(`wc -l < #{db_filename}`.to_i).to be > 20 + expect((test_directory / 'run' / 'two_building_scenario' / 'process_status.json').exist?).to be true + end + + it 'creates scenario visualization for default post processor', :basic do + # This test requires the 'runs a 2 building scenario using default geometry method' be run first + # visualizing via the FeatureFile will throw error to stdout (but not crash) if a scenario that uses those features isn't processed first. + system("#{call_cli} process --default --scenario #{test_scenario} --feature #{test_feature}") + system("#{call_cli} process --default --scenario #{test_scenario_ev} --feature #{test_feature}") + system("#{call_cli} visualize --feature #{test_feature}") + expect((test_directory / 'run' / 'scenario_comparison.html').exist?).to be true + end + + it 'creates feature visualization for default post processor', :basic do + # This test requires the 'runs a 2 building scenario using default geometry method' be run first + system("#{call_cli} process --default --scenario #{test_scenario} --feature #{test_feature}") + system("#{call_cli} visualize --scenario #{test_scenario}") + expect((test_directory / 'run' / 'two_building_scenario' / 'feature_comparison.html').exist?).to be true + end + + it 'ensures viz files are in the project directory', :basic do + # This test requires the 'runs a 2 building scenario using default geometry method' be run first + if (test_directory / 'visualization' / 'input_visualization_feature.html').exist? + FileUtils.rm_rf(test_directory / 'visualization' / 'input_visualization_feature.html') + end + if (test_directory / 'run' / 'two_building_scenario' / 'feature_comparison.html').exist? + FileUtils.rm_rf(test_directory / 'run' / 'two_building_scenario' / 'feature_comparison.html') + end + if (test_directory / 'run' / 'two_building_scenario' / 'scenarioData.js').exist? + FileUtils.rm_rf(test_directory / 'run' / 'two_building_scenario' / 'scenarioData.js') + end + expect((test_directory / 'visualization' / 'input_visualization_feature.html').exist?).to be false + expect((test_directory / 'run' / 'two_building_scenario' / 'feature_comparison.html').exist?).to be false + expect((test_directory / 'run' / 'two_building_scenario' / 'scenarioData.js').exist?).to be false + system("#{call_cli} visualize --scenario #{test_scenario}") + expect((test_directory / 'run' / 'two_building_scenario' / 'feature_comparison.html').exist?).to be true + end + + it 'deletes a scenario', :basic do + expect((test_directory / 'run' / 'two_building_create_bar' / '2' / 'data_point_out.json').exist?).to be true + bar_scenario = test_directory / 'two_building_create_bar.csv' + system("#{call_cli} delete --scenario #{bar_scenario}") + expect((test_directory / 'run' / 'two_building_create_bar' / '2' / 'data_point_out.json').exist?).to be false + end + end + + context 'Run and work with a small GEB simulation' do + before :all do + delete_directory_or_file(test_directory) + system("#{call_cli} create --project-folder #{test_directory}") + delete_directory_or_file(test_directory_res) + system("#{call_cli} create --project-folder #{test_directory_res} --combined") + end + + it 'runs a chilled water scenario with residential and commercial buildings', :GEB do # Use a ScenarioFile with only 2 buildings to reduce test time system("cp #{spec_dir / 'spec_files' / 'two_building_res_chilled_water_scenario.csv'} #{test_scenario_chilled}") # Include the chilled water mapper file @@ -332,7 +459,7 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', expect((test_directory_res / 'run' / 'two_building_chilled' / '16' / 'finished.job').exist?).to be true end - it 'runs a peak-hours MEL reduction scenario with residential and commercial buildings' do + it 'runs a peak-hours MEL reduction scenario with residential and commercial buildings', :GEB do # Use a ScenarioFile with only 2 buildings to reduce test time system("cp #{spec_dir / 'spec_files' / 'two_building_res_peak_hours_mel_reduction.csv'} #{test_scenario_mels_reduction}") # Include the MEL reduction mapper file @@ -348,7 +475,7 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', expect((test_directory_res / 'run' / 'two_building_mels_reduction' / '16' / 'finished.job').exist?).to be true end - it 'runs a peak-hours thermostat adjustment scenario with residential and commercial buildings' do + it 'runs a peak-hours thermostat adjustment scenario with residential and commercial buildings', :GEB do # Use a ScenarioFile with only 2 buildings to reduce test time system("cp #{spec_dir / 'spec_files' / 'two_building_res_stat_adjustment.csv'} #{test_scenario_stat_adjustment}") # Include the thermostat adjustment mapper file @@ -364,7 +491,7 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', expect((test_directory_res / 'run' / 'two_building_stat_adjustment' / '16' / 'finished.job').exist?).to be true end - it 'runs a flexible hot water scenario' do + it 'runs a flexible hot water scenario', :GEB do # https://github.com/NREL/openstudio-load-flexibility-measures-gem/tree/master/lib/measures/add_hpwh # Use a ScenarioFile with only 2 buildings to reduce test time system("cp #{spec_dir / 'spec_files' / 'two_building_flexible_hot_water.csv'} #{test_scenario_flexible_hot_water}") @@ -381,7 +508,7 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', expect((test_directory / 'run' / 'two_building_flexible_hot_water' / '2' / 'finished.job').exist?).to be true end - it 'runs a ice-storage scenario' do + it 'runs a ice-storage scenario', :GEB do # https://github.com/NREL/openstudio-load-flexibility-measures-gem/tree/master/lib/measures/add_central_ice_storage # Use a ScenarioFile with only 2 buildings to reduce test time system("cp #{spec_dir / 'spec_files' / 'two_building_thermal_storage_scenario.csv'} #{test_scenario_thermal_storage}") @@ -397,18 +524,28 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', expect((test_directory / 'run' / 'two_building_thermal_storage' / '1' / 'finished.job').exist?).to be true expect((test_directory / 'run' / 'two_building_thermal_storage' / '12' / 'finished.job').exist?).to be true end + end + + context 'Run and work with a small residential simulation' do + before :all do + delete_directory_or_file(test_directory_res) + system("#{call_cli} create --project-folder #{test_directory_res} --combined") + end - it 'runs a 2 building scenario with residential and commercial buildings' do + it 'runs a 2 building scenario with residential and commercial buildings', :residential do system("cp #{spec_dir / 'spec_files' / 'two_building_res.csv'} #{test_scenario_res}") system("#{call_cli} run --scenario #{test_scenario_res} --feature #{test_feature_res}") expect((test_directory_res / 'run' / 'two_building_res' / '5' / 'finished.job').exist?).to be true expect((test_directory_res / 'run' / 'two_building_res' / '16' / 'finished.job').exist?).to be true end - it 'returns graceful error message when non-US weather file is provided' do + it 'returns graceful error message when non-US weather file is provided', :residential do + skip('Skipping for GHA CI only. Error message is returned as expected but GHA runner does not see it.') csv_data = CSV.read(test_weather_file) - existing_wmo = csv_data[0][5] - # Replace the WMO with a non-US WMO (Vancouver, BC) + original_wmo = csv_data[0][5] # first row, 5th column + # Confirm we're using the correct piece of data + expect(original_wmo).to eq '725280' + # Re-write the weather file using a non-US WMO (Vancouver, BC) csv_data[0][5] = 718920 CSV.open(test_weather_file, 'w') do |csv| csv_data.each do |row| @@ -418,62 +555,61 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', # Attempt to run the residential project system("cp #{spec_dir / 'spec_files' / 'two_building_res.csv'} #{test_scenario_res}") - expect { system("#{call_cli} run --scenario #{test_scenario_res} --feature #{test_feature_res}") } - .to output(a_string_including('This is known to happen when your weather file is from somewhere outside of the United States.')) - .to_stdout_from_any_process + + stdout, stderr, status = Open3.capture3("#{call_cli} run --scenario #{test_scenario_res} --feature #{test_feature_res}") + expect(stderr).to include('This is known to happen when your weather file is from somewhere outside of the United States.') csv_data = CSV.read(test_weather_file) - # Restore the original WMO - csv_data[0][5] = existing_wmo + # Put the original WMO back into the weather file + csv_data[0][5] = original_wmo CSV.open(test_weather_file, 'w') do |csv| csv_data.each do |row| csv << row end end + # Confirm we have restored the original WMO + expect(CSV.open(test_weather_file, 'r', &:first)[5]).to eq(original_wmo) end - it 'runs a 2 building scenario using create bar geometry method' do - # Copy create bar specific files - system("cp #{example_dir / 'mappers' / 'CreateBar.rb'} #{test_directory / 'mappers' / 'CreateBar.rb'}") - system("cp #{example_dir / 'mappers' / 'createbar_workflow.osw'} #{test_directory / 'mappers' / 'createbar_workflow.osw'}") - system("cp #{spec_dir / 'spec_files' / 'two_building_create_bar.csv'} #{test_directory / 'two_building_create_bar.csv'}") - system("#{call_cli} run --scenario #{test_directory / 'two_building_create_bar.csv'} --feature #{test_feature}") - expect((test_directory / 'run' / 'two_building_create_bar' / '2' / 'finished.job').exist?).to be true - end - - it 'runs a 2 building scenario using floorspace geometry method' do - # Copy floorspace specific files - system("cp #{example_dir / 'mappers' / 'Floorspace.rb'} #{test_directory / 'mappers' / 'Floorspace.rb'}") - system("cp #{example_dir / 'mappers' / 'floorspace_workflow.osw'} #{test_directory / 'mappers' / 'floorspace_workflow.osw'}") - system("cp #{example_dir / 'osm_building' / '7_floorspace.json'} #{test_directory / 'osm_building' / '7_floorspace.json'}") - system("cp #{example_dir / 'osm_building' / '7_floorspace.osm'} #{test_directory / 'osm_building' / '7_floorspace.osm'}") - system("cp #{example_dir / 'example_floorspace_project.json'} #{test_directory / 'example_floorspace_project.json'}") - system("cp #{spec_dir / 'spec_files' / 'two_building_floorspace.csv'} #{test_directory / 'two_building_floorspace.csv'}") - expect((test_directory / 'osm_building' / '7_floorspace.osm').exist?).to be true - system("#{call_cli} run --scenario #{test_directory / 'two_building_floorspace.csv'} --feature #{test_directory / 'example_floorspace_project.json'}") - expect((test_directory / 'run' / 'two_building_floorspace' / '5' / 'finished.job').exist?).to be true - expect((test_directory / 'run' / 'two_building_floorspace' / '7' / 'finished.job').exist?).to be true + it 'validates eui', :residential do + # This test requires the 'runs a 2 building scenario with residential and commercial buildings' be run first + test_validation_file = test_directory_res / 'validation_schema.yaml' + expect { system("#{call_cli} validate --eui #{test_validation_file} --scenario #{test_scenario_res} --feature #{test_feature_res}") } + .to output(a_string_including('is within bounds set by')) + .to_stdout_from_any_process + system("cp #{spec_dir / 'spec_files' / 'out_of_bounds_validation.yaml'} #{test_validate_bounds}") + expect { system("#{call_cli} validate --eui #{test_validate_bounds} --scenario #{test_scenario_res} --feature #{test_feature_res}") } + .to output(a_string_including('kBtu/ft2/yr is greater than the validation maximum')) + .to_stdout_from_any_process + expect { system("#{call_cli} validate --eui #{test_validate_bounds} --scenario #{test_scenario_res} --feature #{test_feature_res}") } + .to output(a_string_including('is less than the validation minimum')) + .to_stdout_from_any_process + expect { system("#{call_cli} validate --eui #{test_validate_bounds} --scenario #{test_scenario_res} --feature #{test_feature_res} --units SI") } + .to output(a_string_including('kWh/m2/yr is less than the validation minimum')) + .to_stdout_from_any_process end + end - it 'runs an ev-charging scenario' do - # copy ev-charging specific files - system("cp #{spec_dir / 'spec_files' / 'two_building_ev_scenario.csv'} #{test_ev_scenario}") - system("#{call_cli} run --scenario #{test_ev_scenario} --feature #{test_feature}") - expect((test_directory / 'run' / 'two_building_ev_scenario' / '5' / 'finished.job').exist?).to be true - expect((test_directory / 'run' / 'two_building_ev_scenario' / '2' / 'finished.job').exist?).to be true + context 'Run and work with a small electric simulation' do + before :all do + delete_directory_or_file(test_directory_elec) + # use this to test both opendss and disco workflows + system("#{call_cli} create --project-folder #{test_directory_elec} --disco") + delete_directory_or_file(test_directory_pv) + system("#{call_cli} create --project-folder #{test_directory_pv} --photovoltaic") end - it 'runs an electrical network scenario' do + it 'runs an electrical network scenario', :electric do system("cp #{spec_dir / 'spec_files' / 'electrical_scenario.csv'} #{test_scenario_elec}") system("#{call_cli} run --scenario #{test_scenario_elec} --feature #{test_feature_elec}") expect((test_directory_elec / 'run' / 'electrical_scenario' / '13' / 'finished.job').exist?).to be true end - it 'runs a PV scenario when called with reopt' do - system("cp #{spec_dir / 'spec_files' / 'REopt_scenario.csv'} #{test_reopt_scenario}") + it 'runs a PV scenario when called with reopt', :electric do + system("cp #{spec_dir / 'spec_files' / 'REopt_scenario.csv'} #{test_scenario_reopt}") # Copy in reopt folder system("cp -R #{spec_dir / 'spec_files' / 'reopt'} #{test_directory_pv / 'reopt'}") - system("#{call_cli} run --scenario #{test_reopt_scenario} --feature #{test_feature_pv}") + system("#{call_cli} run --scenario #{test_scenario_reopt} --feature #{test_feature_pv}") expect((test_directory_pv / 'reopt').exist?).to be true expect((test_directory_pv / 'reopt' / 'base_assumptions.json').exist?).to be true expect((test_directory_pv / 'run' / 'reopt_scenario' / '5' / 'finished.job').exist?).to be true @@ -481,51 +617,7 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', expect((test_directory_pv / 'run' / 'reopt_scenario' / '3' / 'finished.job').exist?).to be false end - it 'post-processor closes gracefully if given an invalid type' do - # Type is totally random - expect { system("#{call_cli} process --foobar --scenario #{test_scenario} --feature #{test_feature}") } - .to output(a_string_including("unknown argument '--foobar'")) - .to_stderr_from_any_process - # Type is valid, but with extra characters - expect { system("#{call_cli} process --reopt-scenariot --scenario #{test_scenario} --feature #{test_feature}") } - .to output(a_string_including("unknown argument '--reopt-scenariot'")) - .to_stderr_from_any_process - # Type would be valid if not missing characters - expect { system("#{call_cli} process --reopt-scenari --scenario #{test_scenario} --feature #{test_feature}") } - .to output(a_string_including("unknown argument '--reopt-scenari'")) - .to_stderr_from_any_process - end - - it 'post-processor closes gracefully if not given a type' do - expect { system("#{call_cli} process --scenario #{test_scenario} --feature #{test_feature}") } - .to output(a_string_including('No valid process type entered')) - .to_stderr_from_any_process - end - - it 'default post-processes a scenario' do - # This test requires the 'runs a 2 building scenario using default geometry method' be run first - test_scenario_report = test_directory / 'run' / 'two_building_scenario' / 'default_scenario_report.csv' - system("#{call_cli} process --default --scenario #{test_scenario} --feature #{test_feature}") - expect(`wc -l < #{test_scenario_report}`.to_i).to be > 2 - expect((test_directory / 'run' / 'two_building_scenario' / 'process_status.json').exist?).to be true - end - - it 'successfully runs the rnm workflow' do - # This test requires the 'runs a 2 building scenario using default geometry method' be run first - # copy featurefile in dir - rnm_file = 'example_project_with_streets.json' - system("cp #{spec_dir / 'spec_files' / rnm_file} #{test_directory / rnm_file}") - # call rnm - test_rnm_file = test_directory / rnm_file - system("#{call_cli} rnm --scenario #{test_scenario} --feature #{test_rnm_file}") - # check that rnm inputs and outputs were created - expect((test_directory / 'run' / 'two_building_scenario' / 'rnm-us' / 'inputs.zip').exist?).to be true - expect((test_directory / 'run' / 'two_building_scenario' / 'rnm-us' / 'results').exist?).to be true - expect((test_directory / 'run' / 'two_building_scenario' / 'scenario_report_rnm.json').exist?).to be true - expect((test_directory / 'run' / 'two_building_scenario' / 'feature_file_rnm.json').exist?).to be true - end - - it 'successfully gets results from the opendss cli' do + it 'successfully gets results from the opendss cli', :electric do # This test requires the 'runs an electrical network scenario' be run first system("#{call_cli} process --default --scenario #{test_scenario_elec} --feature #{test_feature_elec}") system("#{call_cli} opendss --scenario #{test_scenario_elec} --feature #{test_feature_elec} --start-date 2017/01/15 --start-time 01:00:00 --end-date 2017/01/16 --end-time 00:00:00") @@ -536,24 +628,16 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', expect((test_directory_elec / 'run' / 'electrical_scenario' / 'opendss' / 'profiles' / 'load_1.csv').exist?).to be true end - it 'successfully runs disco simulation' do + it 'successfully runs disco simulation', :electric do # This test requires the 'runs an electrical network scenario' be run first system("#{call_cli} disco --scenario #{test_scenario_elec} --feature #{test_feature_elec}") expect((test_directory_elec / 'run' / 'electrical_scenario' / 'disco').exist?).to be true end - it 'saves post-process output as a database file' do - # This test requires the 'runs a 2 building scenario using default geometry method' be run first - db_filename = test_directory / 'run' / 'two_building_scenario' / 'default_scenario_report.db' - system("#{call_cli} process --default --with-database --scenario #{test_scenario} --feature #{test_feature}") - expect(`wc -l < #{db_filename}`.to_i).to be > 20 - expect((test_directory / 'run' / 'two_building_scenario' / 'process_status.json').exist?).to be true - end - - it 'reopt post-processes a scenario and visualize' do + it 'reopt post-processes a scenario and visualize', :electric do # This test requires the 'runs a PV scenario when called with reopt' be run first - system("#{call_cli} process --default --scenario #{test_reopt_scenario} --feature #{test_feature_pv}") - system("#{call_cli} process --reopt-scenario --scenario #{test_reopt_scenario} --feature #{test_feature_pv}") + system("#{call_cli} process --default --scenario #{test_scenario_reopt} --feature #{test_feature_pv}") + system("#{call_cli} process --reopt-scenario --scenario #{test_scenario_reopt} --feature #{test_feature_pv}") expect((test_directory_pv / 'run' / 'reopt_scenario' / 'scenario_optimization.json').exist?).to be true expect((test_directory_pv / 'run' / 'reopt_scenario' / 'process_status.json').exist?).to be true # and visualize @@ -561,100 +645,41 @@ def select_measures(test_dir, measure_name_list, workflow = 'base_workflow.osw', expect((test_directory_pv / 'run' / 'scenario_comparison.html').exist?).to be true end - it 'reopt post-processes a scenario with specified scenario assumptions file' do + it 'reopt post-processes a scenario with specified scenario assumptions file', :electric do # This test requires the 'runs a PV scenario when called with reopt' be run first - system("#{call_cli} process --default --scenario #{test_reopt_scenario} --feature #{test_feature_pv}") - expect { system("#{call_cli} process --reopt-scenario -a #{test_reopt_scenario_assumptions_file} --scenario #{test_reopt_scenario} --feature #{test_feature_pv}") } + system("#{call_cli} process --default --scenario #{test_scenario_reopt} --feature #{test_feature_pv}") + expect { system("#{call_cli} process --reopt-scenario -a #{test_reopt_scenario_assumptions_file} --scenario #{test_scenario_reopt} --feature #{test_feature_pv}") } .to output(a_string_including('multiPV_assumptions.json')) .to_stdout_from_any_process expect((test_directory_pv / 'run' / 'reopt_scenario' / 'scenario_optimization.json').exist?).to be true expect((test_directory_pv / 'run' / 'reopt_scenario' / 'process_status.json').exist?).to be true end - it 'reopt post-processes a scenario with resilience reporting' do + it 'reopt post-processes a scenario with resilience reporting', :electric do # This test requires the 'runs a PV scenario when called with reopt' be run first - system("#{call_cli} process --default --scenario #{test_reopt_scenario} --feature #{test_feature_pv}") - system("#{call_cli} process --reopt-scenario --reopt-resilience --scenario #{test_reopt_scenario} --feature #{test_feature_pv}") + system("#{call_cli} process --default --scenario #{test_scenario_reopt} --feature #{test_feature_pv}") + system("#{call_cli} process --reopt-scenario --reopt-resilience --scenario #{test_scenario_reopt} --feature #{test_feature_pv}") expect((test_directory_pv / 'run' / 'reopt_scenario' / 'scenario_optimization.json').exist?).to be true expect((test_directory_pv / 'run' / 'reopt_scenario' / 'process_status.json').exist?).to be true # path_to_resilience_report_file = test_directory_pv / 'run' / 'reopt_scenario' / 'reopt' / 'scenario_report_reopt_scenario_reopt_run_resilience.json' end - it 'reopt post-processes each feature and visualize' do + it 'reopt post-processes each feature and visualize', :electric do # This test requires the 'runs a PV scenario when called with reopt' be run first - system("#{call_cli} process --default --scenario #{test_reopt_scenario} --feature #{test_feature_pv}") - system("#{call_cli} process --reopt-feature --scenario #{test_reopt_scenario} --feature #{test_feature_pv}") + system("#{call_cli} process --default --scenario #{test_scenario_reopt} --feature #{test_feature_pv}") + system("#{call_cli} process --reopt-feature --scenario #{test_scenario_reopt} --feature #{test_feature_pv}") expect((test_directory_pv / 'run' / 'reopt_scenario' / 'feature_optimization.csv').exist?).to be true # and visualize - system("#{call_cli} visualize --scenario #{test_reopt_scenario}") + system("#{call_cli} visualize --scenario #{test_scenario_reopt}") expect((test_directory_pv / 'run' / 'reopt_scenario' / 'feature_comparison.html').exist?).to be true end - it 'opendss post-processes a scenario' do + it 'opendss post-processes a scenario', :electric do # This test requires the 'successfully gets results from the opendss cli' be run first expect((test_directory_elec / 'run' / 'electrical_scenario' / '2' / 'feature_reports' / 'default_feature_report_opendss.csv').exist?).to be false system("#{call_cli} process --opendss --scenario #{test_scenario_elec} --feature #{test_feature_elec}") expect((test_directory_elec / 'run' / 'electrical_scenario' / '2' / 'feature_reports' / 'default_feature_report_opendss.csv').exist?).to be true expect((test_directory_elec / 'run' / 'electrical_scenario' / 'process_status.json').exist?).to be true end - - it 'creates scenario visualization for default post processor' do - # This test requires the 'runs a 2 building scenario using default geometry method' be run first - # visualizing via the FeatureFile will throw error to stdout (but not crash) if a scenario that uses those features isn't processed first. - system("#{call_cli} process --default --scenario #{test_scenario} --feature #{test_feature}") - system("#{call_cli} process --default --scenario #{test_ev_scenario} --feature #{test_feature}") - system("#{call_cli} visualize --feature #{test_feature}") - expect((test_directory / 'run' / 'scenario_comparison.html').exist?).to be true - end - - it 'creates feature visualization for default post processor' do - # This test requires the 'runs a 2 building scenario using default geometry method' be run first - system("#{call_cli} process --default --scenario #{test_scenario} --feature #{test_feature}") - system("#{call_cli} visualize --scenario #{test_scenario}") - expect((test_directory / 'run' / 'two_building_scenario' / 'feature_comparison.html').exist?).to be true - end - - it 'ensures viz files are in the project directory' do - # This test requires the 'runs a 2 building scenario using default geometry method' be run first - if (test_directory / 'visualization' / 'input_visualization_feature.html').exist? - FileUtils.rm_rf(test_directory / 'visualization' / 'input_visualization_feature.html') - end - if (test_directory / 'run' / 'two_building_scenario' / 'feature_comparison.html').exist? - FileUtils.rm_rf(test_directory / 'run' / 'two_building_scenario' / 'feature_comparison.html') - end - if (test_directory / 'run' / 'two_building_scenario' / 'scenarioData.js').exist? - FileUtils.rm_rf(test_directory / 'run' / 'two_building_scenario' / 'scenarioData.js') - end - expect((test_directory / 'visualization' / 'input_visualization_feature.html').exist?).to be false - expect((test_directory / 'run' / 'two_building_scenario' / 'feature_comparison.html').exist?).to be false - expect((test_directory / 'run' / 'two_building_scenario' / 'scenarioData.js').exist?).to be false - system("#{call_cli} visualize --scenario #{test_scenario}") - expect((test_directory / 'run' / 'two_building_scenario' / 'feature_comparison.html').exist?).to be true - end - - it 'validates eui' do - # This test requires the 'runs a 2 building scenario with residential and commercial buildings' be run first - test_validation_file = test_directory_res / 'validation_schema.yaml' - expect { system("#{call_cli} validate --eui #{test_validation_file} --scenario #{test_scenario_res} --feature #{test_feature_res}") } - .to output(a_string_including('is within bounds set by')) - .to_stdout_from_any_process - system("cp #{spec_dir / 'spec_files' / 'out_of_bounds_validation.yaml'} #{test_validate_bounds}") - expect { system("#{call_cli} validate --eui #{test_validate_bounds} --scenario #{test_scenario_res} --feature #{test_feature_res}") } - .to output(a_string_including('kBtu/ft2/yr is greater than the validation maximum')) - .to_stdout_from_any_process - expect { system("#{call_cli} validate --eui #{test_validate_bounds} --scenario #{test_scenario_res} --feature #{test_feature_res}") } - .to output(a_string_including('is less than the validation minimum')) - .to_stdout_from_any_process - expect { system("#{call_cli} validate --eui #{test_validate_bounds} --scenario #{test_scenario_res} --feature #{test_feature_res} --units SI") } - .to output(a_string_including('kWh/m2/yr is less than the validation minimum')) - .to_stdout_from_any_process - end - - it 'deletes a scenario' do - expect((test_directory / 'run' / 'two_building_create_bar' / '2' / 'data_point_out.json').exist?).to be true - bar_scenario = test_directory / 'two_building_create_bar.csv' - system("#{call_cli} delete --scenario #{bar_scenario}") - expect((test_directory / 'run' / 'two_building_create_bar' / '2' / 'data_point_out.json').exist?).to be false - end end end diff --git a/uo_cli.gemspec b/uo_cli.gemspec index ea8e1693..951fc2d9 100644 --- a/uo_cli.gemspec +++ b/uo_cli.gemspec @@ -46,6 +46,4 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'rubocop', '~> 1.15.0' spec.add_development_dependency 'rubocop-checkstyle_formatter', '~> 0.4.0' spec.add_development_dependency 'rubocop-performance', '~> 1.11.3' - spec.add_development_dependency 'simplecov', '~> 0.18.2' - spec.add_development_dependency 'simplecov-lcov', '~> 0.8.0' end