diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..bc18a32e37 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Operating System (please complete the following information):** + - OS: [e.g. iOS] + - Volttron Version [develop, releases/8.2, main] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..bbcbbe7d61 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/code_analysis.yml b/.github/workflows/code_analysis.yml new file mode 100644 index 0000000000..8c5d325b11 --- /dev/null +++ b/.github/workflows/code_analysis.yml @@ -0,0 +1,74 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ main, develop ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main, develop ] + schedule: + - cron: '25 18 * * 2' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'javascript', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/pytest-auth.yml b/.github/workflows/pytest-auth.yml index 600c27c755..feaeca61bf 100644 --- a/.github/workflows/pytest-auth.yml +++ b/.github/workflows/pytest-auth.yml @@ -14,7 +14,21 @@ name: Testing platform auth # Determine what events are going to trigger a running of the workflow -on: [pull_request] +on: + workflow_dispatch: + push: + branches: + - develop + - releases/** + pull_request: + branches: + - main + - develop + - releases/** + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: # The job named build @@ -25,8 +39,8 @@ jobs: fail-fast: false matrix: # Each entry in the os and python-version matrix will be run so for the 3 x 4 there will be 12 jobs run - os: [ ubuntu-18.04, ubuntu-20.04 ] - python-version: [ 3.6, 3.7] # , 3.8, 3.9 ] + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] # Run-on determines the operating system available to run on # - At the current time there is only ubuntu machines between 16.04 and 20.04 available @@ -36,17 +50,17 @@ jobs: # Each step will be run in order of listing. steps: # checkout the volttron repository and set current direectory to it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v2 + uses: volttron/volttron-build-action@v6 with: python_version: ${{ matrix.python-version }} os: ${{ matrix.os }} @@ -55,20 +69,20 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report path: output/test-auth-${{matrix.os}}-${{ matrix.python-version }}-results.xml - + # - name: Publish Unit Test Results # uses: EnricoMi/publish-unit-test-result-action@v1.5 # if: always() # with: # github_token: ${{ secrets.WORKFLOW_ACCESS_TOKEN }} # files: output/test-testutils*.xml - - + + #-cov=com --cov-report=xml --cov-report=html # pytest tests.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html # - name: Lint with flake8 diff --git a/.github/workflows/pytest-dbutils-backup_db.yml b/.github/workflows/pytest-dbutils-backup_db.yml index 39834920e2..bc677c6b8d 100644 --- a/.github/workflows/pytest-dbutils-backup_db.yml +++ b/.github/workflows/pytest-dbutils-backup_db.yml @@ -15,15 +15,20 @@ name: Testing BackupDatabase # Determine what events are going to trigger a running of the workflow on: + workflow_dispatch: push: branches: - develop - releases/** pull_request: branches: - - main - - develop - - releases/** + - main + - develop + - releases/** + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: build: @@ -36,8 +41,8 @@ jobs: fail-fast: false matrix: # Each entry in the os and python-version matrix will be run. For example, on a list of 3 os's and 4 python versions, 12 jobs will be run - os: [ ubuntu-18.04, ubuntu-20.04 ] - python-version: [ 3.6, 3.7] # , 3.8, 3.9 ] + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] # Run-on determines the operating system available to run on # - At the current time there is only ubuntu machines between 16.04 and 20.04 available @@ -46,11 +51,11 @@ jobs: steps: # checkout the volttron repository and set current directory to it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -60,7 +65,7 @@ jobs: # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v1 + uses: volttron/volttron-build-action@v6 timeout-minutes: 600 with: python_version: ${{ matrix.python-version }} @@ -70,7 +75,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-influxdbfuncts.yml b/.github/workflows/pytest-dbutils-influxdbfuncts.yml index 205f50c56d..47b335e9e8 100644 --- a/.github/workflows/pytest-dbutils-influxdbfuncts.yml +++ b/.github/workflows/pytest-dbutils-influxdbfuncts.yml @@ -7,15 +7,19 @@ name: Testing influxdbutils on: + workflow_dispatch: push: branches: - develop - releases/** pull_request: branches: - - main - - develop - - releases/** + - main + - develop + - releases/** +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: build: @@ -28,19 +32,19 @@ jobs: fail-fast: false matrix: # Each entry in the os and python-version matrix will be run so for the 3 x 4 there will be 12 jobs run - os: [ ubuntu-18.04, ubuntu-20.04 ] - python-version: [ 3.6, 3.7] # , 3.8, 3.9 ] + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] runs-on: ${{ matrix.os }} steps: # checkout the volttron repository and set current directory to it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -50,7 +54,7 @@ jobs: # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v1 + uses: volttron/volttron-build-action@v6 timeout-minutes: 600 with: python_version: ${{ matrix.python-version }} @@ -60,7 +64,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-mysqlfuncts.yml b/.github/workflows/pytest-dbutils-mysqlfuncts.yml index cc2e324efb..6454fa4a86 100644 --- a/.github/workflows/pytest-dbutils-mysqlfuncts.yml +++ b/.github/workflows/pytest-dbutils-mysqlfuncts.yml @@ -7,15 +7,19 @@ name: Testing mysqlfuncts on: + workflow_dispatch: push: branches: - develop - releases/** pull_request: branches: - - main - - develop - - releases/** + - main + - develop + - releases/** +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: build: @@ -28,19 +32,19 @@ jobs: fail-fast: false matrix: # Each entry in the os and python-version matrix will be run so for the 3 x 4 there will be 12 jobs run - os: [ ubuntu-18.04, ubuntu-20.04 ] - python-version: [ 3.6, 3.7] # , 3.8, 3.9 ] + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] runs-on: ${{ matrix.os }} steps: # checkout the volttron repository and set current directory to it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -50,7 +54,7 @@ jobs: # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v1 + uses: volttron/volttron-build-action@v6 timeout-minutes: 600 with: python_version: ${{ matrix.python-version }} @@ -60,7 +64,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-postgresqlfuncts.yml b/.github/workflows/pytest-dbutils-postgresqlfuncts.yml index 229d300a1e..87499a2384 100644 --- a/.github/workflows/pytest-dbutils-postgresqlfuncts.yml +++ b/.github/workflows/pytest-dbutils-postgresqlfuncts.yml @@ -7,15 +7,19 @@ name: Testing postgresqlfuncts on: + workflow_dispatch: push: branches: - develop - releases/** pull_request: branches: - - main - - develop - - releases/** + - main + - develop + - releases/** +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: build: @@ -28,19 +32,19 @@ jobs: fail-fast: false matrix: # Each entry in the os and python-version matrix will be run so for the 3 x 4 there will be 12 jobs run - os: [ ubuntu-18.04, ubuntu-20.04 ] - python-version: [ 3.6, 3.7] # , 3.8, 3.9 ] + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] runs-on: ${{ matrix.os }} steps: # checkout the volttron repository and set current directory to it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -50,7 +54,7 @@ jobs: # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v1 + uses: volttron/volttron-build-action@v6 timeout-minutes: 600 with: python_version: ${{ matrix.python-version }} @@ -60,7 +64,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-sqlitefuncts.yml b/.github/workflows/pytest-dbutils-sqlitefuncts.yml index 8887030fba..e5bb742d7d 100644 --- a/.github/workflows/pytest-dbutils-sqlitefuncts.yml +++ b/.github/workflows/pytest-dbutils-sqlitefuncts.yml @@ -7,16 +7,19 @@ name: Testing sqlitefuncts on: + workflow_dispatch: push: branches: - develop - releases/** pull_request: branches: - - main - - develop - - releases/** - + - main + - develop + - releases/** +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: build: env: @@ -28,19 +31,19 @@ jobs: fail-fast: false matrix: # Each entry in the os and python-version matrix will be run so for the 3 x 4 there will be 12 jobs run - os: [ ubuntu-18.04, ubuntu-20.04 ] - python-version: [ 3.6, 3.7] # , 3.8, 3.9 ] + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] runs-on: ${{ matrix.os }} steps: # checkout the volttron repository and set current directory to it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -51,7 +54,7 @@ jobs: # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v1 + uses: volttron/volttron-build-action@v6 timeout-minutes: 600 with: python_version: ${{ matrix.python-version }} @@ -60,7 +63,7 @@ jobs: test_output_suffix: ${{ env.OUTPUT_SUFFIX }} - name: Archive test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-dbutils-timescaldbfuncts.yml b/.github/workflows/pytest-dbutils-timescaldbfuncts.yml index 9362850218..7639d195c7 100644 --- a/.github/workflows/pytest-dbutils-timescaldbfuncts.yml +++ b/.github/workflows/pytest-dbutils-timescaldbfuncts.yml @@ -7,16 +7,19 @@ name: Testing postgresql_timescaledb_functs on: + workflow_dispatch: push: branches: - develop - releases/** pull_request: branches: - - main - - develop - - releases/** - + - main + - develop + - releases/** +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: build: env: @@ -28,19 +31,19 @@ jobs: fail-fast: false matrix: # Each entry in the os and python-version matrix will be run so for the 3 x 4 there will be 12 jobs run - os: [ ubuntu-18.04, ubuntu-20.04 ] - python-version: [ 3.6, 3.7] # , 3.8, 3.9 ] + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] runs-on: ${{ matrix.os }} steps: # checkout the volttron repository and set current directory to it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -50,7 +53,7 @@ jobs: # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v1 + uses: volttron/volttron-build-action@v6 timeout-minutes: 600 with: python_version: ${{ matrix.python-version }} @@ -60,7 +63,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-miscellaneous-tests.yml b/.github/workflows/pytest-miscellaneous-tests.yml new file mode 100644 index 0000000000..80dc6d8d27 --- /dev/null +++ b/.github/workflows/pytest-miscellaneous-tests.yml @@ -0,0 +1,105 @@ +--- +# This workflow is meant as a foundational workflow for running integration/unit tests on multiple targeted +# ubuntu versions with multiple python versions. +# +# This workflow utilizes the build-dependency-cache workflow which sets up the environment dependencies using +# bootstrap.py --all +# + +# Documentation for the syntax of this file is located +# https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions + +# The workflow name will show up in the action tab on github during execution +# https://github.com/VOLTTRON/volttron/actions (or if you are pushing to your own fork change the user) +name: Miscellaneous platform tests + +on: + workflow_dispatch: + push: + branches: + - develop + - releases/** + pull_request: + branches: + - main + - develop + - releases/** +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + # The job named build + build: + # The strategy allows customization of the build and allows matrixing the version of os and software + # https://docs.github.com/en/free-pro-team@l.atest/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstrategy + strategy: + fail-fast: false + matrix: + # Each entry in the os and python-version matrix will be run + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] + + # Run-on determines the operating system available to run on + # - At the current time there is only ubuntu machine 20.04 available + # - This uses the matrix os from the strategy above + runs-on: ${{ matrix.os }} + + # Each step will be run in order of listing. + steps: + # Checkout the volttron repository and set current direectory to it + - uses: actions/checkout@v4 + + # Setup the python environment for the operating system + - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + # Run the specified tests and save the results to a unique file that can be archived for later analysis + - name: Run certs test on ${{ matrix.python-version }}, ${{ matrix.os }} + uses: volttron/volttron-build-action@v6 + with: + python_version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + test_path: volttrontesting/platform/web/test_certs.py + test_output_suffix: misc + + - name: Run core agent test on ${{ matrix.python-version }}, ${{ matrix.os }} + uses: volttron/volttron-build-action@v6 + with: + python_version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + test_path: volttrontesting/platform/test_core_agent.py + test_output_suffix: misc + + - name: Run packaging test on ${{ matrix.python-version }}, ${{ matrix.os }} + uses: volttron/volttron-build-action@v6 + with: + python_version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + test_path: volttrontesting/platform/test_packaging.py + test_output_suffix: misc + + - name: Run platform init test on ${{ matrix.python-version }}, ${{ matrix.os }} + uses: volttron/volttron-build-action@v6 + with: + python_version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + test_path: volttrontesting/platform/test_platform_init.py + test_output_suffix: misc + + - name: Run sqlite3 test on ${{ matrix.python-version }}, ${{ matrix.os }} + uses: volttron/volttron-build-action@v6 + with: + python_version: ${{ matrix.python-version }} + os: ${{ matrix.os }} + test_path: volttrontesting/platform/test_sqlite3_fix.py + test_output_suffix: misc + + # Archive the results from the pytest to storage. + - name: Archive test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: pytest-report + path: output/test-web-${{matrix.os}}-${{ matrix.python-version }}-results.xml diff --git a/.github/workflows/pytest-testutils.yml b/.github/workflows/pytest-testutils.yml index 51852ede0e..fa1a06887b 100644 --- a/.github/workflows/pytest-testutils.yml +++ b/.github/workflows/pytest-testutils.yml @@ -9,16 +9,19 @@ name: Testing testutils directory on: + workflow_dispatch: push: branches: - develop - releases/** pull_request: branches: - - main - - develop - - releases/** - + - main + - develop + - releases/** +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: build: # The strategy allows customization of the build and allows matrixing the version of os and software @@ -27,25 +30,25 @@ jobs: fail-fast: false matrix: # Each entry in the os and python-version matrix will be run so for the 3 x 4 there will be 12 jobs run - os: [ ubuntu-18.04, ubuntu-20.04 ] - python-version: [ 3.6, 3.7] # , 3.8, 3.9 ] + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] runs-on: ${{ matrix.os }} steps: # checkout the volttron repository and set current directory to it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v1 + uses: volttron/volttron-build-action@v6 with: python_version: ${{ matrix.python-version }} os: ${{ matrix.os }} @@ -54,7 +57,7 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report diff --git a/.github/workflows/pytest-vctl.yml b/.github/workflows/pytest-vctl.yml index 8354c3cde1..b541753236 100644 --- a/.github/workflows/pytest-vctl.yml +++ b/.github/workflows/pytest-vctl.yml @@ -15,16 +15,19 @@ name: Testing volttron-ctl # Determine what events are going to trigger a running of the workflow on: + workflow_dispatch: push: branches: - develop - releases/** pull_request: branches: - - main - - develop - - releases/** - + - main + - develop + - releases/** +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: # The job named build build: @@ -34,8 +37,8 @@ jobs: fail-fast: false matrix: # Each entry in the os and python-version matrix will be run so for the 3 x 4 there will be 12 jobs run - os: [ ubuntu-18.04, ubuntu-20.04 ] - python-version: [ 3.6, 3.7] # , 3.8, 3.9 ] + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] # Run-on determines the operating system available to run on # - At the current time there is only ubuntu machines between 16.04 and 20.04 available @@ -45,18 +48,18 @@ jobs: # Each step will be run in order of listing. steps: # checkout the volttron repository and set current directory to it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # Attempt to restore the cache from the build-dependency-cache workflow if present then # the output value steps.check_files.outputs.files_exists will be set (see the next step for usage) - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v1 + uses: volttron/volttron-build-action@v6 with: python_version: ${{ matrix.python-version }} os: ${{ matrix.os }} @@ -65,10 +68,9 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report # should match test-- ... path: output/test-control_tests-${{matrix.os}}-${{ matrix.python-version }}-results.xml - diff --git a/.github/workflows/pytest-web.yml b/.github/workflows/pytest-web.yml index bad54e9421..fe0496f35a 100644 --- a/.github/workflows/pytest-web.yml +++ b/.github/workflows/pytest-web.yml @@ -14,16 +14,19 @@ name: Testing platform web on: + workflow_dispatch: push: branches: - develop - releases/** pull_request: branches: - - main - - develop - - releases/** - + - main + - develop + - releases/** +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: # The job named build build: @@ -33,8 +36,8 @@ jobs: fail-fast: false matrix: # Each entry in the os and python-version matrix will be run so for the 3 x 4 there will be 12 jobs run - os: [ ubuntu-18.04, ubuntu-20.04 ] - python-version: [ 3.6, 3.7] # , 3.8, 3.9 ] + os: [ ubuntu-20.04 ] + python-version: [ 3.8 ] # Run-on determines the operating system available to run on # - At the current time there is only ubuntu machines between 16.04 and 20.04 available @@ -44,17 +47,17 @@ jobs: # Each step will be run in order of listing. steps: # checkout the volttron repository and set current direectory to it - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 # setup the python environment for the operating system - name: Set up Python ${{matrix.os}} ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} # Run the specified tests and save the results to a unique file that can be archived for later analysis. - name: Run pytest on ${{ matrix.python-version }}, ${{ matrix.os }} - uses: volttron/volttron-build-action@v2 + uses: volttron/volttron-build-action@v6 with: python_version: ${{ matrix.python-version }} os: ${{ matrix.os }} @@ -63,20 +66,20 @@ jobs: # Archive the results from the pytest to storage. - name: Archive test results - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 if: always() with: name: pytest-report path: output/test-web-${{matrix.os}}-${{ matrix.python-version }}-results.xml - + # - name: Publish Unit Test Results # uses: EnricoMi/publish-unit-test-result-action@v1.5 # if: always() # with: # github_token: ${{ secrets.WORKFLOW_ACCESS_TOKEN }} # files: output/test-testutils*.xml - - + + #-cov=com --cov-report=xml --cov-report=html # pytest tests.py --doctest-modules --junitxml=junit/test-results.xml --cov=com --cov-report=xml --cov-report=html # - name: Lint with flake8 diff --git a/.gitignore b/.gitignore index 8bed92dec4..43b6d6be2b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ node_modules applications .cache .idea +.vscode/ /env/ /Agents/*/{build,dist}/ /Agents/*.egg @@ -34,12 +35,16 @@ rabbitmq.log htmlcov/ MnesiaCore.* rabbitmq-server.download.tar.xz -/MagicMock/mock/ +*MagicMock* /docs/source/volttron_api/ +*ecobee_*.json +.env* +dist/ + Pipfile modbus_configs modbus_tk_configs activateenv volttron.service upgrade-scripts -volttron_modbus.log \ No newline at end of file +volttron_modbus.log diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000..40ba57ee61 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,95 @@ +# This file is a template, and might need editing before it works on your project. +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml + +# See https://docs.gitlab.com/ee/ci/yaml/index.html for all available options + +# you can delete this line if you're not using Docker +# image: busybox:latest + +stages: + - build + - test + +.parallel-tests: + parallel: + matrix: + - TEST: + - services/core/ActuatorAgent/tests + - services/core/DataMover/tests/ + - services/core/DNP3OutstationAgent/tests + - services/core/OpenADRVenAgent/tests + - services/core/PlatformDriverAgent/tests + - services/core/SQLHistorian/tests + - services/core/VolttronCentral/tests + - services/core/VolttronCentralPlatform/tests + - services/core/WeatherDotGov/tests + - services/ops + - volttrontesting/gevent/yield_test.py + - volttrontesting/platform/auth_tests + - volttrontesting/platform/control_tests + - volttrontesting/platform/dbutils + - volttrontesting/platform/web + - volttrontesting/platform/test_basehistorian.py + - volttrontesting/platform/test_connection.py + - volttrontesting/platform/test_core_agent.py + - volttrontesting/platform/test_instance_setup.py + - volttrontesting/platform/test_keystore.py + - volttrontesting/platform/test_packaging.py + - volttrontesting/platform/test_platform_init.py + - volttrontesting/platform/test_platform_rmq.py + - volttrontesting/platform/test_platform_web.py + - volttrontesting/platform/test_rmq_platform_shutdown.py + - volttrontesting/platform/test_sqlite3_fix.py + - volttrontesting/services/historian + - volttrontesting/services/aggregate_historian + - volttrontesting/services/tagging + - volttrontesting/services/weather + - volttrontesting/services/test_pubsub_service.py + - volttrontesting/subsystems + - volttrontesting/testutils + - volttrontesting/zmq + +build 20.04: + stage: build + tags: + - ubuntu2004 + + before_script: + #- killall -9 volttron beam.smp python + - rm -rf dist ~/.volttron ~/.volttron_instances + - rm -rf /tmp/tmp* + - rm -rf ~/rabbitmq_server + + script: + - python3 bootstrap.py --all + - source env/bin/activate + - python3 bootstrap.py --rabbitmq + - echo "BUILD_DIR_20_04=`pwd`" >> build.env + - echo "$BUILD_DIR_20_04" + - echo `pwd` + + artifacts: + reports: + dotenv: build.env + +test 20.04: + stage: test + needs: [build 20.04] + variables: + GIT_CHECKOUT: "false" + tags: + - ubuntu2004 + extends: .parallel-tests + script: + - cd $BUILD_DIR_20_04 + - echo `pwd` + - source env/bin/activate + - pytest $TEST + + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000..4c033a1021 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: debug-statements + - id: requirements-txt-fixer diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c4456b33e0..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -language: python2.7 - -# Each array entry will execute 1 job. -env: - - NUM_PROCESSES=10 CI="travis" - -services: - - docker - -script: ci-integration/run-test-docker.sh - diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..122dc82468 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,128 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +volttron@pnnl.gov. +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.0, available at +https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. + +Community Impact Guidelines were inspired by [Mozilla's code of conduct +enforcement ladder](https://github.com/mozilla/diversity). + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at +https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe5ba34cc8..877db0cdfc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ Thank you for wanting to contribute back to Volttron! We really appreciate it. -We have a lengthy [Guide to Contributions](https://volttron.readthedocs.io/en/develop/community_resources/contributing.html) on our ReadTheDocs site, where all of our documentation lives. This guide focuses on helpful instructions for newcomers to GitHub and Git. +We have a lengthy [Guide to Contributions](https://volttron.readthedocs.io/en/main/developing-volttron/contributing-code.html) on our ReadTheDocs site, where all of our documentation lives. This guide focuses on helpful instructions for newcomers to GitHub and Git. ### Branch Naming diff --git a/COPYRIGHT b/COPYRIGHT deleted file mode 100644 index f7fcb5303e..0000000000 --- a/COPYRIGHT +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2020, Battelle Memorial Institute. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# diff --git a/DISCLAIMER b/DISCLAIMER new file mode 100644 index 0000000000..23106ccc96 --- /dev/null +++ b/DISCLAIMER @@ -0,0 +1,26 @@ +Disclaimer + +This material was prepared as an account of work sponsored by an agency +of the United States Government. Neither the United States Government +nor the United States Department of Energy, nor Battelle, nor any of +their employees, nor any jurisdiction or organization that has cooperated +in the development of these materials, makes any warranty, express or +implied, or assumes any legal liability or responsibility for the accuracy, +completeness, or usefulness or any information, apparatus, product, software, +or process disclosed, or represents that its use would not infringe privately +owned rights. + +Reference herein to any specific commercial product, process, or service by +trade name, trademark, manufacturer, or otherwise does not necessarily +constitute or imply its endorsement, recommendation, or favoring by the United +States Government or any agency thereof, or Battelle Memorial Institute. The +views and opinions of authors expressed herein do not necessarily state or +reflect those of the United States Government or any agency thereof. + + + PACIFIC NORTHWEST NATIONAL LABORATORY + operated by + BATTELLE + for the + UNITED STATES DEPARTMENT OF ENERGY + under Contract DE-AC05-76RL01830 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..8e86760c69 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ + +Copyright 2023, Battelle Memorial Institute. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not +use this file except in compliance with the License. You may obtain a copy +of the License at + + + +The patent license grant shall only be applicable to the following patent +and patent application (Battelle IPID 17008-E), as assigned to the Battelle +Memorial Institute, as used in conjunction with this Work: +• US Patent No. 9,094,385, issued 7/28/15 • USPTO Patent App. No. 14/746,577, +filed 6/22/15, published as US 2016-0006569. + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index 778686f428..0000000000 --- a/LICENSE.md +++ /dev/null @@ -1,11 +0,0 @@ - - -Copyright 2019, Battelle Memorial Institute. - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -The patent license grant shall only be applicable to the following patent and patent application (Battelle IPID 17008-E), as assigned to the Battelle Memorial Institute, as used in conjunction with this Work: • US Patent No. 9,094,385, issued 7/28/15 • USPTO Patent App. No. 14/746,577, filed 6/22/15, published as US 2016-0006569. - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/README.md b/README.md index 398d7cf8dc..bea8985a36 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,25 @@ ![image](docs/source/files/VOLLTRON_Logo_Black_Horizontal_with_Tagline.png) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/fcf58045b4804edf8f4d3ecde3016f76)](https://app.codacy.com/gh/VOLTTRON/volttron?utm_source=github.com&utm_medium=referral&utm_content=VOLTTRON/volttron&utm_campaign=Badge_Grade_Settings) -![example workflow](https://github.com/volttron/volttron/actions/workflows/pytest-testutils.yml/badge.svg) +# VOLTTRON + +This repository is for the current production VOLTTRON. We are working on VOLTTRON 10 (modular) which is available under +github at https://github.com/eclipse-volttron/. The modular version of VOLTTRON will help ease deployment and support +flexible deployment where in only required agents/applications can be installed, thereby simplifying setup and upgrade +steps for the end user. The VOLTTRON team are currently working on porting agents from monolithic VOLTTRON to the +modular version of VOLTTRON. To know more about modular VOLTTRON, please visit our new documentation site available +at https://eclipse-volttron.readthedocs.io/en/latest/. We would love for you to try it out and give us early +feedback. Also, until our work on modular VOLTTRON is completed, please continue cloning and using this +repository for your production systems. VOLTTRON™ is an open source platform for distributed sensing and control. The platform provides services for collecting and storing data from buildings and devices and provides an environment for developing applications which interact with that data. -## Upgrading to VOLTTRON 8.x +## Upgrading Pre-8 to VOLTTRON 9.x -VOLTTRON 8 introduces three changes that require an explict upgrade step when upgrading from a earlier VOLTTRON version +VOLTTRON 8 introduces four changes that require an explict upgrade step when upgrading from an earlier VOLTTRON version 1. Dynamic RPC authorization feature - This requires a modification to the auth file. If you have a pre-existing instance of VOLTTRON running on an older version, the auth file will need to be updated. @@ -23,6 +32,10 @@ VOLTTRON 8 introduces three changes that require an explict upgrade step when up topics table instead of separate metadata table. SQLHistorians with version >= 4.0.0 can work with existing database with older schema however the historian agent code should be upgraded to newer version (>=4.0.0) to run with VOLTTRON 8 core. + 4. VOLTTRON feature to run individual agents as unique Unix users is now named "agent-isolation-mode" and is + consistently referred to using this name in code, configuration, and documentation. Before VOLTTRON 8.2 this + configuration parameter was called "secure-agent-users" and related documentation referred to this mode as + "secure mode". To upgrade: @@ -32,9 +45,10 @@ To upgrade: 2. Update volttron source code version to VOLTTRON 8 3. activate the volttron environment, and run ```python bootstrap.py --force```. If you have any additional bootstrap options that you need (rabbitmq, web, drivers, etc.) include these in the above command. - 4. Run ```volttron-upgrade``` to update the auth file and move historian cache files into agent-data directory. - Note that the upgrade script will only move the backup.sqlite file and will not move sqlite historian's db file - if they are within the install directory. If using a SQLite historian, please backup the database file of + 4. Run ```volttron-upgrade``` to update the auth file, move historian cache files into agent-data directory, and + rename the config parameter "secure-agent-users" in VOLTTRON_HOME/config to "agent-isolation-mode" + **Note** that the upgrade script will only move the backup.sqlite file and will not move sqlite historian's db + file if they are within the install directory. If using a SQLite historian, please backup the database file of sqlite historian before upgrading to the latest historian version. 5. Start VOLTTRON 6. Run ```vctl install --force --vip-identity --agent-config ``` to upgrade @@ -109,38 +123,42 @@ You can deactivate the environment at any time by running `deactivate`. #### Steps for RabbitMQ -##### 1. Install Erlang version 21 packages - -For RabbitMQ based VOLTTRON, some RabbitMQ specific software packages must be installed. - -###### On Debian based systems and CentOS 6/7 +##### 1. Install Erlang version 25 packages -If you are running an Debian or CentOS system, you can install the RabbitMQ dependencies by running the rabbit - dependencies script, passing in the OS name and appropriate distribution as parameters. The following are supported: - -- `debian bionic` (for Ubuntu 18.04) - -- `debian xenial` (for Ubuntu 16.04) +###### Install Erlang pre-requisites +```shell +sudo apt-get update +sudo apt-get install -y gnupg apt-transport-https libsctp1 libncurses5 +``` +Please note there could be other pre-requisites that erlang requires based on the version of Erlang and OS. If there are other pre-requisites required, +install of erlang should fail with appropriate error message. -- `debian xenial` (for Linux Mint 18.04) +###### Purge previous versions of Erlang +```shell +sudo apt-get purge -yf erlang-base +``` -- `debian stretch` (for Debian Stretch) +###### Install Erlang -- `debian buster` (for Debian Buster) +Download and install ErlangOTP from [Erlang Solutions](https://www.erlang-solutions.com/downloads/). +RMQ uses components - ssl, public_key, asn1, and crypto. These are by default included in the OTP +RabbitMQ 3.9.29 is compatible with Erlang versions 24.3.4.2 to 25.2. VOLTTRON was tested with Erlang version 25.2-1 -- `raspbian buster` (for Raspbian/Raspberry Pi OS buster) +Example: -Example command: +On Ubuntu 22.04: -```sh -./scripts/rabbit_dependencies.sh debian xenial +```shell +wget https://binaries2.erlang-solutions.com/ubuntu/pool/contrib/e/esl-erlang/esl-erlang_25.2-1~ubuntu~jammy_amd64.deb +sudo dpkg -i esl-erlang_25.2-1~ubuntu~jammy_amd64.deb ``` -###### Alternatively +On Ubuntu 20.04: +```shell +wget https://binaries2.erlang-solutions.com/ubuntu/pool/contrib/e/esl-erlang/esl-erlang_25.2-1~ubuntu~focal_amd64.deb +sudo dpkg -i esl-erlang_25.2-1~ubuntu~focal_amd64.deb +``` -You can download and install Erlang from [Erlang Solutions](https://www.erlang-solutions.com/resources/download.html). -Please include OTP/components - ssl, public_key, asn1, and crypto. -Also lock your version of Erlang using the [yum-plugin-versionlock](https://access.redhat.com/solutions/98873) ##### 2. Configure hostname @@ -153,9 +171,14 @@ connect to empd (port 4369) on ." Note: RabbitMQ startup error would s and not in RabbitMQ logs (/var/log/rabbitmq/rabbitmq@hostname.log) ##### 3. Bootstrap +Remove older version of rabbitmq_server directory if you are upgrading from a older version. +Defaults to /rabbitmq_server/rabbitmq_server-3.9.7 + +Run the rabbitmq boostrap command within an activated VOLTTRON environment ```sh cd volttron +source env/bin/activate python3 bootstrap.py --rabbitmq [optional install directory. defaults to /rabbitmq_server] ``` @@ -163,17 +186,17 @@ python3 bootstrap.py --rabbitmq [optional install directory. defaults to This will build the platform and create a virtual Python environment and dependencies for RabbitMQ. It also installs RabbitMQ server as the current user. If an install path is provided, that path should exist and the user should have -write permissions. RabbitMQ will be installed under `/rabbitmq_server-3.7.7`. -The rest of the documentation refers to the directory `/rabbitmq_server-3.7.7` as +write permissions. RabbitMQ will be installed under `/rabbitmq_server-`. +The rest of the documentation refers to the directory `/rabbitmq_server-` as `$RABBITMQ_HOME` You can check if the RabbitMQ server is installed by checking its status. Please note, the `RABBITMQ_HOME` environment variable can be set in ~/.bashrc. If doing so, it needs to be set to the RabbitMQ installation directory (default path is -`/rabbitmq_server/rabbitmq_server-3.7.7`) +`/rabbitmq_server/rabbitmq_server-`) ```sh -echo 'export RABBITMQ_HOME=$HOME/rabbitmq_server/rabbitmq_server-3.7.7'|sudo tee --append ~/.bashrc +echo 'export RABBITMQ_HOME=$HOME/rabbitmq_server/rabbitmq_server-3.9.29'|sudo tee --append ~/.bashrc source ~/.bashrc $RABBITMQ_HOME/sbin/rabbitmqctl status @@ -230,7 +253,7 @@ Your VOLTTRON_HOME currently set to: /home/vdev/new_vhome2 Is this the volttron you are attempting to setup? [Y]: Creating rmq config yml -RabbitMQ server home: [/home/vdev/rabbitmq_server/rabbitmq_server-3.7.7]: +RabbitMQ server home: [/home/vdev/rabbitmq_server/rabbitmq_server-3.9.29]: Fully qualified domain name of the system: [cs_cbox.pnl.gov]: Enable SSL Authentication: [Y]: @@ -250,7 +273,7 @@ AMQPS (SSL) port RabbitMQ address: [5671]: https port for the RabbitMQ management plugin: [15671]: INFO:rmq_setup.pyc:Starting rabbitmq server Warning: PID file not written; -detached was passed. -INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.7.7 +INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.9.29 INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost @@ -264,7 +287,7 @@ INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost INFO:rmq_setup.pyc:**Stopped rmq server Warning: PID file not written; -detached was passed. -INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.7.7 +INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.9.29 INFO:rmq_setup.pyc: ####################### diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md deleted file mode 100644 index bb4c129d47..0000000000 --- a/RELEASE_NOTES.md +++ /dev/null @@ -1,17 +0,0 @@ - -The VOLTTRON 1.0 release includes the following features: - - * Scheduler 2.0: The new scheduler allows applications to reserve devices ahead of time - * SchedulerExample: This simple agent provides an example of publishing a schedule request. - -VOLTTRON v1.0 also includes features in a beta stage. These features are slated for release in v1.1 but are included in 1.0 for those who wish to begin investigating them. These features are: - - * Multi-node communication: Enables platforms to publish and subscribe to each other - * BACNet Driver: Enables reading/writing to devices using the BACNet protocol - -Included are PNNL developed applications: AFDD and DR which are in the process of being modified to work with the new scheduler.DR will not currently function with Scheduler 2.0. - -For details of the Scheduler, see: https://svn.pnl.gov/RTUNetwork/wiki/ActuatorAgent -For details of Multi-node, see: https://svn.pnl.gov/RTUNetwork/wiki/MultiBuildingMessaging - -Please contact the VOLTTRON team if there are any questions. \ No newline at end of file diff --git a/bootstrap.py b/bootstrap.py index 836f44c2cf..8121b03c4c 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -1,44 +1,30 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} """bootstrap - Prepare a VOLTTRON virtual environment. -Bootstrapping is broken into two stages. The first stage should only be +Bootstrapping is done in two stages. The first stage should only be invoked once per virtual environment. It downloads virtualenv and creates a virtual Python environment in the virtual environment directory (defaults to a subdirectory named env in the same directory as @@ -72,20 +58,22 @@ import argparse import errno import logging +import os +import shutil import subprocess import sys -from urllib.request import urlopen - -import os import traceback +from urllib.request import urlopen from requirements import extras_require, option_requirements + _log = logging.getLogger(__name__) _WINDOWS = sys.platform.startswith('win') default_rmq_dir = os.path.join(os.path.expanduser("~"), "rabbitmq_server") -rabbitmq_server = 'rabbitmq_server-3.7.7' +rmq_version = "3.9.29" +rabbitmq_server = f"rabbitmq_server-{rmq_version}" def shescape(args): @@ -95,7 +83,6 @@ def shescape(args): def bootstrap(dest, prompt='(volttron)', version=None, verbose=None): - import shutil args = [sys.executable, "-m", "venv", dest, "--prompt", prompt] complete = subprocess.run(args, stdout=subprocess.PIPE) @@ -107,12 +94,12 @@ def bootstrap(dest, prompt='(volttron)', version=None, verbose=None): return os.path.join(dest, "bin/python") -def pip(operation, args, verbose=None, upgrade=False, offline=False): +def pip(operation, args, verbose=None, offline=False): """Call pip in the virtual environment to perform operation.""" cmd = ['pip', operation] if verbose is not None: cmd.append('--verbose' if verbose else '--quiet') - if upgrade and operation == 'install': + if operation == 'install': cmd.append('--upgrade') if offline: cmd.extend(['--retries', '0', '--timeout', '1']) @@ -122,7 +109,7 @@ def pip(operation, args, verbose=None, upgrade=False, offline=False): subprocess.check_call(cmd) -def update(operation, verbose=None, upgrade=False, offline=False, optional_requirements=[], rabbitmq_path=None): +def update(operation, verbose=None, offline=False, optional_requirements=[], rabbitmq_path=None): """Install dependencies in setup.py and requirements.txt.""" print("UPDATE: {}".format(optional_requirements)) assert operation in ['install', 'wheel'] @@ -136,14 +123,14 @@ def update(operation, verbose=None, upgrade=False, offline=False, optional_requi # option_requirements contains wheel as first entry # Build option_requirements separately to pass install options - build_option = '--build-option' if wheeling else '--install-option' + build_option = '--build-option' if wheeling else '--config-settings' for requirement, options in option_requirements: args = [] for opt in options: args.extend([build_option, opt]) args.extend(['--no-deps', requirement]) - pip(operation, args, verbose, upgrade, offline) + pip(operation, args, verbose, offline) # Install local packages and remaining dependencies args = [] @@ -159,7 +146,7 @@ def update(operation, verbose=None, upgrade=False, offline=False, optional_requi target += '[' + ','.join(optional_requirements) + ']' args.extend(['--editable', target]) print(f"Target: {target}") - pip(operation, args, verbose, upgrade, offline) + pip(operation, args, verbose, offline) try: # Install rmq server if needed @@ -171,7 +158,7 @@ def update(operation, verbose=None, upgrade=False, offline=False, optional_requi def install_rabbit(rmq_install_dir): # Install gevent friendly pika - pip('install', ['gevent-pika==0.3'], False, True, offline=False) + pip('install', ['pika==1.2.0'], False, offline=False) # try: process = subprocess.Popen(["which", "erl"], stderr=subprocess.PIPE, stdout=subprocess.PIPE) (output, error) = process.communicate() @@ -202,7 +189,7 @@ def install_rabbit(rmq_install_dir): "Skipping rabbitmq server install".format( rmq_install_dir, rabbitmq_server)) else: - url = "https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.7.7/rabbitmq-server-generic-unix-3.7.7.tar.xz" + url = f"https://github.com/rabbitmq/rabbitmq-server/releases/download/v{rmq_version}/rabbitmq-server-generic-unix-{rmq_version}.tar.xz" f = urlopen(url) data = f.read() filename = "rabbitmq-server.download.tar.xz" @@ -240,8 +227,8 @@ def main(argv=sys.argv): sys.exit(77) # Python3 for life! - if sys.version_info.major < 3 or sys.version_info.minor < 6: - sys.stderr.write('error: Python >= 3.6 is required\n') + if sys.version_info.major < 3 or sys.version_info.minor < 8: + sys.stderr.write('error: Python >= 3.8 is required\n') sys.exit(1) # Build the parser @@ -336,9 +323,6 @@ def main(argv=sys.argv): '--offline', action='store_true', default=False, help='install from cache without downloading') ex = up.add_mutually_exclusive_group() - ex.add_argument( - '-u', '--upgrade', action='store_true', default=False, - help='upgrade installed packages') ex.add_argument( '-w', '--wheel', action='store_const', const='wheel', dest='operation', help='build wheels in the pip wheelhouse') @@ -361,8 +345,7 @@ def main(argv=sys.argv): # Main script logic to perform bootstrapping or updating if sys.base_prefix != sys.prefix: # The script was called from a virtual environment Python, so update - update(options.operation, options.verbose, - options.upgrade, options.offline, options.optional_args, options.rabbitmq) + update(options.operation, options.verbose, options.offline, options.optional_args, options.rabbitmq) else: # The script was called from the system Python, so bootstrap try: diff --git a/ci-integration/run-test-docker.sh b/ci-integration/run-test-docker.sh index 8e282e5f07..c982a7e993 100755 --- a/ci-integration/run-test-docker.sh +++ b/ci-integration/run-test-docker.sh @@ -114,7 +114,7 @@ process_pid(){ if [[ ${FAST_FAIL} -eq 0 && -n ${CI} ]]; then docker logs "${containernames[$index]}" fi - if [ ${FAST_FAIL} ]; then + if [ "${FAST_FAIL}" ]; then echo "Exiting cleanly now!" exit_cleanly else @@ -141,7 +141,7 @@ process_pid(){ #LOOP through set of directories and run bunch of test files in parallel for dir in "${testdirs[@]}" do - for file in $( find $dir -type f -name "test*.py" -o -name "*test.py" ! -name "*conftest.py" ) + for file in $( find "$dir" -type f -name "test*.py" -o -name "*test.py" ! -name "*conftest.py" ) do echo "$file"; ignore=0 diff --git a/ci-integration/run-tests.sh b/ci-integration/run-tests.sh index 6895e64379..e0ede38058 100755 --- a/ci-integration/run-tests.sh +++ b/ci-integration/run-tests.sh @@ -46,12 +46,12 @@ echo "bootstrapping RABBITMQ" python bootstrap.py --rabbitmq --market echo "rabbitmq status" -$HOME/rabbitmq_server/rabbitmq_server-3.7.7/sbin/rabbitmqctl status +"$HOME/rabbitmq_server/rabbitmq_server-3.9.29/sbin/rabbitmqctl" status echo "TestDirs" for dir in $testdirs; do echo "*********TESTDIR: $dir" - py.test -s -v $dir + pytest -s -v "$dir" tmp_code=$? exit_code=$tmp_code @@ -71,7 +71,7 @@ for dir in $splitdirs; do for D in $dir; do for p in $testdirs; do - if [ "$p" == "$d" ]; then + if [ "$p" = "$D" ]; then echo "ALREADY TESTED DIR: $p"; continue; fi; @@ -79,7 +79,7 @@ for dir in $splitdirs; do if [ -d "${D}" ]; then echo "*********SPLITDIR: $D" - py.test -s -v ${D} + pytest -s -v "${D}" tmp_code=$? if [ $tmp_code -ne 0 ]; then if [ $tmp_code -ne 5 ]; then @@ -97,14 +97,14 @@ done echo "File tests" for dir in $filedirs; do echo "File test for dir: $dir" - for testfile in $dir/*.py; do + for testfile in "$dir"/*.py; do echo "Using testfile: $testfile" - if [ $testfile != "volttrontesting/platform/packaging-tests.py" ]; then - py.test -s -v $testfile + if [ "$testfile" != "volttrontesting/platform/packaging-tests.py" ]; then + pytest -s -v "$testfile" tmp_code=$? exit_code=$tmp_code - echo $exit_code + echo "$exit_code" if [ $tmp_code -ne 0 ]; then if [ $tmp_code -ne 5 ]; then if [ ${FAST_FAIL} ]; then diff --git a/ci-integration/virtualization/Dockerfile b/ci-integration/virtualization/Dockerfile index 7cc7e7c594..18af962aac 100644 --- a/ci-integration/virtualization/Dockerfile +++ b/ci-integration/virtualization/Dockerfile @@ -31,8 +31,8 @@ RUN chmod +x /startup/entrypoint.sh && \ USER $VOLTTRON_USER RUN mkdir $RMQ_ROOT RUN set -eux \ - && wget -P $VOLTTRON_USER_HOME https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.7.7/rabbitmq-server-generic-unix-3.7.7.tar.xz \ - && tar -xf $VOLTTRON_USER_HOME/rabbitmq-server-generic-unix-3.7.7.tar.xz --directory $RMQ_ROOT \ + && wget -P $VOLTTRON_USER_HOME https://github.com/rabbitmq/rabbitmq-server/releases/download/v3.9.29/rabbitmq-server-generic-unix-3.9.29.tar.xz \ + && tar -xf $VOLTTRON_USER_HOME/rabbitmq-server-generic-unix-3.9.29.tar.xz --directory $RMQ_ROOT \ && $RMQ_HOME/sbin/rabbitmq-plugins enable rabbitmq_management rabbitmq_federation rabbitmq_federation_management rabbitmq_shovel rabbitmq_shovel_management rabbitmq_auth_mechanism_ssl rabbitmq_trust_store RUN python3 -m pip install gevent-pika --user ############################################ diff --git a/ci-integration/virtualization/core/entrypoint.sh b/ci-integration/virtualization/core/entrypoint.sh index 8532454888..0ecbbd1d37 100644 --- a/ci-integration/virtualization/core/entrypoint.sh +++ b/ci-integration/virtualization/core/entrypoint.sh @@ -21,9 +21,9 @@ export HOME=${VOLTTRON_USER_HOME} # Add the pip user bin to the path since we aren't using the # virtualenv environment in the distribution. export PATH=$HOME/.local/bin:$PATH -VOLTTRON_UID_ORIGINAL=`id -u volttron` +VOLTTRON_UID_ORIGINAL=$(id -u volttron) -if [ $VOLTTRON_UID_ORIGINAL != $USER_ID ]; then +if [ "$VOLTTRON_UID_ORIGINAL" != "$USER_ID" ]; then echo "Changing volttron USER_ID to match passed LOCAL_USER_ID ${USER_ID} " usermod -u $USER_ID volttron fi diff --git a/ci-integration/virtualization/requirements_test.txt b/ci-integration/virtualization/requirements_test.txt index 7b8e453b09..5af6c6ee27 100644 --- a/ci-integration/virtualization/requirements_test.txt +++ b/ci-integration/virtualization/requirements_test.txt @@ -7,7 +7,7 @@ numpy>1.13<2 pandas watchdog==0.10.2 watchdog-gevent==0.1.1 -cryptography==2.3 +cryptography docker psycopg2 mysql-connector-python-rf diff --git a/debugging_utils/peerlist-watcher.py b/debugging_utils/peerlist-watcher.py index e5b681438f..715498590c 100644 --- a/debugging_utils/peerlist-watcher.py +++ b/debugging_utils/peerlist-watcher.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} """ @@ -50,7 +36,6 @@ """ import os -from pathlib import Path from typing import Optional import gevent @@ -61,10 +46,11 @@ def get_volttron_home(): - if Path("/tmp/volttron_home.txt").exists(): - with open("/tmp/volttron_home.txt") as fp: + try: + with open("/tmp/volttron_home.txt", "r") as fp: return fp.read().strip() - return None + except FileNotFoundError: + return None def get_public_private_key(volttron_home): @@ -111,4 +97,3 @@ def get_public_private_key(volttron_home): print(last_output) gevent.sleep(0.1) - diff --git a/services/core/Ambient/README.md b/deprecated/Ambient/README.md similarity index 100% rename from services/core/Ambient/README.md rename to deprecated/Ambient/README.md diff --git a/services/core/Ambient/ambient/__init__.py b/deprecated/Ambient/ambient/__init__.py similarity index 100% rename from services/core/Ambient/ambient/__init__.py rename to deprecated/Ambient/ambient/__init__.py diff --git a/services/core/Ambient/ambient/agent.py b/deprecated/Ambient/ambient/agent.py similarity index 81% rename from services/core/Ambient/ambient/agent.py rename to deprecated/Ambient/ambient/agent.py index 1de617306e..e69b55c9e1 100644 --- a/services/core/Ambient/ambient/agent.py +++ b/deprecated/Ambient/ambient/agent.py @@ -1,58 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright (c) 2017, Battelle Memorial Institute -# All rights reserved. +# Component of Eclipse VOLTTRON # -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: +# ===----------------------------------------------------------------------=== # -# 1. Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. +# Copyright 2023 Battelle Memorial Institute # -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -# FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at # -# The views and conclusions contained in the software and documentation are -# those of the authors and should not be interpreted as representing official, -# policies either expressed or implied, of the FreeBSD Project. +# http://www.apache.org/licenses/LICENSE-2.0 # - -# This material was prepared as an account of work sponsored by an -# agency of the United States Government. Neither the United States -# Government nor the United States Department of Energy, nor Battelle, -# nor any of their employees, nor any jurisdiction or organization -# that has cooperated in the development of these materials, makes -# any warranty, express or implied, or assumes any legal liability -# or responsibility for the accuracy, completeness, or usefulness or -# any information, apparatus, product, software, or process disclosed, -# or represents that its use would not infringe privately owned rights. -# -# Reference herein to any specific commercial product, process, or -# service by trade name, trademark, manufacturer, or otherwise does -# not necessarily constitute or imply its endorsement, recommendation, -# r favoring by the United States Government or any agency thereof, -# or Battelle Memorial Institute. The views and opinions of authors -# expressed herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY -# operated by BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 - +# ===----------------------------------------------------------------------=== # }}} __docformat__ = 'reStructuredText' diff --git a/services/core/Ambient/ambient/data/name_mapping.csv b/deprecated/Ambient/ambient/data/name_mapping.csv similarity index 100% rename from services/core/Ambient/ambient/data/name_mapping.csv rename to deprecated/Ambient/ambient/data/name_mapping.csv diff --git a/services/core/Ambient/config b/deprecated/Ambient/config similarity index 100% rename from services/core/Ambient/config rename to deprecated/Ambient/config diff --git a/services/core/Ambient/conftest.py b/deprecated/Ambient/conftest.py similarity index 100% rename from services/core/Ambient/conftest.py rename to deprecated/Ambient/conftest.py diff --git a/services/core/Ambient/requirements.txt b/deprecated/Ambient/requirements.txt similarity index 100% rename from services/core/Ambient/requirements.txt rename to deprecated/Ambient/requirements.txt diff --git a/deprecated/Ambient/setup.py b/deprecated/Ambient/setup.py new file mode 100644 index 0000000000..94d0434a01 --- /dev/null +++ b/deprecated/Ambient/setup.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2023 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + +from os import path +from setuptools import setup, find_packages + +MAIN_MODULE = 'agent' + +# Find the agent package that contains the main module +packages = find_packages('.') +agent_package = '' + +for package in find_packages(): + # Because there could be other packages such as tests + if path.isfile(package + '/' + MAIN_MODULE + '.py') is True: + agent_package = package +if not agent_package: + raise RuntimeError('None of the packages under {dir} contain the file ' + '{main_module}'.format(main_module=MAIN_MODULE + '.py', + dir=path.abspath(''))) + +# Find the version number from the main module +agent_module = agent_package + '.' + MAIN_MODULE +_temp = __import__(agent_module, globals(), locals(), ['__version__'], 0) +__version__ = _temp.__version__ + +# Setup +setup( + name=agent_package + 'agent', + version=__version__, + description='Agent for interfacing with the Ambient weather station API service', + install_requires=['volttron'], + packages=packages, + package_data={'ambient': ['data/name_mapping.csv']}, + entry_points={ + 'setuptools.installation': [ + 'eggsecutable = ' + agent_module + ':main', + ] + } +) diff --git a/services/core/Ambient/tests/test_ambient_agent.py b/deprecated/Ambient/tests/test_ambient_agent.py similarity index 87% rename from services/core/Ambient/tests/test_ambient_agent.py rename to deprecated/Ambient/tests/test_ambient_agent.py index 6c7d56c448..64757dcef8 100644 --- a/services/core/Ambient/tests/test_ambient_agent.py +++ b/deprecated/Ambient/tests/test_ambient_agent.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2017, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} import pytest @@ -347,5 +333,3 @@ def test_polling_locations_valid_config(volttron_instance, query_agent, config, if agent_uuid: volttron_instance.stop_agent(agent_uuid) volttron_instance.remove_agent(agent_uuid) - - diff --git a/services/core/CrateHistorian/README.md b/deprecated/CrateHistorian/README.md similarity index 100% rename from services/core/CrateHistorian/README.md rename to deprecated/CrateHistorian/README.md diff --git a/services/core/CrateHistorian/config b/deprecated/CrateHistorian/config similarity index 100% rename from services/core/CrateHistorian/config rename to deprecated/CrateHistorian/config diff --git a/services/core/CrateHistorian/conftest.py b/deprecated/CrateHistorian/conftest.py similarity index 100% rename from services/core/CrateHistorian/conftest.py rename to deprecated/CrateHistorian/conftest.py diff --git a/services/core/CrateHistorian/cratedb/__init__.py b/deprecated/CrateHistorian/cratedb/__init__.py similarity index 100% rename from services/core/CrateHistorian/cratedb/__init__.py rename to deprecated/CrateHistorian/cratedb/__init__.py diff --git a/services/core/CrateHistorian/cratedb/historian.py b/deprecated/CrateHistorian/cratedb/historian.py similarity index 92% rename from services/core/CrateHistorian/cratedb/historian.py rename to deprecated/CrateHistorian/cratedb/historian.py index 2e57e8545f..b6368b2b50 100644 --- a/services/core/CrateHistorian/cratedb/historian.py +++ b/deprecated/CrateHistorian/cratedb/historian.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} import logging @@ -167,9 +153,9 @@ def configure(self, configuration): """ The expectation that configuration will have at least the following items - + .. code: python - + { "connection": { "params": { @@ -177,8 +163,8 @@ def configure(self, configuration): } } } - - :param configuration: + + :param configuration: """ connection = configuration.get("connection", {}) tables_def, table_names = self.parse_table_def(configuration.get("tables_def")) diff --git a/services/core/CrateHistorian/requirements.txt b/deprecated/CrateHistorian/requirements.txt similarity index 100% rename from services/core/CrateHistorian/requirements.txt rename to deprecated/CrateHistorian/requirements.txt diff --git a/services/core/CrateHistorian/scripts/crate_config b/deprecated/CrateHistorian/scripts/crate_config similarity index 100% rename from services/core/CrateHistorian/scripts/crate_config rename to deprecated/CrateHistorian/scripts/crate_config diff --git a/services/core/CrateHistorian/scripts/drop_tables.py b/deprecated/CrateHistorian/scripts/drop_tables.py similarity index 100% rename from services/core/CrateHistorian/scripts/drop_tables.py rename to deprecated/CrateHistorian/scripts/drop_tables.py diff --git a/services/core/CrateHistorian/scripts/mongo_config b/deprecated/CrateHistorian/scripts/mongo_config similarity index 100% rename from services/core/CrateHistorian/scripts/mongo_config rename to deprecated/CrateHistorian/scripts/mongo_config diff --git a/services/core/CrateHistorian/scripts/mongo_to_crate.py b/deprecated/CrateHistorian/scripts/mongo_to_crate.py similarity index 100% rename from services/core/CrateHistorian/scripts/mongo_to_crate.py rename to deprecated/CrateHistorian/scripts/mongo_to_crate.py diff --git a/deprecated/CrateHistorian/setup.py b/deprecated/CrateHistorian/setup.py new file mode 100644 index 0000000000..ce3fdb17dd --- /dev/null +++ b/deprecated/CrateHistorian/setup.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2023 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + +from os import path +from setuptools import setup, find_packages + +MAIN_MODULE = 'historian' + +# Find the agent package that contains the main module +packages = find_packages('.') +agent_package = '' +for package in find_packages(): + # Because there could be other packages such as tests + if path.isfile(package + '/' + MAIN_MODULE + '.py') is True: + agent_package = package +if not agent_package: + raise RuntimeError('None of the packages under {dir} contain the file ' + '{main_module}'.format(main_module=MAIN_MODULE + '.py', + dir=path.abspath(''))) + +# Find the version number from the main module +agent_module = agent_package + '.' + MAIN_MODULE +_temp = __import__(agent_module, globals(), locals(), ['__version__'], 0) +__version__ = _temp.__version__ + +# Setup +setup( + name=agent_package + 'agent', + version=__version__, + install_requires=['volttron'], + packages=packages, + entry_points={ + 'setuptools.installation': [ + 'eggsecutable = ' + agent_module + ':main', + ] + } +) diff --git a/services/core/CrateHistorian/tests/test_crate_historian.py b/deprecated/CrateHistorian/tests/test_crate_historian.py similarity index 73% rename from services/core/CrateHistorian/tests/test_crate_historian.py rename to deprecated/CrateHistorian/tests/test_crate_historian.py index fd57b0b61a..47c2c20f57 100644 --- a/services/core/CrateHistorian/tests/test_crate_historian.py +++ b/deprecated/CrateHistorian/tests/test_crate_historian.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} diff --git a/services/core/Darksky/README.md b/deprecated/Darksky/README.md similarity index 100% rename from services/core/Darksky/README.md rename to deprecated/Darksky/README.md diff --git a/services/core/Darksky/config b/deprecated/Darksky/config similarity index 100% rename from services/core/Darksky/config rename to deprecated/Darksky/config diff --git a/services/core/Darksky/conftest.py b/deprecated/Darksky/conftest.py similarity index 100% rename from services/core/Darksky/conftest.py rename to deprecated/Darksky/conftest.py diff --git a/services/core/DNP3Agent/dnp3/mesa/__init__.py b/deprecated/Darksky/darksky/__init__.py similarity index 100% rename from services/core/DNP3Agent/dnp3/mesa/__init__.py rename to deprecated/Darksky/darksky/__init__.py diff --git a/services/core/Darksky/darksky/agent.py b/deprecated/Darksky/darksky/agent.py similarity index 93% rename from services/core/Darksky/darksky/agent.py rename to deprecated/Darksky/darksky/agent.py index dc547cb5ac..647a22041f 100644 --- a/services/core/Darksky/darksky/agent.py +++ b/deprecated/Darksky/darksky/agent.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} __docformat__ = 'reStructuredText' @@ -72,7 +58,7 @@ 'get_minutely_forecast': {'json_name': 'minutely', 'type': 'forecast'} } -LAT_LONG_REGEX = re.compile("^-?[0-9]{1,3}(\.[0-9]{1,4})?$") +LAT_LONG_REGEX = re.compile(r"^-?[0-9]{1,3}(\.[0-9]{1,4})?$") def darksky(config_path, **kwargs): @@ -525,7 +511,7 @@ def generate_response_error(self, url, response_code): def main(): """Main method called to start the agent.""" - utils.vip_main(darksky, + utils.vip_main(darksky, version=__version__) diff --git a/services/core/Darksky/darksky/data/name_mapping.csv b/deprecated/Darksky/darksky/data/name_mapping.csv similarity index 100% rename from services/core/Darksky/darksky/data/name_mapping.csv rename to deprecated/Darksky/darksky/data/name_mapping.csv diff --git a/services/core/Darksky/requirements.txt b/deprecated/Darksky/requirements.txt similarity index 100% rename from services/core/Darksky/requirements.txt rename to deprecated/Darksky/requirements.txt diff --git a/deprecated/Darksky/setup.py b/deprecated/Darksky/setup.py new file mode 100644 index 0000000000..202a385e40 --- /dev/null +++ b/deprecated/Darksky/setup.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2023 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + + +from os import path +from setuptools import setup, find_packages + +MAIN_MODULE = 'agent' + +# Find the agent package that contains the main module +packages = find_packages('.') +agent_package = '' + +for package in find_packages(): + # Because there could be other packages such as tests + if path.isfile(package + '/' + MAIN_MODULE + '.py') is True: + agent_package = package +if not agent_package: + raise RuntimeError('None of the packages under {dir} contain the file ' + '{main_module}'.format(main_module=MAIN_MODULE + '.py', + dir=path.abspath(''))) + +# Find the version number from the main module +agent_module = agent_package + '.' + MAIN_MODULE +# -1 not valid import level in python3 +_temp = __import__(agent_module, globals(), locals(), ['__version__'], 0) +__version__ = _temp.__version__ + +# Setup +setup( + name=agent_package + 'agent', + version=__version__, + description="Agent for interfacing with the Darksky Weather API service", + install_requires=['volttron'], + packages=packages, + package_data={'darksky': ['data/name_mapping.csv']}, + entry_points={ + 'setuptools.installation': [ + 'eggsecutable = ' + agent_module + ':main', + ] + } +) diff --git a/services/core/Darksky/tests/test_darksky.py b/deprecated/Darksky/tests/test_darksky.py similarity index 94% rename from services/core/Darksky/tests/test_darksky.py rename to deprecated/Darksky/tests/test_darksky.py index a4a1b93b24..38c62cb95a 100644 --- a/services/core/Darksky/tests/test_darksky.py +++ b/deprecated/Darksky/tests/test_darksky.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} import pytest @@ -365,11 +351,10 @@ def test_success_forecast(volttron_instance, cleanup_cache, weather, query_agent num_records = cursor.fetchone()[0] if service_name == service: assert num_records is records_amount * len(locations) + elif identity == 'platform.darksky_perf': + assert num_records is 0 else: - if identity == 'platform.darksky_perf': - assert num_records is 0 - else: - assert num_records is records_amount * len(locations) + assert num_records is records_amount * len(locations) assert len(query_data) == len(locations) diff --git a/services/core/ExternalData/README.md b/deprecated/ExternalData/README.md similarity index 100% rename from services/core/ExternalData/README.md rename to deprecated/ExternalData/README.md diff --git a/services/core/ExternalData/config b/deprecated/ExternalData/config similarity index 100% rename from services/core/ExternalData/config rename to deprecated/ExternalData/config diff --git a/services/core/DNP3Agent/tests/MesaTestAgent/testagent/__init__.py b/deprecated/ExternalData/external_data/__init__.py similarity index 100% rename from services/core/DNP3Agent/tests/MesaTestAgent/testagent/__init__.py rename to deprecated/ExternalData/external_data/__init__.py diff --git a/services/core/ExternalData/external_data/agent.py b/deprecated/ExternalData/external_data/agent.py similarity index 86% rename from services/core/ExternalData/external_data/agent.py rename to deprecated/ExternalData/external_data/agent.py index 0a2ba4fd41..52dbb9c599 100644 --- a/services/core/ExternalData/external_data/agent.py +++ b/deprecated/ExternalData/external_data/agent.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} import logging diff --git a/deprecated/ExternalData/setup.py b/deprecated/ExternalData/setup.py new file mode 100644 index 0000000000..2bcb2c0b88 --- /dev/null +++ b/deprecated/ExternalData/setup.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2023 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + +from os import path +from setuptools import setup, find_packages + +MAIN_MODULE = 'agent' + +# Find the agent package that contains the main module +packages = find_packages('.') +agent_package = '' +for package in find_packages(): + # Because there could be other packages such as tests + if path.isfile(package + '/' + MAIN_MODULE + '.py') is True: + agent_package = package +if not agent_package: + raise RuntimeError('None of the packages under {dir} contain the file ' + '{main_module}'.format(main_module=MAIN_MODULE + '.py', + dir=path.abspath(''))) + +# Find the version number from the main module +agent_module = agent_package + '.' + MAIN_MODULE +_temp = __import__(agent_module, globals(), locals(), ['__version__'], 0) +__version__ = _temp.__version__ + +# Setup +setup( + name=agent_package + 'agent', + version=__version__, + install_requires=['volttron'], + packages=packages, + entry_points={ + 'setuptools.installation': [ + 'eggsecutable = ' + agent_module + ':main', + ] + } +) diff --git a/deprecated/FailoverAgent/failover/agent.py b/deprecated/FailoverAgent/failover/agent.py index a5d233baf4..f026abc1e9 100644 --- a/deprecated/FailoverAgent/failover/agent.py +++ b/deprecated/FailoverAgent/failover/agent.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} diff --git a/deprecated/FailoverAgent/setup.py b/deprecated/FailoverAgent/setup.py index cc64769bff..5284d29efb 100644 --- a/deprecated/FailoverAgent/setup.py +++ b/deprecated/FailoverAgent/setup.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} from os import path diff --git a/services/unsupported/MongodbAggregateHistorian/README.md b/deprecated/MongodbAggregateHistorian/README.md similarity index 100% rename from services/unsupported/MongodbAggregateHistorian/README.md rename to deprecated/MongodbAggregateHistorian/README.md diff --git a/services/core/Darksky/darksky/__init__.py b/deprecated/MongodbAggregateHistorian/aggregator/__init__.py similarity index 100% rename from services/core/Darksky/darksky/__init__.py rename to deprecated/MongodbAggregateHistorian/aggregator/__init__.py diff --git a/services/unsupported/MongodbAggregateHistorian/aggregator/aggregator.py b/deprecated/MongodbAggregateHistorian/aggregator/aggregator.py similarity index 82% rename from services/unsupported/MongodbAggregateHistorian/aggregator/aggregator.py rename to deprecated/MongodbAggregateHistorian/aggregator/aggregator.py index 30f3417203..e0ee5f70e7 100644 --- a/services/unsupported/MongodbAggregateHistorian/aggregator/aggregator.py +++ b/deprecated/MongodbAggregateHistorian/aggregator/aggregator.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} diff --git a/services/unsupported/MongodbAggregateHistorian/config b/deprecated/MongodbAggregateHistorian/config similarity index 100% rename from services/unsupported/MongodbAggregateHistorian/config rename to deprecated/MongodbAggregateHistorian/config diff --git a/services/unsupported/MongodbAggregateHistorian/conftest.py b/deprecated/MongodbAggregateHistorian/conftest.py similarity index 100% rename from services/unsupported/MongodbAggregateHistorian/conftest.py rename to deprecated/MongodbAggregateHistorian/conftest.py diff --git a/services/unsupported/MongodbAggregateHistorian/requirements.txt b/deprecated/MongodbAggregateHistorian/requirements.txt similarity index 100% rename from services/unsupported/MongodbAggregateHistorian/requirements.txt rename to deprecated/MongodbAggregateHistorian/requirements.txt diff --git a/deprecated/MongodbAggregateHistorian/setup.py b/deprecated/MongodbAggregateHistorian/setup.py new file mode 100644 index 0000000000..727121352e --- /dev/null +++ b/deprecated/MongodbAggregateHistorian/setup.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2023 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + +from os import path +from setuptools import setup, find_packages + +MAIN_MODULE = 'aggregator' + +# Find the agent package that contains the main module +packages = find_packages('.') +agent_package = '' +for package in find_packages(): + # Because there could be other packages such as tests + if path.isfile(package + '/' + MAIN_MODULE + '.py') is True: + agent_package = package +if not agent_package: + raise RuntimeError('None of the packages under {dir} contain the file ' + '{main_module}'.format(main_module=MAIN_MODULE + '.py', + dir=path.abspath(''))) + +# Find the version number from the main module +agent_module = agent_package + '.' + MAIN_MODULE +_temp = __import__(agent_module, globals(), locals(), ['__version__'], 0) +__version__ = _temp.__version__ + +# Setup +setup( + name=agent_package + 'agent', + version=__version__, + install_requires=['volttron'], + packages=packages, + entry_points={ + 'setuptools.installation': [ + 'eggsecutable = ' + agent_module + ':main', + ] + } +) diff --git a/services/unsupported/MongodbHistorian/README.md b/deprecated/MongodbHistorian/README.md similarity index 100% rename from services/unsupported/MongodbHistorian/README.md rename to deprecated/MongodbHistorian/README.md diff --git a/services/unsupported/MongodbHistorian/config b/deprecated/MongodbHistorian/config similarity index 100% rename from services/unsupported/MongodbHistorian/config rename to deprecated/MongodbHistorian/config diff --git a/services/unsupported/MongodbHistorian/conftest.py b/deprecated/MongodbHistorian/conftest.py similarity index 100% rename from services/unsupported/MongodbHistorian/conftest.py rename to deprecated/MongodbHistorian/conftest.py diff --git a/services/core/ExternalData/external_data/__init__.py b/deprecated/MongodbHistorian/mongodb/__init__.py similarity index 100% rename from services/core/ExternalData/external_data/__init__.py rename to deprecated/MongodbHistorian/mongodb/__init__.py diff --git a/services/unsupported/MongodbHistorian/mongodb/historian.py b/deprecated/MongodbHistorian/mongodb/historian.py similarity index 96% rename from services/unsupported/MongodbHistorian/mongodb/historian.py rename to deprecated/MongodbHistorian/mongodb/historian.py index 8f5b114f74..7c4a465af1 100644 --- a/services/unsupported/MongodbHistorian/mongodb/historian.py +++ b/deprecated/MongodbHistorian/mongodb/historian.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} @@ -930,7 +916,7 @@ def query_topic_list(self): def query_topics_by_pattern(self, topics_pattern): _log.debug("In query topics by pattern: {}".format(topics_pattern)) db = self._client.get_default_database() - topics_pattern = topics_pattern.replace('/', '\/') + topics_pattern = topics_pattern.replace('/', r'\/') pattern = {'topic_name': {'$regex': topics_pattern, '$options': 'i'}} cursor = db[self._topic_collection].find(pattern) topic_id_map = dict() diff --git a/services/unsupported/MongodbHistorian/requirements.txt b/deprecated/MongodbHistorian/requirements.txt similarity index 100% rename from services/unsupported/MongodbHistorian/requirements.txt rename to deprecated/MongodbHistorian/requirements.txt diff --git a/services/unsupported/MongodbHistorian/scripts/count_data_by_topic_pattern.py b/deprecated/MongodbHistorian/scripts/count_data_by_topic_pattern.py similarity index 91% rename from services/unsupported/MongodbHistorian/scripts/count_data_by_topic_pattern.py rename to deprecated/MongodbHistorian/scripts/count_data_by_topic_pattern.py index 872d39ac6b..6078fb22c2 100644 --- a/services/unsupported/MongodbHistorian/scripts/count_data_by_topic_pattern.py +++ b/deprecated/MongodbHistorian/scripts/count_data_by_topic_pattern.py @@ -21,5 +21,5 @@ count = count + db.data.find( {"topic_id":x['_id'], "ts":{"$gte":s_dt, "$lt":e_dt}}).count() -print (count) -print ("time taken: {}".format(datetime.datetime.now()-start)) \ No newline at end of file +print(count) +print("time taken: {}".format(datetime.datetime.now() - start)) diff --git a/services/unsupported/MongodbHistorian/scripts/rollup_data_by_time.py b/deprecated/MongodbHistorian/scripts/rollup_data_by_time.py similarity index 97% rename from services/unsupported/MongodbHistorian/scripts/rollup_data_by_time.py rename to deprecated/MongodbHistorian/scripts/rollup_data_by_time.py index c351d4713d..52514f00e4 100644 --- a/services/unsupported/MongodbHistorian/scripts/rollup_data_by_time.py +++ b/deprecated/MongodbHistorian/scripts/rollup_data_by_time.py @@ -236,7 +236,7 @@ def execute_batch(table_type, bulk, count, topic_id, topic_name): "bulk execute of {} data for {}:{}.\nnumber of op sent to " "bulk execute ({}) does not match nModified count".format( table_type, topic_id, topic_name, count)) - print ("bulk execute result {}".format(result)) + print("bulk execute result {}".format(result)) errors = True except BulkWriteError as ex: print(str(ex.details)) @@ -338,7 +338,7 @@ def init_hourly_data(db, data_collection, start_dt, end_dt): if __name__ == '__main__': start = datetime.utcnow() - print ("Starting rollup of data from {} to {}. current time: {}".format( + print("Starting rollup of data from {} to {}. current time: {}".format( start_date, end_date, start)) pool = Pool(size=10) @@ -370,13 +370,13 @@ def init_hourly_data(db, data_collection, start_dt, end_dt): source_db = connect_mongodb(local_source_params) s_dt = datetime.strptime(start_date, '%d%b%YT%H:%M:%S.%f') e_dt = datetime.strptime(end_date, '%d%b%YT%H:%M:%S.%f') - print ("Starting init of tables") + print("Starting init of tables") init_start = datetime.utcnow() init_daily_data(source_db, source_tables['data_table'], s_dt, e_dt) - print ("Total time for init of daily data " + print("Total time for init of daily data " "between {} and {} : {} " "".format(start_date, end_date, datetime.utcnow() - init_start)) @@ -385,7 +385,7 @@ def init_hourly_data(db, data_collection, start_dt, end_dt): source_tables['data_table'], s_dt, e_dt) - print ("Total time for init of hourly data " + print("Total time for init of hourly data " "between {} and {} : {} " "".format(start_date, end_date, datetime.utcnow() - init_start)) @@ -419,7 +419,7 @@ def init_hourly_data(db, data_collection, start_dt, end_dt): print("Exception processing data: {}".format(e.args)) finally: pool.kill() - print ("Total time for roll up of data : {}".format( + print("Total time for roll up of data : {}".format( datetime.utcnow() - start)) if log: log.close() diff --git a/services/unsupported/MongodbHistorian/scripts/rollup_data_using_groupby.py b/deprecated/MongodbHistorian/scripts/rollup_data_using_groupby.py similarity index 99% rename from services/unsupported/MongodbHistorian/scripts/rollup_data_using_groupby.py rename to deprecated/MongodbHistorian/scripts/rollup_data_using_groupby.py index b67b1f9c55..9f8d603f71 100644 --- a/services/unsupported/MongodbHistorian/scripts/rollup_data_using_groupby.py +++ b/deprecated/MongodbHistorian/scripts/rollup_data_using_groupby.py @@ -32,13 +32,13 @@ def connect_mongodb(connection_params): - print ("setup mongodb") + print("setup mongodb") mongo_conn_str = 'mongodb://{user}:{passwd}@{host}:{port}/{database}' if connection_params.get('authSource'): mongo_conn_str = mongo_conn_str+ '?authSource={authSource}' params = connection_params mongo_conn_str = mongo_conn_str.format(**params) - print (mongo_conn_str) + print(mongo_conn_str) mongo_client = pymongo.MongoClient(mongo_conn_str) db = mongo_client[connection_params['database']] return db diff --git a/deprecated/MongodbHistorian/setup.py b/deprecated/MongodbHistorian/setup.py new file mode 100644 index 0000000000..ce3fdb17dd --- /dev/null +++ b/deprecated/MongodbHistorian/setup.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2023 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + +from os import path +from setuptools import setup, find_packages + +MAIN_MODULE = 'historian' + +# Find the agent package that contains the main module +packages = find_packages('.') +agent_package = '' +for package in find_packages(): + # Because there could be other packages such as tests + if path.isfile(package + '/' + MAIN_MODULE + '.py') is True: + agent_package = package +if not agent_package: + raise RuntimeError('None of the packages under {dir} contain the file ' + '{main_module}'.format(main_module=MAIN_MODULE + '.py', + dir=path.abspath(''))) + +# Find the version number from the main module +agent_module = agent_package + '.' + MAIN_MODULE +_temp = __import__(agent_module, globals(), locals(), ['__version__'], 0) +__version__ = _temp.__version__ + +# Setup +setup( + name=agent_package + 'agent', + version=__version__, + install_requires=['volttron'], + packages=packages, + entry_points={ + 'setuptools.installation': [ + 'eggsecutable = ' + agent_module + ':main', + ] + } +) diff --git a/services/unsupported/MongodbHistorian/tests/fixtures.py b/deprecated/MongodbHistorian/tests/fixtures.py similarity index 100% rename from services/unsupported/MongodbHistorian/tests/fixtures.py rename to deprecated/MongodbHistorian/tests/fixtures.py diff --git a/services/unsupported/MongodbHistorian/tests/mongod.conf b/deprecated/MongodbHistorian/tests/mongod.conf similarity index 100% rename from services/unsupported/MongodbHistorian/tests/mongod.conf rename to deprecated/MongodbHistorian/tests/mongod.conf diff --git a/services/unsupported/MongodbHistorian/tests/mongosetup.sh b/deprecated/MongodbHistorian/tests/mongosetup.sh similarity index 100% rename from services/unsupported/MongodbHistorian/tests/mongosetup.sh rename to deprecated/MongodbHistorian/tests/mongosetup.sh diff --git a/services/unsupported/MongodbHistorian/tests/purgemongo.sh b/deprecated/MongodbHistorian/tests/purgemongo.sh similarity index 100% rename from services/unsupported/MongodbHistorian/tests/purgemongo.sh rename to deprecated/MongodbHistorian/tests/purgemongo.sh diff --git a/services/unsupported/MongodbHistorian/tests/test_mongohistorian.py b/deprecated/MongodbHistorian/tests/test_mongohistorian.py similarity index 97% rename from services/unsupported/MongodbHistorian/tests/test_mongohistorian.py rename to deprecated/MongodbHistorian/tests/test_mongohistorian.py index b82da3ee33..0ca3144285 100644 --- a/services/unsupported/MongodbHistorian/tests/test_mongohistorian.py +++ b/deprecated/MongodbHistorian/tests/test_mongohistorian.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} import os diff --git a/services/unsupported/MongodbHistorian/tests/test_prod_query_mongo.py b/deprecated/MongodbHistorian/tests/test_prod_query_mongo.py similarity index 100% rename from services/unsupported/MongodbHistorian/tests/test_prod_query_mongo.py rename to deprecated/MongodbHistorian/tests/test_prod_query_mongo.py diff --git a/services/core/ObixHistoryPublish/README.md b/deprecated/ObixHistoryPublish/README.md similarity index 100% rename from services/core/ObixHistoryPublish/README.md rename to deprecated/ObixHistoryPublish/README.md diff --git a/services/core/ObixHistoryPublish/config b/deprecated/ObixHistoryPublish/config similarity index 100% rename from services/core/ObixHistoryPublish/config rename to deprecated/ObixHistoryPublish/config diff --git a/services/core/IEEE2030_5Agent/tests/IEEE2030_5DriverTestAgent/__init__.py b/deprecated/ObixHistoryPublish/obix_history/__init__.py similarity index 100% rename from services/core/IEEE2030_5Agent/tests/IEEE2030_5DriverTestAgent/__init__.py rename to deprecated/ObixHistoryPublish/obix_history/__init__.py diff --git a/services/core/ObixHistoryPublish/obix_history/agent.py b/deprecated/ObixHistoryPublish/obix_history/agent.py similarity index 90% rename from services/core/ObixHistoryPublish/obix_history/agent.py rename to deprecated/ObixHistoryPublish/obix_history/agent.py index 091b7ca3f2..b4aaacee01 100644 --- a/services/core/ObixHistoryPublish/obix_history/agent.py +++ b/deprecated/ObixHistoryPublish/obix_history/agent.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} __docformat__ = 'reStructuredText' @@ -100,7 +86,7 @@ def obix_history(config_path, **kwargs): **kwargs) -class Register(object): +class Register: def __init__(self, url, device_topic, point_name, obix_point, last_read): self.url = url diff --git a/services/core/ObixHistoryPublish/registry_config.csv b/deprecated/ObixHistoryPublish/registry_config.csv similarity index 100% rename from services/core/ObixHistoryPublish/registry_config.csv rename to deprecated/ObixHistoryPublish/registry_config.csv diff --git a/deprecated/ObixHistoryPublish/setup.py b/deprecated/ObixHistoryPublish/setup.py new file mode 100644 index 0000000000..bb876a55fe --- /dev/null +++ b/deprecated/ObixHistoryPublish/setup.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2023 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + +from setuptools import setup, find_packages + +MAIN_MODULE = 'agent' + +# Find the agent package that contains the main module +packages = find_packages('.') +agent_package = 'obix_history' + +# Find the version number from the main module +agent_module = agent_package + '.' + MAIN_MODULE +_temp = __import__(agent_module, globals(), locals(), ['__version__'], 0) +__version__ = _temp.__version__ + +# Setup +setup( + name=agent_package + 'agent', + version=__version__, + author_email="kyle.monson@pnnl.gov", + description="Retrieve and publish historical data from Obix service.", + author="Kyle Monson", + install_requires=['volttron'], + packages=packages, + entry_points={ + 'setuptools.installation': [ + 'eggsecutable = ' + agent_module + ':main', + ] + } +) diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/IEEE2030_5.py b/deprecated/OldPlatformDrivers/Old2030_5Driver/IEEE2030_5.py similarity index 86% rename from services/core/PlatformDriverAgent/platform_driver/interfaces/IEEE2030_5.py rename to deprecated/OldPlatformDrivers/Old2030_5Driver/IEEE2030_5.py index 3c24056045..30fd686647 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/IEEE2030_5.py +++ b/deprecated/OldPlatformDrivers/Old2030_5Driver/IEEE2030_5.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} from datetime import datetime, timedelta diff --git a/services/core/PlatformDriverAgent/platform_driver/interfaces/dnp3.py b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py similarity index 99% rename from services/core/PlatformDriverAgent/platform_driver/interfaces/dnp3.py rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py index 93e1690465..90334d57ed 100644 --- a/services/core/PlatformDriverAgent/platform_driver/interfaces/dnp3.py +++ b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/platform_driver/interfaces/dnp3.py @@ -35,7 +35,7 @@ from datetime import datetime, timedelta import logging -from . import BaseInterface, BaseRegister, BasicRevert +from services.core.PlatformDriverAgent.platform_driver.interfaces import BaseInterface, BaseRegister, BasicRevert _log = logging.getLogger(__name__) type_mapping = {"string": str, diff --git a/services/core/PlatformDriverAgent/tests/test_dnp3_driver.py b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py similarity index 95% rename from services/core/PlatformDriverAgent/tests/test_dnp3_driver.py rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py index f0a12f4dfb..44fc8d0b06 100644 --- a/services/core/PlatformDriverAgent/tests/test_dnp3_driver.py +++ b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/PlatformDriverAgent/tests/test_dnp3_driver.py @@ -80,7 +80,7 @@ def agent(request, volttron_instance): test_agent = volttron_instance.build_agent() def update_config(agent_id, name, value, cfg_type): - test_agent.vip.rpc.call('config.store', 'manage_store', agent_id, name, value, config_type=cfg_type) + test_agent.vip.rpc.call('config.store', 'set_config', agent_id, name, value, config_type=cfg_type) capabilities = {'edit_config_store': {'identity': PLATFORM_DRIVER}} volttron_instance.add_capabilities(test_agent.core.publickey, capabilities) @@ -95,7 +95,7 @@ def update_config(agent_id, name, value, cfg_type): # Build and start PlatformDriverAgent - test_agent.vip.rpc.call('config.store', 'manage_delete_store', PLATFORM_DRIVER) + test_agent.vip.rpc.call('config.store', 'delete_store', PLATFORM_DRIVER) platform_uuid = volttron_instance.install_agent(agent_dir=get_services_core("PlatformDriverAgent"), config_file={}, diff --git a/docs/source/driver-framework/dnp3-driver/dnp3-driver.rst b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/dnp3-driver.rst similarity index 100% rename from docs/source/driver-framework/dnp3-driver/dnp3-driver.rst rename to deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driver/dnp3-driver.rst diff --git a/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv new file mode 100644 index 0000000000..571d2e9a8e --- /dev/null +++ b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/dnp3.csv @@ -0,0 +1,13 @@ +Volttron Point Name,Group,Index,Scaling,Units,Writable +DCHD.WTgt,41,65,1.0,NA,FALSE +DCHD.WTgt-In,30,90,1.0,NA,TRUE +DCHD.WinTms,41,66,1.0,NA,FALSE +DCHD.RmpTms,41,67,1.0,NA,FALSE +DCHD.RevtTms,41,68,1.0,NA,FALSE +DCHD.RmpUpRte,41,69,1.0,NA,FALSE +DCHD.RmpDnRte,41,70,1.0,NA,FALSE +DCHD.ChaRmpUpRte,41,71,1.0,NA,FALSE +DCHD.ChaRmpDnRte,41,72,1.0,NA,FALSE +DCHD.ModPrty,41,9,1.0,NA,FALSE +DCHD.VArAct,41,10,1.0,NA,FALSE +DCHD.ModEna,12,5,1.0,NA,FALSE diff --git a/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config new file mode 100644 index 0000000000..7296eae4bc --- /dev/null +++ b/deprecated/OldPlatformDrivers/OldDnp3/OldDnp3Driverexamples/configurations/drivers/test_dnp3.config @@ -0,0 +1,13 @@ +{ + "driver_config": { + "dnp3_agent_id": "dnp3agent" + }, + "campus": "campus", + "building": "building", + "unit": "dnp3", + "driver_type": "dnp3", + "registry_config": "config://dnp3.csv", + "interval": 15, + "timezone": "US/Pacific", + "heart_beat_point": "Heartbeat" +} \ No newline at end of file diff --git a/services/core/OpenEISHistorian/README.md b/deprecated/OpenEISHistorian/README.md similarity index 100% rename from services/core/OpenEISHistorian/README.md rename to deprecated/OpenEISHistorian/README.md diff --git a/services/core/OpenEISHistorian/config b/deprecated/OpenEISHistorian/config similarity index 100% rename from services/core/OpenEISHistorian/config rename to deprecated/OpenEISHistorian/config diff --git a/services/core/IEEE2030_5Agent/tests/IEEE2030_5DriverTestAgent/test_agent/__init__.py b/deprecated/OpenEISHistorian/openeis/__init__.py similarity index 100% rename from services/core/IEEE2030_5Agent/tests/IEEE2030_5DriverTestAgent/test_agent/__init__.py rename to deprecated/OpenEISHistorian/openeis/__init__.py diff --git a/services/core/OpenEISHistorian/openeis/historian.py b/deprecated/OpenEISHistorian/openeis/historian.py similarity index 73% rename from services/core/OpenEISHistorian/openeis/historian.py rename to deprecated/OpenEISHistorian/openeis/historian.py index 585183d760..482d09b09f 100644 --- a/services/core/OpenEISHistorian/openeis/historian.py +++ b/deprecated/OpenEISHistorian/openeis/historian.py @@ -1,39 +1,25 @@ # -*- coding: utf-8 -*- {{{ -# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: +# ===----------------------------------------------------------------------=== # -# Copyright 2020, Battelle Memorial Institute. +# Component of Eclipse VOLTTRON # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# ===----------------------------------------------------------------------=== # -# http://www.apache.org/licenses/LICENSE-2.0 +# Copyright 2023 Battelle Memorial Institute # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 # -# This material was prepared as an account of work sponsored by an agency of -# the United States Government. Neither the United States Government nor the -# United States Department of Energy, nor Battelle, nor any of their -# employees, nor any jurisdiction or organization that has cooperated in the -# development of these materials, makes any warranty, express or -# implied, or assumes any legal liability or responsibility for the accuracy, -# completeness, or usefulness or any information, apparatus, product, -# software, or process disclosed, or represents that its use would not infringe -# privately owned rights. Reference herein to any specific commercial product, -# process, or service by trade name, trademark, manufacturer, or otherwise -# does not necessarily constitute or imply its endorsement, recommendation, or -# favoring by the United States Government or any agency thereof, or -# Battelle Memorial Institute. The views and opinions of authors expressed -# herein do not necessarily state or reflect those of the -# United States Government or any agency thereof. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. # -# PACIFIC NORTHWEST NATIONAL LABORATORY operated by -# BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY -# under Contract DE-AC05-76RL01830 +# ===----------------------------------------------------------------------=== # }}} @@ -57,16 +43,16 @@ def historian(config_path, **kwargs): config = utils.load_config(config_path) connection = config.get('connection') - + assert connection assert connection.get('type') == 'openeis' - + params = connection.get('params') assert params - + uri = params.get('uri') assert uri - + login = params.get('login') assert login @@ -74,61 +60,61 @@ def historian(config_path, **kwargs): assert password # Auth will get passed to the server through the requests python framework. auth = (login, password) - + datasets = config.get("dataset_definitions") assert datasets assert len(datasets) > 0 - + headers = {'content-type': 'application/json'} class OpenEISHistorian(BaseHistorian): '''An OpenEIS historian which allows the publishing of dynamic. - - This historian publishes to an openeis instance with the following + + This historian publishes to an openeis instance with the following example json payload: - + dataset_extension = { "dataset_id": dataset_id, - "point_map": + "point_map": { "New building/WholeBuildingPower": [["2/5/2014 10:00",48.78], ["2/5/2014 10:05",50.12], ["2/5/2014 10:10",48.54]], "New building/OutdoorAirTemperature": [["2/5/2014 10:00",48.78], ["2/5/2014 10:05",10.12], ["2/5/2014 10:10",48.54]] } } - + The dataset must exist on the openeis webserver. The mapping (defined) in the configuration file must include both the input (from device) topic and output (openeis schema topic). See openeis.historian.config for a full description of how those are specified in the coniguration file. - + This service will publish to server/api/datasets/append endpoint. ''' @doc_inherit def publish_to_historian(self, to_publish_list): _log.debug("publish_to_historian number of items: {}".format(len(to_publish_list))) - + # print(to_publish_list) dataset_uri = uri + "/api/datasets/append" - + # Build a payload for each of the points in each of the dataset definitions. for dsk, dsv in datasets.items(): ds_id = dsv["dataset_id"] ds_points = dsv['points'] # [unicode(p) for p in dsv['points']] - ignore_unmapped = dsv.get('ignore_unmapped_points', 0) - + ignore_unmapped = dsv.get('ignore_unmapped_points', 0) + point_map = {} try_publish = [] for to_pub in to_publish_list: - for point in ds_points: + for point in ds_points: if to_pub['topic'] in point.keys(): try_publish.append(to_pub) # gets the value of the sensor for publishing. openeis_sensor = point[to_pub['topic']] if not openeis_sensor in point_map: point_map[openeis_sensor] = [] - + point_map[openeis_sensor].append([to_pub['timestamp'], to_pub['value']]) else: if ignore_unmapped: @@ -136,9 +122,9 @@ def publish_to_historian(self, to_publish_list): else: err = 'Point {topic} was not found in point map.'.format(**to_pub) _log.error(err) - + # pprint(point_map) - + if len(point_map) > 0: payload = {'dataset_id': ds_id, 'point_map': point_map} @@ -146,17 +132,17 @@ def publish_to_historian(self, to_publish_list): try: # resp = requests.post(login_uri, auth=auth) resp = requests.put(dataset_uri, verify=False, headers=headers, data=payload) - if resp.status_code == requests.codes.ok: + if resp.status_code == requests.codes.ok: self.report_handled(try_publish) except ConnectionError: _log.error('Unable to connect to openeis at {}'.format(uri)) return ''' Transform the to_publish_list into a dictionary like the following - + dataset_extension = { "dataset_id": dataset_id, - "point_map": + "point_map": { "New building/WholeBuildingPower": [["2/5/2014 10:00",48.78], ["2/5/2014 10:05",50.12], ["2/5/2014 10:10",48.54]], "New building/OutdoorAirTemperature": [["2/5/2014 10:00",48.78], ["2/5/2014 10:05",10.12], ["2/5/2014 10:10",48.54]] diff --git a/deprecated/OpenEISHistorian/setup.py b/deprecated/OpenEISHistorian/setup.py new file mode 100644 index 0000000000..ed132974bf --- /dev/null +++ b/deprecated/OpenEISHistorian/setup.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- {{{ +# ===----------------------------------------------------------------------=== +# +# Component of Eclipse VOLTTRON +# +# ===----------------------------------------------------------------------=== +# +# Copyright 2023 Battelle Memorial Institute +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# ===----------------------------------------------------------------------=== +# }}} + +from os import path +from setuptools import setup, find_packages + +MAIN_MODULE = 'historian' + +# Find the agent package that contains the main module +packages = find_packages('.') +agent_package = '' +for package in find_packages(): + # Because there could be other packages such as tests + if path.isfile(package + '/' + MAIN_MODULE + '.py') is True: + agent_package = package +if not agent_package: + raise RuntimeError('None of the packages under {dir} contain the file ' + '{main_module}'.format(main_module=MAIN_MODULE + '.py', + dir=path.abspath(''))) + +# Find the version number from the main module +agent_module = agent_package + '.' + MAIN_MODULE +_temp = __import__(agent_module, globals(), locals(), ['__version__'], 0) +__version__ = _temp.__version__ + +# Setup +setup( + name=agent_package + 'historian', + version=__version__, + install_requires=['volttron', 'requests'], + packages=packages, + entry_points={ + 'setuptools.installation': [ + 'eggsecutable = ' + agent_module + ':main', + ] + } +) diff --git a/docs/.readthedocs.yaml b/docs/.readthedocs.yaml new file mode 100644 index 0000000000..8915bd8d0d --- /dev/null +++ b/docs/.readthedocs.yaml @@ -0,0 +1,29 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.10" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# If using Sphinx, optionally build your docs in additional formats such as PDF +# formats: +# - pdf + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt diff --git a/docs/requirements.txt b/docs/requirements.txt index 84232c5b90..b30f7a5b2a 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ -sphinx_rtd_theme -sphinx-autobuild -sphinx==3.3.0 -m2r2 +sphinx==5.1.1 +sphinx-rtd-theme==1.0.0 +m2r2==0.3.2 +sphinxcontrib-mermaid bacpypes enum34 funcsigs @@ -27,3 +27,5 @@ zmq ply psutil ws4py +Jinja2==3.1.2 +PyYaml diff --git a/docs/source/agent-framework/agents-overview.rst b/docs/source/agent-framework/agents-overview.rst index 6208b90f53..dcde688852 100644 --- a/docs/source/agent-framework/agents-overview.rst +++ b/docs/source/agent-framework/agents-overview.rst @@ -9,6 +9,14 @@ on behalf of the user. Agents may perform a huge variety of tasks, but common u control of ICS and IOT devices, and various platform management tasks. Agents implemented using the VOLTTRON agent framework inherit a number of capabilities, including message bus connectivity and agent lifecycle. +.. attention:: + Agents are split into the directories of **core**, **contributed**, **operations** and **unsupported**. :: + + services/contrib/ + services/core/ + services/ops/ + services/unsupported/ + Agents deployed on VOLTTRON can perform one or more roles which can be broadly classified into the following groups: - Platform Agents: Agents which are part of the platform and provide a service to other agents. Examples are the diff --git a/docs/source/agent-framework/core-service-agents/ieee-2030_5-agent/ieee-2030_5-agent.rst b/docs/source/agent-framework/core-service-agents/ieee-2030_5-agent/ieee-2030_5-agent.rst index 636ee06db8..e45704ffb3 100644 --- a/docs/source/agent-framework/core-service-agents/ieee-2030_5-agent/ieee-2030_5-agent.rst +++ b/docs/source/agent-framework/core-service-agents/ieee-2030_5-agent/ieee-2030_5-agent.rst @@ -1,118 +1,36 @@ .. _IEEE-2030_5-Agent: -===================== -IEEE 2030.5 DER Agent -===================== +=========================== +IEEE 2030.5 EndDevice Agent +=========================== -The IEEE 2030.5 Agent (IEEE2030_5 in the VOLTTRON repository) implements a IEEE 2030.5 server that receives HTTP -`POST`/`PUT` requests from IEEE 2030.5 devices. The requests are routed to the IEEE 2030.5 Agent over the VOLTTRON -message bus by VOLTTRON's Master Web Service. The IEEE 2030.5 Agent returns an appropriate HTTP response. In some -cases (e.g., DERControl requests), this response includes a data payload. +The IEEE 2030.5 Agent (IEEE_2030_5 in the VOLTTRON repository) acts as an IEEE 2030.5 EndDevice (client). This +agent establishes a secure connection to a TLS-enabled 2030.5 server and discovers its capabilities. It verifies +the server's identity based on the Registration function set and uses the FunctionSetAssignments function set to +determine the appropriate DERProgram to run. The agent regularly checks for changes in default controls and +active DERControls and responds accordingly. It also listens to one or more subscriptions to the VOLTTRON message +bus for information (points) to POST/PUT to the 2030.5 server. -The IEEE 2030.5 Agent maps IEEE 2030.5 resource data to a VOLTTRON IEEE 2030.5 data model based on SunSpec, using block -numbers and point names as defined in the SunSpec Information Model, which in turn is harmonized with 61850. The data -model is given in detail below. +You can access the agent code, README, and demo from `IEEE_2030_5 Agent `_. -Each device's data is stored by the IEEE 2030.5 Agent in an `EndDevice` memory structure. This structure is not -persisted to a database. Each `EndDevice` retains only the most recently received value for each field. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Common Smart Inverter Profile (CSIP) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The IEEE2030_5 Agent exposes RPC calls for getting and setting EndDevice data. +This agent is not a fully compliant CSIP client, meaning it does not support all of the function sets +within the CSIP Profile of 2030.5. It provides the following function sets: +- End Device +- Time +- Distributed Energy Resources +- Metering +- Metering Mirror -VOLTTRON IEEE 2030.5 Device Driver ----------------------------------- +As time goes on it is likely that this list will be extended through user supported additions and project needs. -The :ref:`IEEE 2030.5 device driver ` is a new addition to VOLTTRON Platform Driver Agent's -family of standard device drivers. It exposes ``get_point``/``set_point calls`` for IEEE 2030.5 EndDevice fields. +################ +2030.5 Reference +################ -The IEEE 2030.5 device driver periodically issues IEEE2030_5 Agent RPC calls to refresh its cached representation of -EndDevice data. It issues RPC calls to IEEE2030_5Agent as needed when responding to ``get_point``, ``set_point`` and -``scrape_all`` calls. - - -Field Definitions -^^^^^^^^^^^^^^^^^ - -These field IDs correspond to the ones in the IEEE 2030.5 device driver's configuration file, ``ieee2030_5.csv``. -They have been used in that file's "Volttron Point Name" column and also in its "Point Name" column. - -================= ============================= ==================================================== ======= ====== -Field ID IEEE 2030.5 Resource/Property Description Units Type -================= ============================= ==================================================== ======= ====== -b1_Md device_information Model (32 char lim). string - mfModel -b1_Opt device_information Long-form device identifier (32 char lim). string - lfdi -b1_SN abstract_device Short-form device identifier (32 char lim). string - sfdi -b1_Vr device_information Version (16 char lim). string - mfHwVer -b113_A mirror_meter_reading AC current. A float - PhaseCurrentAvg -b113_DCA mirror_meter_reading DC current. A float - InstantPackCurrent -b113_DCV mirror_meter_reading DC voltage. V float - LineVoltageAvg -b113_DCW mirror_meter_reading DC power. W float - PhasePowerAvg -b113_PF mirror_meter_reading AC power factor. % float - PhasePFA -b113_WH mirror_meter_reading AC energy. Wh float - EnergyIMP -b120_AhrRtg der_capability Usable capacity of the battery. Ah float - rtgAh Maximum charge minus minimum charge. -b120_ARtg der_capability Maximum RMS AC current level capability of the A float - rtgA inverter. -b120_MaxChaRte der_capability Maximum rate of energy transfer into the device. W float - rtgMaxChargeRate -b120_MaxDisChaRte der_capability Maximum rate of energy transfer out of the device. W float - rtgMaxDischargeRate -b120_WHRtg der_capability Nominal energy rating of the storage device. Wh float - rtgWh -b120_WRtg der_capability Continuous power output capability of the inverter. W float - rtgW -b121_WMax der_settings Maximum power output. Default to WRtg. W float - setMaxChargeRate -b122_ActWh mirror_meter_reading AC lifetime active (real) energy output. Wh float - EnergyEXP -b122_StorConn der_status CONNECTED=0, AVAILABLE=1, OPERATING=2, TEST=3. enum - storConnectStatus -b124_WChaMax der_control Setpoint for maximum charge. This is the only W float - opModFixedFlow field that is writable with a set_point call. -b403_Tmp mirror_meter_reading Pack temperature. C float - InstantPackTemp -b404_DCW PEVInfo Power flow in or out of the inverter. W float - chargingPowerNow -b404_DCWh der_availability Output energy (absolute SOC). Wh float - availabilityDuration Calculated as (availabilityDuration / 3600) * WMax. -b802_LocRemCtl der_status Control Mode: REMOTE=0, LOCAL=1. enum - localControlModeStatus -b802_SoC der_status State of Charge %. % WHRtg float - stateOfChargeStatus -b802_State der_status DISCONNECTED=1, INITIALIZING=2, CONNECTED=3, enum - inverterStatus STANDBY=4, SOC PROTECTION=5, FAULT=99. -================= ============================= ==================================================== ======= ====== - - -Revising and Expanding the Field Definitions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The IEEE 2030.5-to-SunSpec field mappings in this implementation are a relatively thin subset of all possible -field definitions. Developers are encouraged to expand the definitions. - -The procedure for expanding the field mappings requires you to make changes in two places: - -1. Update the driver's point definitions in ``services/core/PlatformDriverAgent/platform_driver/ieee2030_5.csv`` -2. Update the IEEE 2030.5-to-SunSpec field mappings in ``services/core/IEEE2030_5Agent/ieee2030_5/end_device.py`` and - ``__init__.py`` - -When updating VOLTTRON's IEEE 2030.5 data model, please use field IDs that conform to the SunSpec -block-number-and-field-name model outlined in the SunSpec Information Model Reference (see the link below). - -View the :ref:`IEEE 2030.5 agent specification document ` to learn more about IEEE 2030.5 and -the IEEE 2030.5 agent and driver. - - -.. toctree:: - - ieee-2030_5-specification +`IEEE 2030.5 Standards `_ +`IEEE_2030_5 Agent `_ diff --git a/docs/source/agent-framework/core-service-agents/ieee-2030_5-agent/ieee-2030_5-specification.rst b/docs/source/agent-framework/core-service-agents/ieee-2030_5-agent/ieee-2030_5-specification.rst deleted file mode 100644 index 361f9e1150..0000000000 --- a/docs/source/agent-framework/core-service-agents/ieee-2030_5-agent/ieee-2030_5-specification.rst +++ /dev/null @@ -1,207 +0,0 @@ -.. _IEEE-2030_5-Specification: - -======================= -IEEE 2030.5 DER Support -======================= - -Version 1.0 - -Smart Energy Profile 2.0 (SEP 2.0, IEEE 2030.5) specifies a REST architecture built around the core HTTP verbs: GET, -HEAD, PUT, POST and DELETE. A specification for the IEEE 2030.5 protocol can be found -`here `_. - -IEEE 2030.5 EndDevices (clients) POST XML resources representing their state, and GET XML resources containing command -and control information from the server. The server never reaches out to the client unless a "subscription" is -registered and supported for a particular resource type. This implementation does not use IEEE 2030.5 registered -subscriptions. - -The IEEE 2030.5 specification requires HTTP headers, and it explicitly requires RESTful response codes, for example: - - - 201 - "Created" - - 204 - "No Content" - - 301 - "Moved Permanently" - - etc. - -IEEE 2030.5 message encoding may be either XML or EXI. Only XML is supported in this implementation. - -IEEE 2030.5 requires HTTPS/TLS version 1.2 along with support for the cipher suite TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8. -Production installation requires a certificate issued by a IEEE 2030.5 CA. The encryption requirement can be met by -using a web server such as Apache to proxy the HTTPs traffic. - -IEEE 2030.5 discovery, if supported, must be implemented by an xmDNS server. Avahi can be modified to perform this -function. - - -Function Sets -============= - -IEEE 2030.5 groups XML resources into "Function Sets." Some of these function sets provide a core set of functionality -used across higher-level function sets. This implementation implements resources from the following function sets: - - - Time - - Device Information - - Device Capabilities - - End Device - - Function Set Assignments - - Power Status - - Distributed Energy Resources - - -Distributed Energy Resources (DERs) ------------------------------------ - -Distributed energy resources (DERs) are devices that generate energy, e.g., solar inverters, or store energy, e.g., -battery storage systems, electric vehicle supply equipment (EVSEs). These devices are managed by a IEEE 2030.5 DER -server using DERPrograms which are described by the IEEE 2030.5 specification as follows: - - Servers host one or more DERPrograms, which in turn expose DERControl events to DER clients. - DERControl instances contain attributes that allow DER clients to respond to events - that are targeted to their device type. A DERControl instance also includes scheduling - attributes that allow DER clients to store and process future events. These attributes - include start time and duration, as well an indication of the need for randomization of - the start and / or duration of the event. The IEEE 2030.5 DER client model is based on the - SunSpec Alliance Inverter Control Model [SunSpec] which is derived from - IEC 61850-90-7 [61850] and [EPRI]. - -EndDevices post multiple IEEE 2030.5 resources describing their status. The following is an -example of a Power Status resource that might be posted by an EVSE (vehicle charging station): - -.. code-block:: xml - - - 4 - 1487812095 - 1 - 9300 - - - 3 - -5 - - - 3 - 22 - - - 3 - 7 - - 11280 - 10000 - 9223372036854775807 - 1487812095 - - - - -Design Details --------------- - -.. image:: files/volttron_ieee2030_5.jpg - -VOLTTRON's IEEE 2030.5 implementation includes a IEEE 2030.5 Agent and a IEEE 2030.5 device driver, as described below. - - -VOLTTRON IEEE 2030.5 Device Driver -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The IEEE 2030.5 device driver is a new addition to VOLTTRON Platform Driver Agent's family of standard device drivers. It -exposes `get_point`/`set_point` calls for IEEE 2030.5 EndDevice fields. - -The IEEE 2030.5 device driver periodically issues the IEEE 2030.5 Agent RPC calls to refresh its cached representation -of EndDevice data. It issues RPC calls to the IEEE 2030.5 Agent as needed when responding to `get_point`, `set_point` -and `scrape_all` calls. - -Field Definitions -^^^^^^^^^^^^^^^^^ - -These field IDs correspond to the ones in the IEEE 2030.5 device driver's configuration file, `ieee2030_5.csv`. -They have been used in that file's `Volttron Point Name` column and also in its `Point Name` column. - -================= ============================= ==================================================== ======= ====== -Field ID IEEE 2030.5 Resource/Property Description Units Type -================= ============================= ==================================================== ======= ====== -b1_Md device_information Model (32 char lim). string - mfModel -b1_Opt device_information Long-form device identifier (32 char lim). string - lfdi -b1_SN abstract_device Short-form device identifier (32 char lim). string - sfdi -b1_Vr device_information Version (16 char lim). string - mfHwVer -b113_A mirror_meter_reading AC current. A float - PhaseCurrentAvg -b113_DCA mirror_meter_reading DC current. A float - InstantPackCurrent -b113_DCV mirror_meter_reading DC voltage. V float - LineVoltageAvg -b113_DCW mirror_meter_reading DC power. W float - PhasePowerAvg -b113_PF mirror_meter_reading AC power factor. % float - PhasePFA -b113_WH mirror_meter_reading AC energy. Wh float - EnergyIMP -b120_AhrRtg der_capability Usable capacity of the battery. Ah float - rtgAh Maximum charge minus minimum charge. -b120_ARtg der_capability Maximum RMS AC current level capability of the A float - rtgA inverter. -b120_MaxChaRte der_capability Maximum rate of energy transfer into the device. W float - rtgMaxChargeRate -b120_MaxDisChaRte der_capability Maximum rate of energy transfer out of the device. W float - rtgMaxDischargeRate -b120_WHRtg der_capability Nominal energy rating of the storage device. Wh float - rtgWh -b120_WRtg der_capability Continuous power output capability of the inverter. W float - rtgW -b121_WMax der_settings Maximum power output. Default to WRtg. W float - setMaxChargeRate -b122_ActWh mirror_meter_reading AC lifetime active (real) energy output. Wh float - EnergyEXP -b122_StorConn der_status CONNECTED=0, AVAILABLE=1, OPERATING=2, TEST=3. enum - storConnectStatus -b124_WChaMax der_control Setpoint for maximum charge. This is the only W float - opModFixedFlow field that is writable with a set_point call. -b403_Tmp mirror_meter_reading Pack temperature. C float - InstantPackTemp -b404_DCW PEVInfo Power flow in or out of the inverter. W float - chargingPowerNow -b404_DCWh der_availability Output energy (absolute SOC). Wh float - availabilityDuration Calculated as (availabilityDuration / 3600) * WMax. -b802_LocRemCtl der_status Control Mode: REMOTE=0, LOCAL=1. enum - localControlModeStatus -b802_SoC der_status State of Charge %. % WHRtg float - stateOfChargeStatus -b802_State der_status DISCONNECTED=1, INITIALIZING=2, CONNECTED=3, enum - inverterStatus STANDBY=4, SOC PROTECTION=5, FAULT=99. -================= ============================= ==================================================== ======= ====== - - -Revising and Expanding the Field Definitions --------------------------------------------- - -The IEEE 2030.5-to-SunSpec field mappings in this implementation are a relatively thin subset of all possible -field definitions. Developers are encouraged to expand the definitions. - -The procedure for expanding the field mappings requires you to make changes in two places: - -1. Update the driver's point definitions in `services/core/PlatformDriverAgent/platform_driver/ieee2030_5.csv` -2. Update the IEEE 2030.5-to-SunSpec field mappings in `services/core/IEEE2030_5Agent/ieee2030_5/end_device.py` and - `__init__.py` - -When updating VOLTTRON's IEEE 2030.5 data model, please use field IDs that conform to the SunSpec -block-number-and-field-name model outlined in the SunSpec Information Model Reference (see the link below). - - -For Further Information -======================= - -SunSpec References: - - - Information model specification: http://sunspec.org/wp-content/uploads/2015/06/SunSpec-Information-Models-12041.pdf - - Information model reference spreadsheet: http://sunspec.org/wp-content/uploads/2015/06/SunSpec-Information-Model-Reference.xlsx - - Inverter models: http://sunspec.org/wp-content/uploads/2015/06/SunSpec-Inverter-Models-12020.pdf - - Energy storage models: http://sunspec.org/wp-content/uploads/2015/06/SunSpec-Energy-Storage-Models-12032.pdf - -Questions? Please contact: - - - Rob Calvert (rob@kisensum.com) or James Sheridan (james@kisensum.com) diff --git a/docs/source/agent-framework/core-service-agents/index.rst b/docs/source/agent-framework/core-service-agents/index.rst index 575422b3c0..cc0157a333 100644 --- a/docs/source/agent-framework/core-service-agents/index.rst +++ b/docs/source/agent-framework/core-service-agents/index.rst @@ -4,8 +4,13 @@ Core Services ============= -Agents in the `services/core` directory support the most common use cases of the platform. For details on each, please -refer to the corresponding documents. +Agents which support the most common use cases of the platform. For details on each, please refer to the corresponding +documents. + +.. attention:: + All core agents are located within the **core** directory. :: + + services/core/ .. toctree:: :maxdepth: 1 diff --git a/docs/source/agent-framework/core-service-agents/openadr-ven/ven-agent.rst b/docs/source/agent-framework/core-service-agents/openadr-ven/ven-agent.rst index 5f14dcba5c..dddb567668 100644 --- a/docs/source/agent-framework/core-service-agents/openadr-ven/ven-agent.rst +++ b/docs/source/agent-framework/core-service-agents/openadr-ven/ven-agent.rst @@ -8,14 +8,14 @@ OpenADR (Automated Demand Response) is a standard for alerting and responding to consumption in response to fluctuations in grid demand. OpenADR communications are conducted between Virtual Top Nodes (VTNs) and Virtual End Nodes (VENs). In this -implementation a VOLTTRON agent, the VEN agent, acts as a VEN, communicating with its VTN by means of EIEvent and +implementation, a VOLTTRON agent, the VEN agent, acts as a VEN, communicating with its VTN by means of EIEvent and EIReport services in conformance with a subset of the OpenADR 2.0b specification. This document's -`VOLTTRON Interface `_ section defines how the VEN agent relays information to, +VEN Agent VOLTTRON Interface section defines how the VEN agent relays information to, and receives data from, other VOLTTRON agents. -The OpenADR 2.0b specification (http://www.openadr.org/specification) is available from the OpenADR Alliance. This +The `OpenADR 2.0b specification `_ is available from the OpenADR Alliance. This implementation also generally follows the DR program characteristics of the Capacity Program described in Section 9.2 -of the OpenADR Program Guide (http://www.openadr.org/assets/openadr_drprogramguide_v1.0.pdf). +of the `OpenADR Program Guide `_. DR Capacity Bidding and Events @@ -25,10 +25,10 @@ The OpenADR Capacity Bidding program relies on a pre-committed agreement about t agreement is reached in a bidding process transacted outside of the OpenADR interaction, typically with a long-term scope, perhaps a month or longer. The VTN can “call an event,” indicating that a load-shed event should occur in conformance with this agreement. The VTN indicates the level of load shedding desired, when the event should occur, -and for how long. The VEN responds with an `optIn` acknowledgment. (It can also `optOut`, but since it has been -pre-committed, an `optOut` may incur penalties.) - +and for how long. The VEN responds with an `optIn` acknowledgment. It can also `optOut`, but since it has been +pre-committed, an `optOut` may incur penalties. +.. _my-reference-label: Reporting --------- @@ -43,14 +43,11 @@ Events: - The VEN agent maintains a persistent record of DR events. - Event updates (including creation) trigger publication of event JSON on the VOLTTRON message bus. -- Other VOLTTRON agents can also call a get_events() RPC to retrieve the current status of - particular events, or of all active events. Reporting: - The VEN agent configuration defines telemetry values (data points) that can be reported to the VTN. -- The VEN agent maintains a persistent record of telemetry values over time. -- Other VOLTTRON agents are expected to call report_telemetry() to supply the VEN agent +- Other VOLTTRON agents are expected to call add_report_capability() to supply the VEN agent with a regular stream of telemetry values for reporting. - Other VOLTTRON agents can receive notification of changes in telemetry reporting requirements by subscribing to publication of telemetry parameters. @@ -65,76 +62,11 @@ PubSub: event update .. code-block:: python - def publish_event(self, an_event): - """ - Publish an event. - - When an event is created/updated, it is published to the VOLTTRON bus - with a topic that includes 'openadr/event_update'. - - Event JSON structure: - { - "event_id" : String, - "creation_time" : DateTime, - "start_time" : DateTime, - "end_time" : DateTime or None, - "signals" : String, # Values: json string describing one or more signals. - "status" : String, # Values: unresponded, far, near, active, - # completed, canceled. - "opt_type" : String # Values: optIn, optOut, none. - } - - If an event status is 'unresponded', the VEN agent is awaiting a decision on - whether to optIn or optOut. The downstream agent that subscribes to this PubSub - message should communicate that choice to the VEN agent by calling respond_to_event() - (see below). The VEN agent then relays the choice to the VTN. - - @param an_event: an EiEvent. + def publish_event(self, event: Event) -> None: """ + Publish an event to the Volttron message bus. When an event is created/updated, it is published to the VOLTTRON bus with a topic that includes 'openadr/event_update'. -PubSub: telemetry parameters update - -.. code-block:: python - - def publish_telemetry_parameters_for_report(self, report): - """ - Publish telemetry parameters. - - When the VEN agent telemetry reporting parameters have been updated (by the VTN), - they are published with a topic that includes 'openadr/telemetry_parameters'. - If a particular report has been updated, the reported parameters are for that report. - - Telemetry parameters JSON example: - { - "telemetry": { - "baseline_power_kw": { - "r_id": "baseline_power", - "frequency": "30", - "report_type": "baseline", - "reading_type": "Mean", - "method_name": "get_baseline_power" - } - "current_power_kw": { - "r_id": "actual_power", - "frequency": "30", - "report_type": "reading", - "reading_type": "Mean", - "method_name": "get_current_power" - } - "manual_override": "False", - "report_status": "active", - "online": "False", - } - } - - The above example indicates that, for reporting purposes, telemetry values - for baseline_power and actual_power should be updated -- via report_telemetry() -- at - least once every 30 seconds. - - Telemetry value definitions such as baseline_power and actual_power come from the - agent configuration. - - @param report: (EiReport) The report whose parameters should be published. + :param event: The Event received from the VTN """ RPC calls: @@ -142,164 +74,62 @@ RPC calls: .. code-block:: python @RPC.export - def respond_to_event(self, event_id, opt_in_choice=None): - """ - Respond to an event, opting in or opting out. - - If an event's status=unresponded, it is awaiting this call. - When this RPC is received, the VENAgent sends an eventResponse to - the VTN, indicating whether optIn or optOut has been chosen. - If an event remains unresponded for a set period of time, - it times out and automatically optsIn to the event. - - Since this call causes a change in the event's status, it triggers - a PubSub call for the event update, as described above. - - @param event_id: (String) ID of an event. - @param opt_in_choice: (String) 'OptIn' to opt into the event, anything else is treated as 'OptOut'. - """ - -.. code-block:: python - - @RPC.export - def get_events(self, event_id=None, in_progress_only=True, started_after=None, end_time_before=None): - """ - Return a list of events as a JSON string. - - Sample request: - self.get_events(started_after=utils.get_aware_utc_now() - timedelta(hours=1), - end_time_before=utils.get_aware_utc_now()) - - Return a list of events. - - By default, return only event requests with status=active or status=unresponded. - - If an event's status=active, a DR event is currently in progress. - - @param event_id: (String) Default None. - @param in_progress_only: (Boolean) Default True. - @param started_after: (DateTime) Default None. - @param end_time_before: (DateTime) Default None. - @return: (JSON) A list of events -- see 'PubSub: event update'. - """ - -.. code-block:: python - - @RPC.export - def get_telemetry_parameters(self): + def add_report_capability( + self, + callback: Callable, + report_name: REPORT_NAME, + resource_id: str, + measurement: MEASUREMENTS, + ) -> tuple: """ - Return the VEN agent's current set of telemetry parameters. + Add a new reporting capability to the client. - @return: (JSON) Current telemetry parameters -- see 'PubSub: telemetry parameters update'. - """ - -.. code-block:: python - - @RPC.export - def set_telemetry_status(self, online, manual_override): - """ - Update the VEN agent's reporting status. - - Set these properties to either 'TRUE' or 'FALSE'. - - @param online: (Boolean) Whether the VEN agent's resource is online. - @param manual_override: (Boolean) Whether resource control has been overridden. - """ - -.. code-block:: python - - @RPC.export - def report_telemetry(self, telemetry): - """ - Receive an update of the VENAgent's report metrics, and store them in the agent's database. - - Examples of telemetry are: - { - 'baseline_power_kw': '15.2', - 'current_power_kw': '371.1', - 'start_time': '2017-11-21T23:41:46.051405', - 'end_time': '2017-11-21T23:42:45.951405' - } + This method is remotely accessible by other agents through Volttron's feature Remote Procedure Call (RPC); + for reference on RPC, see https://volttron.readthedocs.io/en/develop/platform-features/message-bus/vip/vip-json-rpc.html?highlight=remote%20procedure%20call - @param telemetry_values: (JSON) Current value of each report metric, with reporting-interval start/end. + :param callback: A callback or coroutine that will fetch the value for a specific report. This callback will be passed the report_id and the r_id of the requested value. + :param report_name: An OpenADR name for this report + :param resource_id: A specific name for this resource within this report. + :param measurement: The quantity that is being measured + :return: Returns a tuple consisting of a report_specifier_id (str) and an r_id (str) an identifier for OpenADR messages """ PubSub: Event Update -------------------- -When an event is created/updated, the event is published with a topic that includes `openadr/event/{ven_id}`. +When an event is created/updated, the event is published with a topic that includes `openadr/event/{ven_name}`. Event JSON structure: :: - { - "event_id" : String, - "creation_time" : DateTime - UTC, - "start_time" : DateTime - UTC, - "end_time" : DateTime - UTC, - "priority" : Integer, # Values: 0, 1, 2, 3. Usually expected to be 1. - "signals" : String, # Values: json string describing one or more signals. - "status" : String, # Values: unresponded, far, near, active, completed, canceled. - "opt_type" : String # Values: optIn, optOut, none. - } - -If an event status is 'unresponded', the VEN is awaiting a decision on whether to `optIn` or `optOut`. The downstream -agent that subscribes to this `PubSub` message should communicate that choice to the VEN by calling respond_to_event() -(see below). The VEN then relays the choice to the VTN. - - -PubSub: Telemetry Parameters Update ------------------------------------ - -When the VEN telemetry reporting parameters have been updated (by the VTN), they are published with a topic that -includes `openadr/status/{ven_id}`. - -These parameters include state information about the current report. - -Telemetry parameters structure: - -:: - - { - 'telemetry': '{ - "baseline_power_kw": { - "r_id" : "baseline_power", # ID of the reporting metric - "report_type" : "baseline", # Type of reporting metric, e.g. baseline or reading - "reading_type" : "Direct Read", # (per OpenADR telemetry_usage report requirements) - "units" : "powerReal", # (per OpenADR telemetry_usage reoprt requirements) - "method_name" : "get_baseline_power", # Name of the VEN agent method that gets the metric - "min_frequency" : (Integer), # Data capture frequency in seconds (minimum) - "max_frequency" : (Integer) # Data capture frequency in seconds (maximum) - }, - "current_power_kw": { - "r_id" : "actual_power", # ID of the reporting metric - "report_type" : "reading", # Type of reporting metric, e.g. baseline or reading - "reading_type" : "Direct Read", # (per OpenADR telemetry_usage report requirements) - "units" : "powerReal", # (per OpenADR telemetry_usage report requirements) - "method_name" : "get_current_power", # Name of the VEN agent method that gets the metric - "min_frequency" : (Integer), # Data capture frequency in seconds (minimum) - "max_frequency" : (Integer) # Data capture frequency in seconds (maximum) - } - }' - 'report parameters': '{ - "status" : (String), # active, inactive, completed, or cancelled - "report_specifier_id" : "telemetry", # ID of the report definition - "report_request_id" : (String), # ID of the report request; supplied by the VTN - "request_id" : (String), # Request ID of the most recent VTN report modification - "interval_secs" : (Integer), # How often a report update is sent to the VTN - "granularity_secs" : (Integer), # How often a report update is sent to the VTN - "start_time" : (DateTime - UTC), # When the report started - "end_time" : (DateTime - UTC), # When the report is scheduled to end - "last_report" : (DateTime - UTC), # When a report update was last sent - "created_on" : (DateTime - UTC) # When this set of information was recorded in the VEN db - }', - 'manual_override' : (Boolean) # VEN manual override status, as supplied by Control Agent - 'online' : (Boolean) # VEN online status, as supplied by Control Agent - } - -Telemetry value definitions such as `baseline_power_kw` and `current_power_kw` come from the VEN agent config. + {'active_period': {'dtstart': DateTime - UTC, + 'duration': TimeDelta - seconds, + 'notification': TimeDelta - seconds, + 'ramp_up': TimeDelta - seconds, + 'recovery': TimeDelta - seconds, + 'tolerance': {'tolerate': {'startafter': TimeDelta - seconds}}}, + 'event_descriptor': {'created_date_time': DateTime - UTC, + 'event_id': String, + 'event_status': String, + 'market_context': String, + 'modification_date_time': DateTime - UTC, + 'modification_number': Integer, + 'modification_reason': Any, + 'priority': Integer, + 'test_event': Boolean, + 'vtn_comment': Any}, + 'event_signals': [{'current_value': Double, + 'intervals': [{'duration': TimeDelta - seconds, + 'signal_payload': Double, + 'uid': Integer}], + 'signal_id': String, + 'signal_name': String, + 'signal_type': String}], + 'response_required': String, + 'targets': [{'ven_id': String}], + 'targets_by_type': {'ven_id': [String]}} .. _OpenADR-VEN-Agent-Config: @@ -308,7 +138,7 @@ OpenADR VEN Agent: Installation and Configuration ================================================= The VEN agent can be configured, built and launched using the VOLTTRON agent installation process described in -http://volttron.readthedocs.io/en/develop/devguides/agent_development/Agent-Development.html#agent-development. +https://volttron.readthedocs.io/en/releases-8.2/developing-volttron/developing-agents/agent-development.html#packaging-and-installation. The VEN agent depends on some third-party libraries that are not in the standard VOLTTRON installation. They should be installed in the VOLTTRON virtual environment prior to building the agent: @@ -321,123 +151,34 @@ installed in the VOLTTRON virtual environment prior to building the agent: where :term:`$VOLTTRON_ROOT ` is the base directory of the cloned VOLTTRON code repository. The VEN agent is designed to work in tandem with a “control agent,” another VOLTTRON agent that uses VOLTTRON RPC calls -to manage events and supply report data. A sample control agent has been provided in the `test/ControlAgentSim` -subdirectory under OpenADRVenAgent. - -The VEN agent maintains a persistent store of event and report data in ``$VOLTTRON_HOME/data/openadr.sqlite``. Some -care should be taken in managing the disk consumption of this data store. If no events or reports are active, it is -safe to take down the VEN agent and delete the file; the persistent store will be reinitialized automatically on agent -startup. +to manage events and supply report data. Configuration Parameters ------------------------ The VEN agent’s configuration file contains JSON that includes several parameters for configuring VTN server -communications and other behavior. A sample configuration file, `config`, has been provided in the agent +communications and other behavior. A sample configuration file, `config_example1.json`, has been provided in the agent directory. -The VEN agent supports the following configuration parameters: +The VEN agent supports the following configuration parameters. Note: Some configurations will be noted as required; otherwise, they are optional: ========================= ======================== ==================================================== Parameter Example Description ========================= ======================== ==================================================== -db_path “$VOLTTRON_HOME/data/ Pathname of the agent's sqlite database. Shell - openadr.sqlite” variables will be expanded if they are present - in the pathname. -ven_id “0” The OpenADR ID of this virtual end node. Identifies - this VEN to the VTN. If automated VEN registration - is used, the ID is assigned by the VTN at that - time. If the VEN is registered manually with the - VTN (i.e., via configuration file settings), then - a common VEN ID should be entered in this config - file and in the VTN's site definition. -ven_name "ven01" Name of this virtual end node. This name is used - during automated registration only, identiying +ven_name "ven01" (Required) Name of this virtual end node. This name is used + during automated registration only, identifying the VEN before its VEN ID is known. -vtn_id “vtn01” OpenADR ID of the VTN with which this VEN - communicates. -vtn_address “http://openadr-vtn. URL and port number of the VTN. +vtn_url “http://openadr-vtn. (Required) URL and port number of the VTN. ki-evi.com:8000” -send_registration “False” (“True” or ”False”) If “True”, the VEN sends - a one-time automated registration request to - the VTN to obtain the VEN ID. If automated - registration will be used, the VEN should be run - in this mode initially, then shut down and run - with this parameter set to “False” thereafter. -security_level “standard” If 'high', the VTN and VEN use a third-party - signing authority to sign and authenticate each - request. The default setting is “standard”: the - XML payloads do not contain Signature elements. -poll_interval_secs 30 (integer) How often the VEN should send an OadrPoll - request to the VTN. The poll interval cannot be - more frequent than the VEN’s 5-second process - loop frequency. -log_xml “False” (“True” or “False”) Whether to write each - inbound/outbound request’s XML data to the - agent's log. -opt_in_timeout_secs 1800 (integer) How long to wait before making a - default optIn/optOut decision. -opt_in_default_decision “optOut” (“True” or “False”) Which optIn/optOut choice - to make by default. -request_events_on_startup "False" ("True" or "False") Whether to ask the VTN for a - list of current events during VEN startup. -report_parameters (see below) A dictionary of definitions of reporting/telemetry - parameters. +debug true Whether or not to print debugging messages +cert "/path/to/my/cert" The path to a PEM-formatted Certificate file to use for signing messages. +key "/path/to/my/priv-key" The path to a PEM-formatted Private Key file to use for signing messages. +passphrase "somepassword12345" The passphrase for the Private Key +vtn_fingerprint "fdfdfdfdfdgfsge" The fingerprint for the VTN’s certificate to verify incoming messages +show_fingerprint false Whether to print your own fingerprint on startup. Defaults to True. +ca_file "/path/to/my/ca-file" The path to the PEM-formatted CA file for validating the VTN server’s certificate. +ven_id "someid" The ID for this VEN. If you leave this blank, a VEN_ID will be assigned by the VTN. +disable_signature true Whether or not to sign outgoing messages using a public-private key pair in PEM format. ========================= ======================== ==================================================== - -Reporting Configuration ------------------------ - -The VEN’s reporting configuration, specified as a dictionary in the agent configuration, defines each telemetry element -(metric) that the VEN can report to the VTN, if requested. By default, it defines reports named “telemetry” and -"telemetry_status", with a report configuration dictionary containing the following parameters: - -======================================================= =========================== ==================================================== -"telemetry" report: parameters Example Description -======================================================= =========================== ==================================================== -report_name "TELEMETRY_USAGE" Friendly name of the report. -report_name_metadata "METADATA_TELEMETRY_USAGE" Friendly name of the report’s metadata, when sent - by the VEN’s oadrRegisterReport request. -report_specifier_id "telemetry" Uniquely identifies the report’s data set. -report_interval_secs_default "300" How often to send a reporting update to the VTN. -telemetry_parameters (baseline_power_kw): r_id "baseline_power" (baseline_power) Unique ID of the metric. -telemetry_parameters (baseline_power_kw): report_type "baseline" (baseline_power) The type of metric being reported. -telemetry_parameters (baseline_power_kw): reading_type "Direct Read" (baseline_power) How the metric was calculated. -telemetry_parameters (baseline_power_kw): units "powerReal" (baseline_power) The reading's data type. -telemetry_parameters (baseline_power_kw): method_name "get_baseline_power" (baseline_power) The VEN method to use when - extracting the data for reporting. -telemetry_parameters (baseline_power_kw): min_frequency 30 (baseline_power) The metric’s minimum sampling - frequency. -telemetry_parameters (baseline_power_kw): max_frequency 60 (baseline_power) The metric’s maximum sampling - frequency. -telemetry_parameters (current_power_kw): r_id "actual_power" (current_power) Unique ID of the metric. -telemetry_parameters (current_power_kw): report_type "reading" (current_power) The type of metric being reported. -telemetry_parameters (current_power_kw): reading_type "Direct Read" (current_power) How the metric was calculated. -telemetry_parameters (current_power_kw): units "powerReal" (baseline_power) The reading's data type. -telemetry_parameters (current_power_kw): method_name "get_current_power" (current_power) The VEN method to use when - extracting the data for reporting. -telemetry_parameters (current_power_kw): min_frequency 30 (current_power) The metric’s minimum sampling - frequency. -telemetry_parameters (current_power_kw): max_frequency 60 (current_power) The metric’s maximum sampling - frequency. -======================================================= =========================== ==================================================== - -======================================================= =========================== ==================================================== -"telemetry_status" report: parameters Example Description -======================================================= =========================== ==================================================== -report_name "TELEMETRY_STATUS" Friendly name of the report. -report_name_metadata "METADATA_TELEMETRY_STATUS" Friendly name of the report’s metadata, when sent - by the VEN’s oadrRegisterReport request. -report_specifier_id "telemetry_status" Uniquely identifies the report’s data set. -report_interval_secs_default "300" How often to send a reporting update to the VTN. -telemetry_parameters (Status): r_id "Status" Unique ID of the metric. -telemetry_parameters (Status): report_type "x-resourceStatus" The type of metric being reported. -telemetry_parameters (Status): reading_type "x-notApplicable" How the metric was calculated. -telemetry_parameters (Status): units "" The reading's data type. -telemetry_parameters (Status): method_name "" The VEN method to use when extracting the data - for reporting. -telemetry_parameters (Status): min_frequency 60 The metric’s minimum sampling frequency. -telemetry_parameters (Status): max_frequency 120 The metric’s maximum sampling frequency. -======================================================= =========================== ==================================================== diff --git a/docs/source/agent-framework/core-service-agents/platform-driver/platform-driver-agent.rst b/docs/source/agent-framework/core-service-agents/platform-driver/platform-driver-agent.rst index 5566db3142..5a62519da7 100644 --- a/docs/source/agent-framework/core-service-agents/platform-driver/platform-driver-agent.rst +++ b/docs/source/agent-framework/core-service-agents/platform-driver/platform-driver-agent.rst @@ -63,7 +63,7 @@ The example platform driver configuration file above can be found in the VOLTTRO `services/core/PlatformDriverAgent/platform-driver.agent`. For information on configuring the Platform Driver with devices, including creating driver configs and using the config -store, please read ref`configuration ` the section in the Driver Framework docs. +store, please read the :ref:`configuration ` section in the Driver Framework docs. .. toctree:: diff --git a/docs/source/agent-framework/core-service-agents/volttron-central/vc-device-configuration-demo.rst b/docs/source/agent-framework/core-service-agents/volttron-central/vc-device-configuration-demo.rst index 7a6758c497..02456fd820 100644 --- a/docs/source/agent-framework/core-service-agents/volttron-central/vc-device-configuration-demo.rst +++ b/docs/source/agent-framework/core-service-agents/volttron-central/vc-device-configuration-demo.rst @@ -9,22 +9,22 @@ enables device detection and configuration for BACnet devices. The following se with performing scans to detect physical devices and get their points, and configuring them as virtual devices installed on VOLTTRON instances. -- `Launching Device Configuration `__ -- `Scanning for Devices `__ -- `Scanning for Points `__ -- `Registry Configuration File `__ -- `Additional Attributes `__ -- `Quick Edit Features `__ -- `Keyboard Commands `__ -- `Registry Preview `__ -- `Registry Configuration Options `__ -- `Reloading Device Points `__ -- `Device Configuration Form `__ -- `Configuring Sub-devices `__ -- `Reconfiguring Devices `__ -- `Exporting Registry Configuration Files `__ - - +- `Launching Device Configuration`_ +- `Scanning for Devices`_ +- `Scanning for Points`_ +- `Registry Configuration File`_ +- `Additional Attributes`_ +- `Quick Edit Features`_ +- `Keyboard Commands`_ +- `Registry Preview`_ +- `Registry Configuration Options`_ +- `Reloading Device Points`_ +- `Device Configuration Form`_ +- `Configuring Sub-devices`_ +- `Reconfiguring Devices`_ +- `Exporting Registry Configuration Files`_ + +.. _launching-device-configuration: Launching Device Configuration ------------------------------ @@ -45,7 +45,7 @@ device IDs will limit the scan to return only devices with IDs in that range. A specify the IP address of a device to detect as well as the ability to change the duration of the scan from the default of five seconds. - +.. _scanning-for-devices: Scanning for Devices -------------------- @@ -58,7 +58,7 @@ clicking the large cog button again. |Devices Found| - +.. _scanning-for-points: Scanning for Points ------------------- @@ -73,9 +73,9 @@ After the points have been retrieved once, the only way to scan the same device device configuration process from the start by clicking on the small cogs button next to the platform instance in the panel tree. - +.. _registry-configuration-file: Registry Configuration File -^^^^^^^^^^^^^^^^^^^^^^^^^^^ +---------------------------- The registry configuration determines which points on the physical device will be associated with the virtual device that uses that particular registry configuration. The registry configuration determines which points' data will be @@ -100,7 +100,7 @@ Type directly in a cell to change an attribute value for a point. |Edit Points| - +.. _additional-attributes: Additional Attributes --------------------- @@ -118,7 +118,7 @@ In the window that opens, point attributes can be changed by typing in the field Checking or unchecking the `Show in Table` box for an attribute will add or remove it as a column in the registry configuration editor. - +.. _quick-edit-features: Quick Edit Features ------------------- @@ -188,7 +188,7 @@ To quickly replace the matched term in the cell with focus, type a replacement t To replace all the matched terms in the column, click on the `Replace All` button. Click the `Clear Search` button to end the search. - +.. _keyboard-commands: Keyboard Commands ----------------- @@ -222,7 +222,7 @@ Click on the `Keyboard Shortcuts` button to show a popup list of the available k |Keyboard Shortcuts| - +.. _registry-preview: Registry Preview ---------------- @@ -244,7 +244,7 @@ Provide a name for the registry configuration file, and click the `Save` button |Registry Saved| - +.. _registry-configuration-options: Registry Configuration Options ------------------------------ @@ -269,7 +269,7 @@ use the file selector window to locate and load the file. |File Import Button| - +.. _reloading-device-points: Reloading Device Points ----------------------- @@ -278,7 +278,7 @@ configuration by clicking on the `Reload Points From Device` button at the end o |Reload Points| - +.. _device-configuration-form: Device Configuration Form ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -295,7 +295,7 @@ device to the platform. |Device Added| - +.. _configuring-sub-devices: Configuring Sub-devices ----------------------- @@ -311,18 +311,20 @@ As devices are configured, they're inserted into position in the side panel tree |Device Added to Tree| - +.. _reconfiguring-devices: Reconfiguring Devices ^^^^^^^^^^^^^^^^^^^^^ A device that's been added to a VOLTTRON instance can be reconfigured by changing its registry configuration or its device configuration. To launch reconfiguration, click on the wrench button next to the device in the side panel tree. +.. _reconfigure-device-button: |Reconfigure Device Button| Reconfiguration reloads the registry configuration editor and the device configuration form for the virtual device. The editor and the form work the same way in reconfiguration as during initial device configuration. +.. _reconfiguring-device: |Reconfiguring Device| The reconfiguration view shows the name, address, and ID of the physical device that the virtual device was configured @@ -341,7 +343,7 @@ To make changes to the device configuration form, click on the `File to Edit` se |Reconfigure Device Config| - +.. _exporting-registry-configuration-files: Exporting Registry Configuration Files -------------------------------------- diff --git a/docs/source/agent-framework/core-service-agents/volttron-central/volttron-central-overview.rst b/docs/source/agent-framework/core-service-agents/volttron-central/volttron-central-overview.rst index c333789add..bf03f3831f 100644 --- a/docs/source/agent-framework/core-service-agents/volttron-central/volttron-central-overview.rst +++ b/docs/source/agent-framework/core-service-agents/volttron-central/volttron-central-overview.rst @@ -15,11 +15,11 @@ and see the current offerings of the interface. VOLTTRON Central will allow you to: - * See a list of platforms being managed. - * Add and remove platforms. - * Install, start and stop agents on the managed platforms. - * Create dynamic graphs from the historians based upon data points. - * Execute functions on remote platforms. + * See a list of platforms being managed. + * Add and remove platforms. + * Install, start and stop agents on the managed platforms. + * Create dynamic graphs from the historians based upon data points. + * Execute functions on remote platforms. Volttron Central Agent diff --git a/docs/source/driver-framework/actuator/actuator-agent.rst b/docs/source/agent-framework/driver-framework/actuator/actuator-agent.rst similarity index 100% rename from docs/source/driver-framework/actuator/actuator-agent.rst rename to docs/source/agent-framework/driver-framework/actuator/actuator-agent.rst diff --git a/docs/source/driver-framework/bacnet/bacnet-auto-configuration.rst b/docs/source/agent-framework/driver-framework/bacnet/bacnet-auto-configuration.rst similarity index 100% rename from docs/source/driver-framework/bacnet/bacnet-auto-configuration.rst rename to docs/source/agent-framework/driver-framework/bacnet/bacnet-auto-configuration.rst diff --git a/docs/source/driver-framework/bacnet/bacnet-driver.rst b/docs/source/agent-framework/driver-framework/bacnet/bacnet-driver.rst similarity index 100% rename from docs/source/driver-framework/bacnet/bacnet-driver.rst rename to docs/source/agent-framework/driver-framework/bacnet/bacnet-driver.rst diff --git a/docs/source/driver-framework/bacnet/bacnet-proxy-agent.rst b/docs/source/agent-framework/driver-framework/bacnet/bacnet-proxy-agent.rst similarity index 100% rename from docs/source/driver-framework/bacnet/bacnet-proxy-agent.rst rename to docs/source/agent-framework/driver-framework/bacnet/bacnet-proxy-agent.rst diff --git a/docs/source/driver-framework/bacnet/bacnet-router-addressing.rst b/docs/source/agent-framework/driver-framework/bacnet/bacnet-router-addressing.rst similarity index 100% rename from docs/source/driver-framework/bacnet/bacnet-router-addressing.rst rename to docs/source/agent-framework/driver-framework/bacnet/bacnet-router-addressing.rst diff --git a/docs/source/driver-framework/bacnet/files/bacnet_cov.png b/docs/source/agent-framework/driver-framework/bacnet/files/bacnet_cov.png similarity index 100% rename from docs/source/driver-framework/bacnet/files/bacnet_cov.png rename to docs/source/agent-framework/driver-framework/bacnet/files/bacnet_cov.png diff --git a/docs/source/driver-framework/chargepoint/chargepoint-driver.rst b/docs/source/agent-framework/driver-framework/chargepoint/chargepoint-driver.rst similarity index 100% rename from docs/source/driver-framework/chargepoint/chargepoint-driver.rst rename to docs/source/agent-framework/driver-framework/chargepoint/chargepoint-driver.rst diff --git a/docs/source/driver-framework/chargepoint/chargepoint-specification.rst b/docs/source/agent-framework/driver-framework/chargepoint/chargepoint-specification.rst similarity index 99% rename from docs/source/driver-framework/chargepoint/chargepoint-specification.rst rename to docs/source/agent-framework/driver-framework/chargepoint/chargepoint-specification.rst index 16c2b6d0ee..4e2cf17bcf 100644 --- a/docs/source/driver-framework/chargepoint/chargepoint-specification.rst +++ b/docs/source/agent-framework/driver-framework/chargepoint/chargepoint-specification.rst @@ -432,7 +432,7 @@ VOLTTRON installation: .. - https://bitbucket.org/jurko/suds + https://pypi.org/project/suds-jurko/ Is there a mechanism for drivers to specify their own requirements.txt ? diff --git a/docs/source/agent-framework/driver-framework/dnp3-driver/dnp3-driver.rst b/docs/source/agent-framework/driver-framework/dnp3-driver/dnp3-driver.rst new file mode 100644 index 0000000000..8aa7b03839 --- /dev/null +++ b/docs/source/agent-framework/driver-framework/dnp3-driver/dnp3-driver.rst @@ -0,0 +1,90 @@ +.. _DNP3-Driver: + +=========== +DNP3 Driver +=========== + +VOLTTRON's DNP3 driver enables the use of `DNP3 `_ (Distributed Network Protocol) +communications, reading and writing points via a DNP3 Outstation. + +In order to use a DNP3 driver to read and write point data, a server component (i.e., Outstation) must also +be configured and running. + +Requirements +============ + +The DNP3 driver requires the `dnp3-python `_ package, a wrapper on Pydnp3 package. +This package can be installed in an activated environment with: + +.. code-block:: bash + + pip install dnp3-python + + +Driver Configuration +==================== + +There is one argument for the "driver_config" section of the DNP3 driver configuration file: + +Here is a sample DNP3 driver configuration file: + +.. code-block:: json + + { + "driver_config": {"master_ip": "0.0.0.0", "outstation_ip": "127.0.0.1", + "master_id": 2, "outstation_id": 1, + "port": 20000}, + "registry_config":"config://udd-Dnp3.csv", + "driver_type": "udd_dnp3", + "interval": 5, + "timezone": "UTC", + "campus": "campus-vm", + "building": "building-vm", + "unit": "Dnp3", + "publish_depth_first_all": true, + "heart_beat_point": "random_bool" + } + +A sample DNP3 driver configuration file can be found in the VOLTTRON repository +in ``services/core/PlatformDriverAgent/example_configurations/test_dnp3.config``. + + +DNP3 Registry Configuration File +================================ + +The driver's registry configuration file, a `CSV `_ file, +specifies which DNP3 points the driver will read and/or write. Each row configures a single DNP3 point. + +The following columns are required for each row: + + - **Volttron Point Name** - The name used by the VOLTTRON platform and agents to refer to the point. + - **Group** - The point's DNP3 group number. + - **Variation** - THe permit negotiated exchange of data formatted, i.e., data type. + - **Index** - The point's index number within its DNP3 data type (which is derived from its DNP3 group number). + - **Scaling** - A factor by which to multiply point values. + - **Units** - Point value units. + - **Writable** - TRUE or FALSE, indicating whether the point can be written by the driver (FALSE = read-only). + +Consult the **DNP3 data dictionary** for a point's Group and Index values. Point +definitions in the data dictionary are by agreement between the DNP3 Outstation and Master. +The VOLTTRON DNP3Agent loads the data dictionary of point definitions from the JSON file +at "point_definitions_path" in the DNP3Agent's config file. + +A sample data dictionary is available in ``services/core/DNP3Agent/dnp3/mesa_points.config``. + +Point definitions in the DNP3 driver's registry should look something like this: + +.. csv-table:: DNP3 + :header: Point Name,Volttron Point Name,Group,Variation,Index,Scaling,Units,Writable,Notes + + AnalogInput_index0,AnalogInput_index0,30,6,0,1,NA,FALSE,Double Analogue input without status + BinaryInput_index0,BinaryInput_index0,1,2,0,1,NA,FALSE,Single bit binary input with status + AnalogOutput_index0,AnalogOutput_index0,40,4,0,1,NA,TRUE,Double-precision floating point with flags + BinaryOutput_index0,BinaryOutput_index0,10,2,0,1,NA,TRUE,Binary Output with flags + + +A sample DNP3 driver registry configuration file is available +in ``services/core/PlatformDriverAgent/example_configurations/dnp3.csv``. + +For more information about Group Variation definition, please refer to `dnp3.Variation +`_. diff --git a/docs/source/driver-framework/drivers-overview.rst b/docs/source/agent-framework/driver-framework/drivers-overview.rst similarity index 96% rename from docs/source/driver-framework/drivers-overview.rst rename to docs/source/agent-framework/driver-framework/drivers-overview.rst index 9ef2b5f240..c0077381a6 100644 --- a/docs/source/driver-framework/drivers-overview.rst +++ b/docs/source/agent-framework/driver-framework/drivers-overview.rst @@ -1,7 +1,7 @@ .. _Driver-Framework: ========================= -Driver Framework Overview +Platform Driver Framework ========================= VOLTTRON drivers act as an interface between agents on the platform and a device. While running on the platform, @@ -160,3 +160,19 @@ To view data being published from the fake driver on the message bus, one can cd tail -f volttron.log + +.. toctree:: + + platform-driver/platform-driver + actuator/actuator-agent + fake-driver/fake-driver + bacnet/bacnet-driver + chargepoint/chargepoint-driver + dnp3-driver/dnp3-driver + ecobee/ecobee-web-driver + ieee-2030_5/ieee-2030_5-driver + modbus/modbus-driver + modbus/modbus-tk-driver + obix/obix + ted-driver/the-energy-detective-driver + home-assistant/HomeAssistantDriver \ No newline at end of file diff --git a/docs/source/driver-framework/ecobee/ecobee-web-driver.rst b/docs/source/agent-framework/driver-framework/ecobee/ecobee-web-driver.rst similarity index 100% rename from docs/source/driver-framework/ecobee/ecobee-web-driver.rst rename to docs/source/agent-framework/driver-framework/ecobee/ecobee-web-driver.rst diff --git a/docs/source/driver-framework/ecobee/files/ecobee_add_app.png b/docs/source/agent-framework/driver-framework/ecobee/files/ecobee_add_app.png similarity index 100% rename from docs/source/driver-framework/ecobee/files/ecobee_add_app.png rename to docs/source/agent-framework/driver-framework/ecobee/files/ecobee_add_app.png diff --git a/docs/source/driver-framework/ecobee/files/ecobee_api_key.png b/docs/source/agent-framework/driver-framework/ecobee/files/ecobee_api_key.png similarity index 100% rename from docs/source/driver-framework/ecobee/files/ecobee_api_key.png rename to docs/source/agent-framework/driver-framework/ecobee/files/ecobee_api_key.png diff --git a/docs/source/driver-framework/ecobee/files/ecobee_apps.png b/docs/source/agent-framework/driver-framework/ecobee/files/ecobee_apps.png similarity index 100% rename from docs/source/driver-framework/ecobee/files/ecobee_apps.png rename to docs/source/agent-framework/driver-framework/ecobee/files/ecobee_apps.png diff --git a/docs/source/driver-framework/ecobee/files/ecobee_console.png b/docs/source/agent-framework/driver-framework/ecobee/files/ecobee_console.png similarity index 100% rename from docs/source/driver-framework/ecobee/files/ecobee_console.png rename to docs/source/agent-framework/driver-framework/ecobee/files/ecobee_console.png diff --git a/docs/source/driver-framework/ecobee/files/ecobee_create_app.png b/docs/source/agent-framework/driver-framework/ecobee/files/ecobee_create_app.png similarity index 100% rename from docs/source/driver-framework/ecobee/files/ecobee_create_app.png rename to docs/source/agent-framework/driver-framework/ecobee/files/ecobee_create_app.png diff --git a/docs/source/driver-framework/ecobee/files/ecobee_developer_menu.png b/docs/source/agent-framework/driver-framework/ecobee/files/ecobee_developer_menu.png similarity index 100% rename from docs/source/driver-framework/ecobee/files/ecobee_developer_menu.png rename to docs/source/agent-framework/driver-framework/ecobee/files/ecobee_developer_menu.png diff --git a/docs/source/driver-framework/ecobee/files/ecobee_pin.png b/docs/source/agent-framework/driver-framework/ecobee/files/ecobee_pin.png similarity index 100% rename from docs/source/driver-framework/ecobee/files/ecobee_pin.png rename to docs/source/agent-framework/driver-framework/ecobee/files/ecobee_pin.png diff --git a/docs/source/driver-framework/ecobee/files/ecobee_verify_pin.png b/docs/source/agent-framework/driver-framework/ecobee/files/ecobee_verify_pin.png similarity index 100% rename from docs/source/driver-framework/ecobee/files/ecobee_verify_pin.png rename to docs/source/agent-framework/driver-framework/ecobee/files/ecobee_verify_pin.png diff --git a/docs/source/driver-framework/fake-driver/fake-driver.rst b/docs/source/agent-framework/driver-framework/fake-driver/fake-driver.rst similarity index 100% rename from docs/source/driver-framework/fake-driver/fake-driver.rst rename to docs/source/agent-framework/driver-framework/fake-driver/fake-driver.rst diff --git a/docs/source/driver-framework/files/driver_flow.png b/docs/source/agent-framework/driver-framework/files/driver_flow.png similarity index 100% rename from docs/source/driver-framework/files/driver_flow.png rename to docs/source/agent-framework/driver-framework/files/driver_flow.png diff --git a/docs/source/driver-framework/files/proxy_driver_flow.png b/docs/source/agent-framework/driver-framework/files/proxy_driver_flow.png similarity index 100% rename from docs/source/driver-framework/files/proxy_driver_flow.png rename to docs/source/agent-framework/driver-framework/files/proxy_driver_flow.png diff --git a/docs/source/agent-framework/driver-framework/home-assistant/HomeAssistantDriver.rst b/docs/source/agent-framework/driver-framework/home-assistant/HomeAssistantDriver.rst new file mode 100644 index 0000000000..ac6d51fa10 --- /dev/null +++ b/docs/source/agent-framework/driver-framework/home-assistant/HomeAssistantDriver.rst @@ -0,0 +1,186 @@ +.. _HomeAssistant-Driver: + +Home Assistant Driver +===================== + +The Home Assistant driver enables VOLTTRON to read any data point from any Home Assistant controlled device. +Currently control(write access) is supported only for lights(state and brightness) and thermostats(state and temperature). + +The following diagram shows interaction between platform driver agent and home assistant driver. + +.. mermaid:: + + sequenceDiagram + HomeAssistant Driver->>HomeAssistant: Retrieve Entity Data (REST API) + HomeAssistant-->>HomeAssistant Driver: Entity Data (Status Code: 200) + HomeAssistant Driver->>PlatformDriverAgent: Publish Entity Data + PlatformDriverAgent->>Controller Agent: Publish Entity Data + + Controller Agent->>HomeAssistant Driver: Instruct to Turn Off Light + HomeAssistant Driver->>HomeAssistant: Send Turn Off Light Command (REST API) + HomeAssistant-->>HomeAssistant Driver: Command Acknowledgement (Status Code: 200) + +Pre-requisites +-------------- +Before proceeding, find your Home Assistant IP address and long-lived access token from `here `_. + +Clone the repository, start volttron, install the listener agent, and the platform driver agent. + +- `Listener agent `_ +- `Platform driver agent `_ + +Configuration +-------------- + +After cloning, generate configuration files. Each device requires one device configuration file and one registry file. +Ensure your registry_config parameter in your device configuration file, links to correct registry config name in the +config store. For more details on how volttron platform driver agent works with volttron configuration store see, +`Platform driver configuration `_ +Examples for lights and thermostats are provided below. + +Device configuration +++++++++++++++++++++ + +Device configuration file contains the connection details to you home assistant instance and driver_type as "home_assistant" + +.. code-block:: json + + { + "driver_config": { + "ip_address": "Your Home Assistant IP", + "access_token": "Your Home Assistant Access Token", + "port": "Your Port" + }, + "driver_type": "home_assistant", + "registry_config": "config://light.example.json", + "interval": 30, + "timezone": "UTC" + } + +Registry Configuration ++++++++++++++++++++++++ + +Registry file can contain one single device and its attributes or a logical group of devices and its +attributes. Each entry should include the full entity id of the device, including but not limited to home assistant provided prefix +such as "light.", "climate." etc. The driver uses these prefixes to convert states into integers. +Like mentioned before, the driver can only control lights and thermostats but can get data from all devices +controlled by home assistant + +Each entry in a registry file should also have a 'Entity Point' and a unique value for 'Volttron Point Name'. The 'Entity ID' maps to the device instance, the 'Entity Point' extracts the attribute or state, and 'Volttron Point Name' determines the name of that point as it appears in VOLTTRON. + +Attributes can be located in the developer tools in the Home Assistant GUI. + +.. image:: home-assistant.png + + +Below is an example file named light.example.json which has attributes of a single light instance with entity +id 'light.example': + + +.. code-block:: json + + [ + { + "Entity ID": "light.example", + "Entity Point": "state", + "Volttron Point Name": "light_state", + "Units": "On / Off", + "Units Details": "on/off", + "Writable": true, + "Starting Value": true, + "Type": "boolean", + "Notes": "lights hallway" + }, + { + "Entity ID": "light.example", + "Entity Point": "brightness", + "Volttron Point Name": "light_brightness", + "Units": "int", + "Units Details": "light level", + "Writable": true, + "Starting Value": 0, + "Type": "int", + "Notes": "brightness control, 0 - 255" + } + ] + + +.. note:: + +When using a single registry file to represent a logical group of multiple physical entities, make sure the +"Volttron Point Name" is unique within a single registry file. + +For example, if a registry file contains entities with +id 'light.instance1' and 'light.instance2' the entry for the attribute brightness for these two light instances could +have "Volttron Point Name" as 'light1/brightness' and 'light2/brightness' respectively. This would ensure that data +is posted to unique topic names and brightness data from light1 is not overwritten by light2 or vice-versa. + +Example Thermostat Registry +*************************** + +For thermostats, the state is converted into numbers as follows: "0: Off, 2: heat, 3: Cool, 4: Auto", + +.. code-block:: json + + [ + { + "Entity ID": "climate.my_thermostat", + "Entity Point": "state", + "Volttron Point Name": "thermostat_state", + "Units": "Enumeration", + "Units Details": "0: Off, 2: heat, 3: Cool, 4: Auto", + "Writable": true, + "Starting Value": 1, + "Type": "int", + "Notes": "Mode of the thermostat" + }, + { + "Entity ID": "climate.my_thermostat", + "Entity Point": "current_temperature", + "Volttron Point Name": "volttron_current_temperature", + "Units": "F", + "Units Details": "Current Ambient Temperature", + "Writable": true, + "Starting Value": 72, + "Type": "float", + "Notes": "Current temperature reading" + }, + { + "Entity ID": "climate.my_thermostat", + "Entity Point": "temperature", + "Volttron Point Name": "set_temperature", + "Units": "F", + "Units Details": "Desired Temperature", + "Writable": true, + "Starting Value": 75, + "Type": "float", + "Notes": "Target Temp" + } + ] + + + +Transfer the registers files and the config files into the VOLTTRON config store using the commands below: + +.. code-block:: bash + + vctl config store platform.driver light.example.json HomeAssistant_Driver/light.example.json + vctl config store platform.driver devices/BUILDING/ROOM/light.example HomeAssistant_Driver/light.example.config + +Upon completion, initiate the platform driver. Utilize the listener agent to verify the driver output: + +.. code-block:: bash + + 2023-09-12 11:37:00,226 (listeneragent-3.3 211531) __main__ INFO: Peer: pubsub, Sender: platform.driver:, Bus: , Topic: devices/BUILDING/ROOM/light.example/all, Headers: {'Date': '2023-09-12T18:37:00.224648+00:00', 'TimeStamp': '2023-09-12T18:37:00.224648+00:00', 'SynchronizedTimeStamp': '2023-09-12T18:37:00.000000+00:00', 'min_compatible_version': '3.0', 'max_compatible_version': ''}, Message: + [{'light_brightness': 254, 'state': 'on'}, + {'light_brightness': {'type': 'integer', 'tz': 'UTC', 'units': 'int'}, + 'state': {'type': 'integer', 'tz': 'UTC', 'units': 'On / Off'}}] + +Running Tests ++++++++++++++++++++++++ +To run tests on the VOLTTRON home assistant driver you need to create a helper in your home assistant instance. This can be done by going to **Settings > Devices & services > Helpers > Create Helper > Toggle**. Name this new toggle **volttrontest**. After that run the pytest from the root of your VOLTTRON file. + +.. code-block:: bash + pytest volttron/services/core/PlatformDriverAgent/tests/test_home_assistant.py + +If everything works, you will see 6 passed tests. diff --git a/docs/source/agent-framework/driver-framework/home-assistant/home-assistant.png b/docs/source/agent-framework/driver-framework/home-assistant/home-assistant.png new file mode 100644 index 0000000000..d75333222b Binary files /dev/null and b/docs/source/agent-framework/driver-framework/home-assistant/home-assistant.png differ diff --git a/docs/source/driver-framework/ieee-2030_5/ieee-2030_5-driver.rst b/docs/source/agent-framework/driver-framework/ieee-2030_5/ieee-2030_5-driver.rst similarity index 100% rename from docs/source/driver-framework/ieee-2030_5/ieee-2030_5-driver.rst rename to docs/source/agent-framework/driver-framework/ieee-2030_5/ieee-2030_5-driver.rst diff --git a/docs/source/driver-framework/modbus/modbus-driver.rst b/docs/source/agent-framework/driver-framework/modbus/modbus-driver.rst similarity index 100% rename from docs/source/driver-framework/modbus/modbus-driver.rst rename to docs/source/agent-framework/driver-framework/modbus/modbus-driver.rst diff --git a/docs/source/driver-framework/modbus/modbus-tk-driver.rst b/docs/source/agent-framework/driver-framework/modbus/modbus-tk-driver.rst similarity index 100% rename from docs/source/driver-framework/modbus/modbus-tk-driver.rst rename to docs/source/agent-framework/driver-framework/modbus/modbus-tk-driver.rst diff --git a/docs/source/driver-framework/obix/obix.rst b/docs/source/agent-framework/driver-framework/obix/obix.rst similarity index 100% rename from docs/source/driver-framework/obix/obix.rst rename to docs/source/agent-framework/driver-framework/obix/obix.rst diff --git a/docs/source/driver-framework/platform-driver/platform-driver.rst b/docs/source/agent-framework/driver-framework/platform-driver/platform-driver.rst similarity index 100% rename from docs/source/driver-framework/platform-driver/platform-driver.rst rename to docs/source/agent-framework/driver-framework/platform-driver/platform-driver.rst diff --git a/docs/source/driver-framework/ted-driver/files/ted-spyders.png b/docs/source/agent-framework/driver-framework/ted-driver/files/ted-spyders.png similarity index 100% rename from docs/source/driver-framework/ted-driver/files/ted-spyders.png rename to docs/source/agent-framework/driver-framework/ted-driver/files/ted-spyders.png diff --git a/docs/source/driver-framework/ted-driver/the-energy-detective-driver.rst b/docs/source/agent-framework/driver-framework/ted-driver/the-energy-detective-driver.rst similarity index 100% rename from docs/source/driver-framework/ted-driver/the-energy-detective-driver.rst rename to docs/source/agent-framework/driver-framework/ted-driver/the-energy-detective-driver.rst diff --git a/docs/source/agent-framework/historian-agents/crate/crate-historian.rst b/docs/source/agent-framework/historian-agents/crate/crate-historian.rst index bb12a915ae..4e70c73760 100644 --- a/docs/source/agent-framework/historian-agents/crate/crate-historian.rst +++ b/docs/source/agent-framework/historian-agents/crate/crate-historian.rst @@ -7,6 +7,11 @@ Crate Historian Crate is an open source SQL database designed on top of a No-SQL design. It allows automatic data replication and self-healing clusters for high availability, automatic sharding, and fast joins, aggregations and sub-selects. +.. note:: + The Crate Historian is located within the **core** directory. :: + + services/core/CrateHistorian/ + Find out more about crate from ``_. diff --git a/docs/source/agent-framework/historian-agents/data-mover/data-mover-historian.rst b/docs/source/agent-framework/historian-agents/data-mover/data-mover-historian.rst index ad0d283015..566e702788 100644 --- a/docs/source/agent-framework/historian-agents/data-mover/data-mover-historian.rst +++ b/docs/source/agent-framework/historian-agents/data-mover/data-mover-historian.rst @@ -11,6 +11,11 @@ efficient by both batching data and by sending an RPC call to a remote historian remote message bus. This allows allows the Data Mover to be more robust by ensuring that the receiving historian is running. If the target is unreachable, the Data Mover will cache data until it is available. +.. note:: + The Data Mover Historian is located within the **core** directory. :: + + services/core/DataMover/ + Configuration ============= diff --git a/docs/source/agent-framework/historian-agents/forwarder/forward-historian.rst b/docs/source/agent-framework/historian-agents/forwarder/forward-historian.rst index 1e415fdc3e..c0c69b7c63 100644 --- a/docs/source/agent-framework/historian-agents/forwarder/forward-historian.rst +++ b/docs/source/agent-framework/historian-agents/forwarder/forward-historian.rst @@ -11,7 +11,10 @@ collected on a potentially less secure/powerful board. Given this use case, it is not optimized for batching large amounts of data when "live-ness" is not needed. For this use case, please see the :ref:`Data Mover Historian `. -The Forward Historian can be found in the `services/core directory`. +.. note:: + The Forward Historian is located within the **core** directory. :: + + services/core/ForwardHistorian/ Forward Historian can be used to forward data between two ZMQ instances, two RMQ instances, or between ZMQ and RMQ instances. For Forward Historian to establish a successful connection to the destination VOLTTRON instance: diff --git a/docs/source/agent-framework/historian-agents/historian-framework.rst b/docs/source/agent-framework/historian-agents/historian-framework.rst index 38ac4d4386..4a2c607452 100644 --- a/docs/source/agent-framework/historian-agents/historian-framework.rst +++ b/docs/source/agent-framework/historian-agents/historian-framework.rst @@ -15,6 +15,13 @@ captured and stored in some sort of data store. Historians exist for the follow - :ref:`MQTT Historian ` Forwards data to an MQTT broker - :ref:`InfluxDB Historian ` +.. attention:: + Historians are split into the directories of **core**, **contributed** and **unsupported**. :: + + services/contrib/ + services/core/ + services/unsupported/ + Other implementations of Historians can be created by following the :ref:`Developing Historian Agents ` guide. diff --git a/docs/source/agent-framework/historian-agents/influxdb/influxdb-historian.rst b/docs/source/agent-framework/historian-agents/influxdb/influxdb-historian.rst index d480214bfc..5ab7d1c55d 100644 --- a/docs/source/agent-framework/historian-agents/influxdb/influxdb-historian.rst +++ b/docs/source/agent-framework/historian-agents/influxdb/influxdb-historian.rst @@ -8,6 +8,11 @@ InfluxDB is an open source time series database with a fast, scalable engine and build DevOps Monitoring (Infrastructure Monitoring, Application Monitoring, Cloud Monitoring), IoT Monitoring, and real-time analytics solutions. +.. note:: + The Influxdb Historian is located within the **contributed** directory. :: + + services/contrib/InfluxdbHistorian/ + More information about InfluxDB is available from ``_. diff --git a/docs/source/agent-framework/historian-agents/mongodb/mongo-historian.rst b/docs/source/agent-framework/historian-agents/mongodb/mongo-historian.rst index 563343e928..d3516ef7a8 100644 --- a/docs/source/agent-framework/historian-agents/mongodb/mongo-historian.rst +++ b/docs/source/agent-framework/historian-agents/mongodb/mongo-historian.rst @@ -8,6 +8,11 @@ MongoDB is a NoSQL document database, which allows for great performance for tra documents do not have a schema, it is easy to store and query data which changes over time. MongoDB also scales horizontally using sharding. +.. note:: + The MongodbHistorian is located within the **unsupported** directory. :: + + services/unsupported/MongodbHistorian/ + For more information about MongoDB, read the `MongoDB documentation `_ diff --git a/docs/source/agent-framework/historian-agents/mqtt/mqtt-historian.rst b/docs/source/agent-framework/historian-agents/mqtt/mqtt-historian.rst index 2ed25a0b54..b2db1c1e09 100644 --- a/docs/source/agent-framework/historian-agents/mqtt/mqtt-historian.rst +++ b/docs/source/agent-framework/historian-agents/mqtt/mqtt-historian.rst @@ -10,6 +10,11 @@ Overview The MQTT Historian agent publishes data to an MQTT broker. The ``mqttlistener.py`` script will connect to the broker and print all messages. +.. note:: + The MQTT Historian is located within the **core** directory. :: + + services/core/MQTTHistorian/ + Dependencies ============ diff --git a/docs/source/agent-framework/historian-agents/openeis/openeis-historian.rst b/docs/source/agent-framework/historian-agents/openeis/openeis-historian.rst index 7909e3c2fb..1b7351ed52 100644 --- a/docs/source/agent-framework/historian-agents/openeis/openeis-historian.rst +++ b/docs/source/agent-framework/historian-agents/openeis/openeis-historian.rst @@ -8,6 +8,11 @@ An OpenEIS Historian has been developed to integrate real time data ingestion in the OpenEIS Historian to be able to communicate with an OpenEIS server a datasource must be created on the OpenEIS server. +.. note:: + The OpenEIS Historian is located within the **core** directory. :: + + services/core/OpenEISHistorian/ + The process of creating a dataset is documented in the `OpenEIS User's Guide `__ under `Creating a Dataset` heading. diff --git a/docs/source/agent-framework/historian-agents/sql-historian/sql-historian.rst b/docs/source/agent-framework/historian-agents/sql-historian/sql-historian.rst index 200c67e98d..77ecbdb3f3 100644 --- a/docs/source/agent-framework/historian-agents/sql-historian/sql-historian.rst +++ b/docs/source/agent-framework/historian-agents/sql-historian/sql-historian.rst @@ -10,6 +10,11 @@ The SQL Historian has been programmed to handle for inconsistent network connect based databases). All additions to the historian are batched and wrapped within a transaction with commit and rollback functions. This allows the maximum throughput of data with the most protection. +.. note:: + The SQL Historian is located within the **core** directory. :: + + services/core/SQLHistorian/ + Configuration ============= @@ -71,11 +76,16 @@ or Sqlite3 Specifics ----------------- -An Sqlite Historian provides a convenient solution for under powered systems. The database is a parameter to a location on the file system; 'database' should be a non-empty string. -By default, the location is relative to the agent's installation directory, however it will respect a rooted or relative path to the database. +An Sqlite Historian provides a convenient solution for under powered systems. The database is a parameter to a location +on the file system; 'database' should be a non-empty string. +By default, the location is relative to the agent's installation directory, however it will respect a rooted or +relative path to the database. -If 'database' does not have a rooted or relative path, the location of the database depends on whether the volttron platform is in secure mode. For more information on secure mode, see :ref:`Running-Agents-as-Unix-User`. -In secure mode, the location will be under /.agent-data directory because this will be the only directory in which the agent will have write-access. +If 'database' does not have a rooted or relative path, the location of the database depends on whether the volttron +platform is in agent isolation mode. For more information on agent isolation mode, +see :ref:`Running-Agents-as-Unix-User`. +In agent isolation mode, the location will be under /.agent-data directory because this +will be the only directory in which the agent will have write-access. In regular mode, the location will be under /data for backward compatibility. The following is a minimal configuration file that uses a relative path to the database. diff --git a/docs/source/agent-framework/operations-agents/index.rst b/docs/source/agent-framework/operations-agents/index.rst index a59d17e0a8..b8d4fd8eff 100644 --- a/docs/source/agent-framework/operations-agents/index.rst +++ b/docs/source/agent-framework/operations-agents/index.rst @@ -7,6 +7,11 @@ Operations Operations agents assist with the operations of the platform systems and provide alerts for various platform and environmental conditions. For details on each, please refer to the corresponding documents. +.. attention:: + All operations agents are located within the **operations** directory. :: + + services/ops/ + .. toctree:: :maxdepth: 2 diff --git a/docs/source/agent-framework/web-framework.rst b/docs/source/agent-framework/web-framework.rst index 2d6d7064bd..4164c1fed0 100644 --- a/docs/source/agent-framework/web-framework.rst +++ b/docs/source/agent-framework/web-framework.rst @@ -8,6 +8,10 @@ This document describes the interaction between web enabled agents and the Platf The web framework enables agent developers to expose JSON, static, and websocket endpoints. +.. note:: + The Platform Web Service Agent also provides a RESTful API which can be used to interact + with the VOLTTRON Platform, the documentation for which can be found :ref:`here `. + Web SubSystem ============= diff --git a/docs/source/api_doc_config.yml b/docs/source/api_doc_config.yml index 5b65cdb57c..b252b52f41 100644 --- a/docs/source/api_doc_config.yml +++ b/docs/source/api_doc_config.yml @@ -3,7 +3,6 @@ services: agent_excludes: - DNP3Agent - MarketServiceAgent - - OpenADRVenAgent - OpenEISHistorian - ObixHistoryPublish - IEEE2030_5Agent diff --git a/docs/source/conf.py b/docs/source/conf.py index 281ec60c20..dc75f60eb2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,37 +16,83 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) +import os import subprocess import sys -import os from glob import glob -from mock import Mock as MagicMock + import yaml +from mock import Mock as MagicMock -from volttron.platform.agent.utils import execute_command + +# Copied from volttron.platform.agent.util because it is not required +# that volttron be installed to utilize this script. +def execute_command(cmds, + env=None, + cwd=None, + logger=None, + err_prefix=None, + use_shell=False) -> str: + """ Executes a command as a subprocess + + If the return code of the call is 0 then return stdout otherwise + raise a RuntimeError. If logger is specified then write the exception + to the logger otherwise this call will remain silent. + + :param cmds:list of commands to pass to subprocess.run + :param env: environment to run the command with + :param cwd: working directory for the command + :param logger: a logger to use if errors occure + :param err_prefix: an error prefix to allow better tracing through the error message + :return: stdout string if successful + + :raises RuntimeError: if the return code is not 0 from suprocess.run + """ + + results = subprocess.run(cmds, + env=env, + cwd=cwd, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + shell=use_shell) + if results.returncode != 0: + err_prefix = err_prefix if err_prefix is not None else "Error executing command" + err_message = "\n{}: Below Command failed with non zero exit code.\n" \ + "Command:{} \nStderr:\n{}\n".format(err_prefix, + results.args, + results.stderr) + if logger: + logger.exception(err_message) + raise RuntimeError(err_message) + else: + raise RuntimeError(err_message) + + return results.stdout.decode('utf-8') class Mock(MagicMock): + @classmethod def __getattr__(cls, name): - return Mock() + return Mock() -autodoc_mock_imports = ['loadshape', 'numpy', 'sympy', 'xlrd', 'stomp', 'oadr2', 'pyodbc', 'lxml', 'pytest', - 'pint', 'pandas', 'suds', 'paho', 'pymongo', 'bson', 'subprocess32', 'heaters', 'meters', - 'hvac', 'blinds', 'vehicles'] +autodoc_mock_imports = [ + 'loadshape', 'numpy', 'sympy', 'xlrd', 'stomp', 'oadr2', 'pyodbc', 'lxml', + 'pytest', 'pint', 'pandas', 'suds', 'paho', 'pymongo', 'bson', + 'subprocess32', 'heaters', 'meters', 'hvac', 'blinds', 'vehicles' +] # -- Project information ----------------------------------------------------- project = 'VOLTTRON' -copyright = '2020, The VOLTTRON Community' +copyright = '2023, The VOLTTRON Community' author = 'The VOLTTRON Community' # The short X.Y version -version = '8.1.3' +version = '9.0' # The full version, including alpha/beta/rc tags -release = '8.1.3' - +release = '9.0' # -- General configuration --------------------------------------------------- @@ -71,7 +117,8 @@ def __getattr__(cls, name): # http://www.sphinx-doc.org/en/master/usage/extensions/todo.html 'sphinx.ext.todo', 'sphinx.ext.intersphinx', - 'm2r2' + 'm2r2', + 'sphinxcontrib.mermaid' ] # prefix sections with the document so that we can cross link @@ -108,7 +155,6 @@ def __getattr__(cls, name): # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' - # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for @@ -137,13 +183,11 @@ def __getattr__(cls, name): # # html_sidebars = {} - # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. htmlhelp_basename = 'VOLTTRONdoc' - # -- Options for LaTeX output ------------------------------------------------ latex_elements = { @@ -172,16 +216,11 @@ def __getattr__(cls, name): 'The VOLTTRON Community', 'manual'), ] - # -- Options for manual page output ------------------------------------------ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (main_doc, 'volttron', 'VOLTTRON Documentation', - [author], 1) -] - +man_pages = [(main_doc, 'volttron', 'VOLTTRON Documentation', [author], 1)] # -- Options for Texinfo output ---------------------------------------------- @@ -189,20 +228,22 @@ def __getattr__(cls, name): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (main_doc, 'VOLTTRON', 'VOLTTRON Documentation', - author, 'VOLTTRON', 'One line description of project.', - 'Miscellaneous'), + (main_doc, 'VOLTTRON', 'VOLTTRON Documentation', author, 'VOLTTRON', + 'One line description of project.', 'Miscellaneous'), ] - # -- Extension configuration ------------------------------------------------- # -- Options for intersphinx extension --------------------------------------- # Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/3.6': None, - 'volttron-ansible': ('https://volttron.readthedocs.io/projects/volttron-ansible/en/main/', - None)} +intersphinx_mapping = { + 'https://docs.python.org/3.6': + None, + 'volttron-ansible': + ('https://volttron.readthedocs.io/projects/volttron-ansible/en/main/', + None) +} # -- Options for todo extension ---------------------------------------------- @@ -247,11 +288,15 @@ def generate_apidoc(app): # generate api-docs for each api docs directory for docs_subdir in config.keys(): docs_subdir_path = os.path.join(apidocs_base_dir, docs_subdir) - agent_dirs = glob(os.path.join(volttron_root, config[docs_subdir]["path"], "*/")) + agent_dirs = glob( + os.path.join(volttron_root, config[docs_subdir]["path"], "*/")) file_excludes = [] if config[docs_subdir].get("file_excludes"): - for exclude_pattern in config[docs_subdir].get("file_excludes", []): - file_excludes.append(os.path.join(volttron_root, config[docs_subdir]["path"], exclude_pattern)) + for exclude_pattern in config[docs_subdir].get( + "file_excludes", []): + file_excludes.append( + os.path.join(volttron_root, config[docs_subdir]["path"], + exclude_pattern)) print("after file excludes. calling apidoc") agent_excludes = \ config[docs_subdir].get("agent_excludes") if config[docs_subdir].get("agent_excludes", []) else [] @@ -268,19 +313,24 @@ def run_apidoc(docs_dir, agent_dirs, agent_excludes, exclude_pattern): :param agent_excludes: agent directories to be skipped :param exclude_pattern: file name patterns to be excluded. This passed on to sphinx-apidoc command for exclude """ - print(f"In run apidoc params {docs_dir}, {agent_dirs}, {agent_excludes}, {exclude_pattern}") + print( + f"In run apidoc params {docs_dir}, {agent_dirs}, {agent_excludes}, {exclude_pattern}" + ) for agent_src_dir in agent_dirs: agent_src_dir = os.path.abspath(agent_src_dir) - agent_src_dir = agent_src_dir[:-1] if agent_src_dir.endswith("/") else agent_src_dir + agent_src_dir = agent_src_dir[:-1] if agent_src_dir.endswith( + "/") else agent_src_dir name = os.path.basename(agent_src_dir) agent_doc_dir = os.path.join(docs_dir, name) if name not in agent_excludes: sys.path.insert(0, agent_src_dir) - cmd = ["sphinx-apidoc", '-e', '-a', '-M', '-d 4', - '-t', os.path.join(script_dir, 'apidocs-templates'), - '--force', '-o', agent_doc_dir, agent_src_dir, - os.path.join(agent_src_dir, "setup.py"), os.path.join(agent_src_dir, "conftest.py") - ] + cmd = [ + "sphinx-apidoc", '-e', '-a', '-M', '-d 4', '-t', + os.path.join(script_dir, 'apidocs-templates'), '--force', '-o', + agent_doc_dir, agent_src_dir, + os.path.join(agent_src_dir, "setup.py"), + os.path.join(agent_src_dir, "conftest.py") + ] cmd.extend(exclude_pattern) subprocess.check_call(cmd) @@ -320,5 +370,6 @@ def clean_api_rst(app, exception): global apidocs_base_dir import shutil if os.path.exists(apidocs_base_dir): - print("Cleanup: Removing apidocs directory {}".format(apidocs_base_dir)) + print( + "Cleanup: Removing apidocs directory {}".format(apidocs_base_dir)) shutil.rmtree(apidocs_base_dir) diff --git a/docs/source/deploying-volttron/bootstrap-process.rst b/docs/source/deploying-volttron/bootstrap-process.rst index 974327c7ee..d3cadcbbb3 100644 --- a/docs/source/deploying-volttron/bootstrap-process.rst +++ b/docs/source/deploying-volttron/bootstrap-process.rst @@ -63,14 +63,12 @@ The options for customizing the location of the virtual environment are as follo --prompt PROMPT provide alternate prompt in activated environment (default: volttron) -Additional options are available for customizing where an environment will retrieve packages and/or upgrade -existing packages installed. +Additional options are available for customizing where an environment will retrieve packages installed. .. code-block:: bash update options: --offline install from cache without downloading - -u, --upgrade upgrade installed packages -w, --wheel build wheels in the pip wheelhouse To help boostrap an environment in the shortest number of steps we have grouped dependency packages under named diff --git a/docs/source/deploying-volttron/multi-platform/multi-platform-multi-bus.rst b/docs/source/deploying-volttron/multi-platform/multi-platform-multi-bus.rst index db9ae5bf5c..e01563c1c6 100644 --- a/docs/source/deploying-volttron/multi-platform/multi-platform-multi-bus.rst +++ b/docs/source/deploying-volttron/multi-platform/multi-platform-multi-bus.rst @@ -83,7 +83,7 @@ Platform agent, SQL historian agent and a Listener agent. The following shows an Is this the volttron you are attempting to setup? [Y]: What type of message bus (rmq/zmq)? [zmq]: rmq Name of this volttron instance: [volttron1]: central - RabbitMQ server home: [/home/user/rabbitmq_server/rabbitmq_server-3.7.7]: + RabbitMQ server home: [/home/user/rabbitmq_server/rabbitmq_server-3.9.29]: Fully qualified domain name of the system: [central]: Would you like to create a new self signed root CAcertificate for this instance: [Y]: @@ -95,7 +95,7 @@ Platform agent, SQL historian agent and a Listener agent. The following shows an Organization Unit: volttron Do you want to use default values for RabbitMQ home, ports, and virtual host: [Y]: 2020-04-13 13:29:36,347 rmq_setup.py INFO: Starting RabbitMQ server - 2020-04-13 13:29:46,528 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.7.7 is running at + 2020-04-13 13:29:46,528 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.29 is running at 2020-04-13 13:29:46,554 volttron.utils.rmq_mgmt DEBUG: Creating new VIRTUAL HOST: volttron 2020-04-13 13:29:46,582 volttron.utils.rmq_mgmt DEBUG: Create READ, WRITE and CONFIGURE permissions for the user: central-admin Create new exchange: volttron, {'durable': True, 'type': 'topic', 'arguments': {'alternate-exchange': 'undeliverable'}} @@ -108,7 +108,7 @@ Platform agent, SQL historian agent and a Listener agent. The following shows an 2020-04-13 13:29:46,601 rmq_setup.py INFO: Creating root ca with the following info: {'C': 'US', 'ST': 'WA', 'L': 'Richland', 'O': 'PNNL', 'OU': 'VOLTTRON', 'CN': 'central-root-ca'} Created CA cert 2020-04-13 13:29:49,668 rmq_setup.py INFO: **Stopped rmq server - 2020-04-13 13:30:00,556 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.7.7 is running at + 2020-04-13 13:30:00,556 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.29 is running at 2020-04-13 13:30:00,557 rmq_setup.py INFO: ####################### @@ -443,7 +443,7 @@ name is set to "collector2". Is this the volttron you are attempting to setup? [Y]: What type of message bus (rmq/zmq)? [zmq]: rmq Name of this volttron instance: [volttron1]: collector2 - RabbitMQ server home: [/home/user/rabbitmq_server/rabbitmq_server-3.7.7]: + RabbitMQ server home: [/home/user/rabbitmq_server/rabbitmq_server-3.9.29]: Fully qualified domain name of the system: [node-rmq]: Would you like to create a new self signed root CA certificate for this instance: [Y]: @@ -455,7 +455,7 @@ name is set to "collector2". Organization Unit: volttron Do you want to use default values for RabbitMQ home, ports, and virtual host: [Y]: 2020-04-13 13:29:36,347 rmq_setup.py INFO: Starting RabbitMQ server - 2020-04-13 13:29:46,528 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.7.7 is running at + 2020-04-13 13:29:46,528 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.29 is running at 2020-04-13 13:29:46,554 volttron.utils.rmq_mgmt DEBUG: Creating new VIRTUAL HOST: volttron 2020-04-13 13:29:46,582 volttron.utils.rmq_mgmt DEBUG: Create READ, WRITE and CONFIGURE permissions for the user: collector2-admin Create new exchange: volttron, {'durable': True, 'type': 'topic', 'arguments': {'alternate-exchange': 'undeliverable'}} @@ -468,7 +468,7 @@ name is set to "collector2". 2020-04-13 13:29:46,601 rmq_setup.py INFO: Creating root ca with the following info: {'C': 'US', 'ST': 'WA', 'L': 'Richland', 'O': 'PNNL', 'OU': 'VOLTTRON', 'CN': 'collector2-root-ca'} Created CA cert 2020-04-13 13:29:49,668 rmq_setup.py INFO: **Stopped rmq server - 2020-04-13 13:30:00,556 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.7.7 is running at + 2020-04-13 13:30:00,556 rmq_setup.py INFO: Rmq server at /home/user/rabbitmq_server/rabbitmq_server-3.9.29 is running at 2020-04-13 13:30:00,557 rmq_setup.py INFO: ####################### diff --git a/docs/source/deploying-volttron/platform-configuration.rst b/docs/source/deploying-volttron/platform-configuration.rst index 5a63429dac..b3b9f5f390 100644 --- a/docs/source/deploying-volttron/platform-configuration.rst +++ b/docs/source/deploying-volttron/platform-configuration.rst @@ -6,11 +6,11 @@ Platform Configuration Each instance of the VOLTTRON platform includes a `config` file which is used to configure the platform instance on startup. This file is kept in :term:`VOLTTRON_HOME` and is created using the `volttron-cfg` (`vcfg`) command, or will -be created with default values on start up of the platform otherwise. +be created with default values on start up of the platform otherwise. `vcfg` also provides commands to configure and +install few frequently used agents and update configuration-store entries Following is helpful information about the `config` file and the `vcfg` command. - VOLTTRON_HOME ============= @@ -75,9 +75,13 @@ The example consists of the following entries: VOLTTRON Config =============== -The `volttron-cfg` or `vcfg` command allows for an easy configuration of the VOLTTRON environment. The command includes -the ability to set up the platform configuration, an instance of the platform historian, VOLTTRON Central UI, and -VOLTTRON Central Platform agent. +The `volttron-cfg` or `vcfg` command allows for an easy configuration of the VOLTTRON environment. +The `vcfg` command can be run without any arguments to go through all available configuration steps sequentially and +pick what is appropriate for a given instance of VOLTTRON. This is useful when starting with a new VOLTTRON instance. + +`vcfg` command can also be used with specific subcommands to configure and install specific agents such as +listener agent, platform historian, VOLTTRON Central UI, and VOLTTRON Central Platform agent +or add one or more agent configurations to VOLTTRON's configuration store. Running `vcfg` will create a `config` file in `VOLTTRON_HOME` which will be populated according to the answers to prompts. This process should be repeated for each platform instance, and can be re-run to reconfigure a platform @@ -158,12 +162,52 @@ Optional Arguments - **-v, --verbose** - Enables verbose output in standard-output (PIP output, etc.) - **--vhome VHOME** - Provide a path to set `VOLTTRON_HOME` for this instance - **--instance-name INSTANCE_NAME** - Provide a name for this instance. Required for running secure agents mode - - **--list-agents** - Display a list of configurable agents (Listener, Platform Driver, Platform Historian, VOLTTRON - Central, VOLTTRON Central Platform) - - **--agent AGENT [AGENT ...]** - Configure listed agents - - **--secure-agent-users** - Require that agents run as their own Unix users (this requires running + - **--agent-isolation-mode** - Require that agents run as their own Unix users (this requires running `scripts/secure_user_permissions.sh` as `sudo`) +Sub commands +------------ +**--list-agents** +~~~~~~~~~~~~~~~~~~ + Display a list of configurable agents (Listener, Platform Driver, Platform Historian, VOLTTRON + Central, VOLTTRON Central Platform) + +**--agent AGENT [AGENT ...]** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Configure and install one of the listed agents for a specific VOLTTRON instance + +**update-config-store --metadata-file METADATA_FILE_OR_DIR [METADATA_FILE_OR_DIR ...]** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Command to bulk add/update multiple agent configurations to VOLTTRON's configuration store. This is especially useful +for automated deployment use cases. For example, a automated deployment setup where devices in a network are +detected, and configurations for platform driver and control agents such as ILCAgent are auto generated using semantic +tags. In such as scenario, adding all the generated configurations to the configuration store using the below command +is more efficient than running the "vctl config store" command once for each configuration + +This command can only be used when volttron instance is not running. + +Usage: + +.. code-block:: bash + + vcfg update-config-store --metadata-file + + +Format for Metadata file: + +.. code-block:: + + { "vip-id": [ + { + "config-name": "optional. name. defaults to config + "config": "json config or string config or config file name", + "config-type": "optional. type of config - csv or json or raw. defaults to json" + }, ... + ],... + } + + RabbitMQ Arguments ------------------ vcfg command to configure a single RabbitMQ instance of VOLTTRON. diff --git a/docs/source/deploying-volttron/secure-deployment-considerations.rst b/docs/source/deploying-volttron/secure-deployment-considerations.rst index 8fce4d0ada..ba5b97408e 100644 --- a/docs/source/deploying-volttron/secure-deployment-considerations.rst +++ b/docs/source/deploying-volttron/secure-deployment-considerations.rst @@ -169,4 +169,33 @@ can be further configured to detect and restart the RabbitMQ service if it crash to detect when the RabbitMQ server crashes/disconnects and reconnect when it becomes available. In this deployment setup, a VOLTTRON platform will not start/stop the RabbitMQ server. +Isolate agent process from core VOLTTRON process +================================================ +It is possible to run a volttron agent process as a unique unix users and restricting the unix level permission for +that agent user such that agent has write access only to its own agent-data folder inside the VOLTTRON_HOME. This +prevents agents from accidentally or intentionally editing/deleting other volttron process or other agents' files or +system files outside of VOLTTRON_HOME + +For more information, refer to :ref:`Agent Isolation Mode: Running Users as unique Unix user`. + +Non-Auth Implementation +======================= + +There may be some use-cases, such as simulating deployments or agent development, where security is not a consideration. +In these cases, it is possible to disable VOLTTRON's authentication and authorization, stripping away the security layer from the +VIP messagebus and simplifying agent connection and RPC communication. Since this is not ideal for any deployment, this can only +be done by manually modifying the volttron configuration file. + +Within the config file located within VOLTTRON_HOME, the allow-auth option must be added and set to False. + +.. code-block:: console + + [volttron] + message-bus = zmq + vip-address = tcp://127.0.0.1:22916 + instance-name = volttron1 + allow-auth = False + +In simulation environments where multiple volttron instances are used, it is important to ensure that auth settings are +the same across all instances. diff --git a/docs/source/deploying-volttron/single-machine.rst b/docs/source/deploying-volttron/single-machine.rst index 2c3ceff277..c80e310b3d 100644 --- a/docs/source/deploying-volttron/single-machine.rst +++ b/docs/source/deploying-volttron/single-machine.rst @@ -144,23 +144,23 @@ Install Agents and Historian Out of the box, VOLTTRON includes a number of agents which may be useful for single machine deployments: - * historians - Historians automatically record a data from a number of topics published to the bus. For more - information on the historian framework or one of the included concrete implementations, view the - :ref:`docs ` - * Listener - This example agent can be useful for debugging drivers or other agents publishing to the bus. - :ref:`docs ` + * historians - Historians automatically record a data from a number of topics published to the bus. For more information on the historian framework or one of the included concrete implementations, view the :ref:`Historian-Framework` + + + * Listener - This example agent can be useful for debugging drivers or other agents publishing to the bus. For more details please read :ref:`Listener-Agent` + * Platform Driver - The :ref:`Platform-Driver` is responsible for managing device communication on a platform instance. - * weather agents - weather agents can be used to collect weather data from sources like - :ref:`Weather.gov ` + * weather agents - weather agents can be used to collect weather data from sources like `weather.gov `_ + - .. note:: +.. note:: The `services/core`, `services/ops`, and `examples` directories in the repository contain additional agents to use to fit individual use cases. For a simple setup example, a Platform Driver, SQLite Historian, and Listener are installed using the following steps: -#. Create a configuration file for the Platform Driver and SQLite Historian (it is advised to create a `configs` directory +1. Create a configuration file for the Platform Driver and SQLite Historian (it is advised to create a `configs` directory in volttron root to keep configs for a deployment). For information on how to create configurations for these agents, view their docs: @@ -168,27 +168,31 @@ For a simple setup example, a Platform Driver, SQLite Historian, and Listener ar * :ref:`SQLite Historian ` * :ref:`Listener ` - For a simple example, the configurations can be copied as-is to the `configs` directory: +For a simple example, the configurations can be copied as-is to the `configs` directory: - .. code-block:: bash +.. code-block:: bash + + mkdir configs - cp services/core/PlatformDriverAgent/platform-driver.agent configs - cp services/core/SQLHistorian/config.sqlite configs +.. code-block:: bash + + cp services/core/PlatformDriverAgent/platform-driver.agent configs/ + cp services/core/SQLHistorian/config.sqlite configs/ cp examples/ListenerAgent/config configs/listener.config -#. Use the `install-agent.py` script to install the agent on the platform: +2. Use the `install-agent.py` script to install the agent on the platform: .. code-block:: bash - python scripts/install-agent.py -s services/core/SQLHistorian -c configs/config.sqlite --tag listener + python scripts/install-agent.py -s services/core/SQLHistorian -c configs/config.sqlite --tag platform_historian python scripts/install-agent.py -s services/core/PlatformDriverAgent -c configs/platform-driver.agent --tag platform_driver - python scripts/install-agent.py -s examples/ListenerAgent -c configs/listener.config --tag platform_historian + python scripts/install-agent.py -s examples/ListenerAgent -c configs/listener.config --tag listener - .. note:: +.. note:: The `volttron.log` file will contain logging indicating that the agent has installed successfully. - .. code-block:: console +.. code-block:: console 2020-10-27 11:42:08,882 () volttron.platform.auth INFO: AUTH: After authenticate user id: control.connection, b'c61dff8e-f362-4906-964f-63c32b99b6d5' 2020-10-27 11:42:08,882 () volttron.platform.auth INFO: authentication success: userid=b'c61dff8e-f362-4906-964f-63c32b99b6d5' domain='vip', address='localhost:1000:1000:3249', mechanism='CURVE', credentials=['ZrDvPG4JNLE26GoPUrTP22rV0PV8uGCnrXThrNFk_Ec'], user='control.connection' @@ -202,7 +206,7 @@ For a simple setup example, a Platform Driver, SQLite Historian, and Listener ar 2020-10-27 11:42:11,415 () volttron.platform.auth WARNING: Get list of peers from subsystem directly 2020-10-27 11:42:11,419 () volttron.platform.auth INFO: auth file /home/james/.volttron/auth.json loaded -#. Use the `vctl status` command to ensure that the agents have been successfully installed: +3. Use the `vctl status` command to ensure that the agents have been successfully installed: .. code-block:: bash @@ -232,15 +236,15 @@ concrete drivers such as the BACnet or Modbus drivers, view their respective doc .. note:: - This section will assume the user has created a `configs` directory in the volttron root directory, activated - the Python virtual environment, and started the platform as noted above. + This section will assume the user is currently in the volttron root directory and has created a `configs` directory, + activated the Python virtual environment, and started the platform as noted above. .. code-block:: console - cp examples/configurations/drivers/fake.config /configs - cp examples/configurations/drivers/fake.csv /configs + cp examples/configurations/drivers/fake.config configs/ + cp examples/configurations/drivers/fake.csv configs/ vctl config store platform.driver devices/campus/building/fake configs/fake.config - vctl config store platform.driver fake.csv devices/fake.csv + vctl config store platform.driver fake.csv configs/fake.csv --csv .. note:: diff --git a/docs/source/developing-volttron/community.rst b/docs/source/developing-volttron/community.rst index 05f32dde04..24718c7d1c 100644 --- a/docs/source/developing-volttron/community.rst +++ b/docs/source/developing-volttron/community.rst @@ -12,18 +12,10 @@ Contributing back to the project, which is encouraged but not required, enhances To learn more, check out :ref:`Contributing ` and :ref:`Documentation `. -Slack Channel -============= - -volttron-community.slack.com is where the |VOLTTRON| community at large can ask questions and meet with others -using |VOLTTRON|. To be added to Slack please email the VOLTTRON team at -`volttron@pnnl.gov `__. - - Mailing List ============ -Join the mailing list by emailing `volttron@pnnl.gov `__. +Join the mailing list at Eclipse: https://projects.eclipse.org/projects/iot.volttron/contact Stack Overflow @@ -39,8 +31,8 @@ Office Hours PNNL hosts office hours every other week on Fridays at 11 AM (PST). These meetings are designed to be very informal where VOLTTRON developers can answer specific questions about the inner workings of VOLTTRON. These meetings are also available for topical discussions of different aspects of the VOLTTRON platform. Currently the office hours are -available through a Zoom meeting. To be invited to the link meeting, contact the volttron team via email: -``__ +available through a Zoom meeting. All members of the mailing list will receive invites to the meetings. Join the +mailing list https://projects.eclipse.org/projects/iot.volttron/contact. Meetings are recorded and can be reviewed `here `__. diff --git a/docs/source/developing-volttron/developing-agents/agent-development.rst b/docs/source/developing-volttron/developing-agents/agent-development.rst index 240d8510a2..92cec9a8f7 100644 --- a/docs/source/developing-volttron/developing-agents/agent-development.rst +++ b/docs/source/developing-volttron/developing-agents/agent-development.rst @@ -918,7 +918,7 @@ VOLTTRON uses *pytest* as a framework for executing tests. All unit tests shoul For instructions on writing unit and integration tests with *pytest*, refer to the :ref:`Writing Agent Tests ` documentation. -*pytest* is not installed with the distribution by default. To install py.test and it's dependencies execute the +*pytest* is not installed with the distribution by default. To install pytest and it's dependencies execute the following: .. code-block:: bash @@ -1008,3 +1008,4 @@ environment. An example of setting `VOLTTRON_HOME` to `/tmp/v1home` is as follo developing-market-agents example-agents/index specifications/index + using-asyncio-in-agents diff --git a/docs/source/developing-volttron/developing-agents/example-agents/scheduler-example-agent.rst b/docs/source/developing-volttron/developing-agents/example-agents/scheduler-example-agent.rst index 83817ae8e8..36939d3535 100644 --- a/docs/source/developing-volttron/developing-agents/example-agents/scheduler-example-agent.rst +++ b/docs/source/developing-volttron/developing-agents/example-agents/scheduler-example-agent.rst @@ -41,7 +41,7 @@ The agent listens to schedule announcements from the actuator and then issues a @PubSub.subscribe('pubsub', topics.ACTUATOR_SCHEDULE_ANNOUNCE(campus='campus', building='building',unit='unit')) def actuate(self, peer, sender, bus, topic, headers, message): - print ("response:",topic,headers,message) + print("response:", topic, headers, message) if headers[headers_mod.REQUESTER_ID] != agent_id: return '''Match the announce for our fake device with our ID diff --git a/docs/source/developing-volttron/developing-agents/using-asyncio-in-agents.rst b/docs/source/developing-volttron/developing-agents/using-asyncio-in-agents.rst new file mode 100644 index 0000000000..07b9e27e4b --- /dev/null +++ b/docs/source/developing-volttron/developing-agents/using-asyncio-in-agents.rst @@ -0,0 +1,107 @@ +.. _Using-Asyncio-In-Agents: + +======================= +Using Asyncio in Agents +======================= + +The purpose of this section to is to show how to use Asyncio with Gevent within the Agent development framework. + +Before we dive into the example, we establish the following concepts: + +* An Event Loop is a software design pattern that handles events concurrently; it waits for and dispatches multiple events concurrently and gives the illusion of executing the events in "parallel". In Python, the `Event Loop `_ contains a list of Tasks which controls when and how those Tasks are executed. + +* A `Task `_ is an object that runs a `coroutine `_ (i.e. asynchronous function). In Python, coroutines are written using the 'async' keyword. + +* A `Greenlet `_ is a "lightweight coroutine for in-process sequential concurrent programming. Greenlets can be used on their own, but they are frequently used with frameworks such as gevent to provide higher-level abstractions and asynchronous I/O. + +* Asyncio is a built-in Python module that allows the developer to write concurrent code. + +* Gevent is a "coroutine-based Python networking library that uses greenlet to provide a high-level synchronous API on top of the libev or libuv event loop". + +* VOLTTRON predates the inclusion of asyncio in python and therefore uses gevent for its base. + +The general steps to use Asyncio within the Volttron Agent framework are the following: + +1. Create an async method. +2. Create a method which creates and starts the Asyncio Event Loop. +3. Use gevent.spawn (or spawn_later) to start a greenlet using the method in step 2. + +Below are code examples of how to implement the steps within an agent. For demonstration purposes, we name this agent, ExampleAsyncioAgent. + +Step 1: Create an async method. + +.. code-block:: python + + class ExampleAsyncioAgent(Agent): + + # This is the async method. + async def handle_event(self, event): + ... + # releases control so other coroutines can run. + await asyncio.sleep(1) + return "hello!" + + +Step 2. Create a method which creates and starts the Asyncio Event Loop. + +.. code-block:: python + + class ExampleAsyncioAgent(Agent): + + # This is a wrapper method that is self contained for launching from gevent. + def _start_asyncio_loop(self): + loop = asyncio.get_event_loop() + loop.create_task(self.handle_event) + loop.run_forever() + + +Step 3. Use gevent.spawn (or spawn_later) to start a greenlet using the method in step 2. + +.. code-block:: python + + class ExampleAsyncioAgent(Agent): + + @Core.receiver('onstart') + def onstart(self, sender, **kwargs): + + # Spawn greenlet in 3 seconds, use self._start_asyncio_loop as a callback for executing + # the greenlet + # + # Does not have to be in onstart can be in any function, but must be after the agent + # has started up. + gevent.spawn_later(3, self._start_asyncio_loop) + + +To review, below is the complete agent class with all the relevant and aforementioned codeblocks: + +.. code-block:: python + + import gevent + import asyncio + + class ExampleAsyncioAgent(Agent): + + @Core.receiver("onstart") + def onstart(self, sender, **kwargs): + gevent.spawn_later(3, self._start_asyncio_loop) + + def _start_asyncio_loop(self): + loop = asyncio.get_event_loop() + loop.create_task(self.ven_client.run()) + loop.run_forever() + + async def handle_event(self, event): + # do things that include a blocking call + ... + + await asyncio.sleep(1) + return "hello!" + + +References + +* `Python Asyncio Primer `_ + +* `Python Asyncio documentation `_ + +* `Gevent documentation `_ diff --git a/docs/source/developing-volttron/developing-drivers/driver-development.rst b/docs/source/developing-volttron/developing-drivers/driver-development.rst index 49377dfddb..61766b5cd8 100644 --- a/docs/source/developing-volttron/developing-drivers/driver-development.rst +++ b/docs/source/developing-volttron/developing-drivers/driver-development.rst @@ -170,19 +170,19 @@ Name" specifying the name of the register, and "Point Value", the current value "boolean": bool} class Interface(BasicRevert, BaseInterface): - def __init__(self, **kwargs): - super(Interface, self).__init__(**kwargs) - self.csv_path = None - - def configure(self, config_dict, registry_config_str): - self.csv_path = config_dict.get("csv_path", "csv_device.csv") - if not os.path.isfile(self.csv_path): - _log.info("Creating csv 'device'") - with open(self.csv_path, "w+") as csv_device: - writer = DictWriter(csv_device, fieldnames=CSV_FIELDNAMES) - writer.writeheader() - writer.writerows(CSV_DEFAULT) - self.parse_config(registry_config_str) + def __init__(self, **kwargs): + super(Interface, self).__init__(**kwargs) + self.csv_path = None + + def configure(self, config_dict, registry_config_str): + self.csv_path = config_dict.get("csv_path", "csv_device.csv") + if not os.path.isfile(self.csv_path): + _log.info("Creating csv 'device'") + with open(self.csv_path, "w+") as csv_device: + writer = DictWriter(csv_device, fieldnames=CSV_FIELDNAMES) + writer.writeheader() + writer.writerows(CSV_DEFAULT) + self.parse_config(registry_config_str) At the end of the configuration method, the Driver parses the registry configuration. The registry configuration is a csv which is used to tell the Driver which register the user wishes to communicate with and includes a few meta-data diff --git a/docs/source/developing-volttron/development-environment/pycharm/index.rst b/docs/source/developing-volttron/development-environment/pycharm/index.rst index c2044d04d0..1001ff61eb 100644 --- a/docs/source/developing-volttron/development-environment/pycharm/index.rst +++ b/docs/source/developing-volttron/development-environment/pycharm/index.rst @@ -6,7 +6,7 @@ Pycharm Development Environment Pycharm is an IDE dedicated to developing python projects. It provides coding assistance and easy access to debugging tools as well as integration with -py.test. It is a popular tool for working with VOLTTRON. +pytest. It is a popular tool for working with VOLTTRON. Jetbrains provides a free community version that can be downloaded from https://www.jetbrains.com/pycharm/ @@ -98,8 +98,8 @@ top level source directory; git will ignore changes to these files. Testing an Agent ---------------- -Agent tests written in py.test can be run simply by right-clicking the tests -directory and selecting `Run 'py.test in tests`, so long as the root directory +Agent tests written in pytest can be run simply by right-clicking the tests +directory and selecting `Run 'pytest in tests`, so long as the root directory is set as the VOLTTRON source root. |Run Tests| diff --git a/docs/source/developing-volttron/python-for-matlab-users.rst b/docs/source/developing-volttron/python-for-matlab-users.rst index 289c874be2..0e24533302 100644 --- a/docs/source/developing-volttron/python-for-matlab-users.rst +++ b/docs/source/developing-volttron/python-for-matlab-users.rst @@ -184,7 +184,7 @@ More Resources -------------- `NumPy for Matlab Users -`_ +`_ Has a nice list of common operations in Matlab and NumPy. `NumPy Homepage diff --git a/docs/source/developing-volttron/quick-start.rst b/docs/source/developing-volttron/quick-start.rst new file mode 100644 index 0000000000..c2692cb0df --- /dev/null +++ b/docs/source/developing-volttron/quick-start.rst @@ -0,0 +1,331 @@ +.. _VOLTTRON-Quick-Start: + +.. role:: bash(code) + :language: bash + +======================= +VOLTTRON Quick Start +======================= + +This tutorial has been written with the intent of helping folks get up-and-running with VOLTTRON. The tutorial is designed to deploy on Linux virtual machines. While not going too much into depth, it covers the following topics: + +- Install the VOLTTRON platform and verify the installation. +- Get familiar with the VOLTTRON components. +- Get familiar with the VOLTTRON commands. + +Prerequisites +============================== + +In this tutorial we will demonstrate installing the VOLTTRON platform at an `Ubuntu 20.04 LTS (Focal Fossa) `_ Virtual machine. In order to follow the tutorial, the prerequisites are as follows: + +- Linux OS image (e.g., Ubuntu 20.04) +- Virtualization software (e.g., VirtualBox, VMware) +- Internet accessibility +- sudo capability + +Installation Steps +============================== + +1. Install prerequisites +------------------------------ + +The first step is to make sure required dependencies are fulfilled. Install the dependencies with the following command: + +.. code-block:: bash + + $ sudo apt-get update + $ sudo apt-get install build-essential python3-dev python3-venv openssl libssl-dev libevent-dev git + +Verify python installation with the following command: + +.. code-block:: bash + + $ python3 --version + +.. code-block:: bash + + # expected output similar to this + Python 3.8.10 + + +Similarly, verify git installation. + +.. code-block:: bash + + $ git --version + +.. code-block:: bash + + # expected output similar to this + git version 2.25.1 + +2. Download VOLTTRON code +------------------------------ + +Download the VOLTTRON code to the default home directory using :code:`git clone` command. + +.. code-block:: bash + + $ cd ~ + $ git clone https://github.com/VOLTTRON/volttron + +.. code-block:: bash + + # expected output similar to this + Cloning into 'volttron'... + remote: Enumerating objects: 82987, done. + remote: Counting objects: 100% (4892/4892), done. + remote: Compressing objects: 100% (1971/1971), done. + remote: Total 82987 (delta 3155), reused 4294 (delta 2890), pack-reused 78095 + Receiving objects: 100% (82987/82987), 102.73 MiB | 4.19 MiB/s, done. + Resolving deltas: 100% (57997/57997), done. + Checking out files: 100% (1807/1807), done. + ... + +.. note:: + + In this tutorial we download the VOLTTRON code to the default home directory. + However, feel free to download the code to a different place as desired. + +.. code-block:: bash + + # $ mkdir + # $ cd + # $ git clone https://github.com/VOLTTRON/volttron + +After successfully downloading the VOLTTRON package, change the current working path to the code path. Then, inspect the source code files. + +.. code-block:: bash + + $ cd volttron + $ ls + +.. code-block:: bash + + # expected output similar to this + bootstrap.py deprecated pylintrc requirements.py stop-volttron + ci-integration docs pytest.ini scripts TERMS.md + CONTRIBUTING.md examples README.md services volttron + COPYRIGHT integrations readthedocs.yml setup.py volttron_data + debugging_utils LICENSE.md RELEASE_NOTES.md start-volttron volttrontesting + +3. Bootstrap VOLTTRON environment +------------------------------ + +VOLTTRON is a Python-based platform. In this step, we will rely on the :code:`bootstrap.py` script in the root directory to bootstrap the platform environment. This process will create a virtual environment and install the package's Python dependencies. + +.. note:: + + VOLTTRON provides different message bus options. In this tutorial we will demonstrate the default ZeroMQ option. (Read more about :ref:`message bus`.) + + +Bootstrap the VOLTTRON environment by running the following command. (This may take a while.) + +.. code-block:: bash + + $ python3 bootstrap.py + +.. code-block:: bash + + # expected output similar to this + UPDATE: [] + Installing required packages + + pip install --no-deps wheel==0.30 + Collecting wheel==0.30 + Using cached + + ... + + +After bootstrap finished, we activate the Python virtual environment with the following command: + +.. code-block:: bash + + $ source env/bin/activate + +You may notice the command prompt has changed and there is the virtual environment name as prefix. e.g., :code:`(volttron) user@host:~/volttron $`. The prefix environment name indicates the virtual environment is activated. + +Alternatively, you can use the following command to verify if the virtual environment is up. + +.. code-block:: bash + + $ env |grep VIRTUAL_ENV |wc -l + +.. code-block:: bash + + # expected output 1(virtual environment is up) or 0 (not up) + + +To deactivate the virtual environment (if you run this command, remember to activate the virtual environment again to follow the rest of the steps.) + +.. code-block:: bash + + # Uncomment, if you run this command, + # remember to activate the virtual environment again + # to follow the rest of the steps + + # deactivate volttron + +4. Start VOLTTRON +------------------------------ + +In this step, we will start the VOLTTRON platform and demonstrate several VOLTTRON commands. + +Start the VOLTTRON platform with the following command: + +.. code-block:: bash + + $ ./start-volttron + +.. code-block:: bash + + # expected output similar to this + ... + Starting VOLTTRON verbosely in the background with VOLTTRON_HOME=/home/user/.volttron + Waiting for VOLTTRON to startup.. + VOLTTRON startup complete + +Check the status of VOLTTRON with the following command: + +.. code-block:: bash + + $ vctl status + +For fresh installation, the result might look like the following since there are no agents installed yet. + +.. code-block:: bash + + # expected output similar to this + No installed Agents found + +.. tip:: + + Use :code:`vctl status` to check status. + This is a very useful command to inspect the status of VOLTTRON. + +VOLTTRON platform comes with several built in services and example agents out of the box. In this demo, we use the Listener Agent - a simple agent that periodically publishes heartbeat message and listens to everything on the message bus. (Read more about :ref:`agent `.) + +Install the Listener agent using the following command: + +.. code-block:: bash + + $ vctl install examples/ListenerAgent --tag listener + + +.. code-block:: bash + + # expected output similar to this + Agent b755bae2-a3f5-44a0-b01f-81e30b989138 installed + + +Start the agent we just installed specified by the `listener` tag. + +.. code-block:: bash + + $ vctl start --tag listener + +.. code-block:: bash + + # expected output similar to this + Starting b755bae2-a3f5-44a0-b01f-81e30b989138 listeneragent-3.3 + +Check the status again. + +.. code-block:: bash + + $ vctl status + +.. code-block:: bash + + # expected output similar to this + UUID AGENT IDENTITY TAG STATUS HEALTH + 8 listeneragent-3.3 listeneragent-3.3_1 listener running [2192] GOOD + + +From the above result, we can tell the listener agent is functioning properly! + +.. tip:: + + While the :code:`--tag` sub-command is optional, a tag is helpful for managing agents by adding semantic tags to different topics, so that topic can be queried by tags instead of specific topic name or topic name pattern. + + You can choose any tag name that makes sense to you, as long as the tags are already defined in the VOLTTRON tagging schema. (Read more about :ref:`tag `.) + +In addition to the :code:`vctl status`, another way to check VOLTTRON status is by inspecting the :code:`volttron.log` file. The file provides rich information about the platform and becomes handy for debug purposes. + +.. code-block:: bash + + $ tail -f volttron.log + + +.. code-block:: bash + + # example output (success) + # listener agent is publishing heartbeat messages successively. + 2022-03-04 14:12:46,463 (listeneragent-3.3 2192) __main__ INFO: Peer: pubsub, Sender: listeneragent-3.3_1:, Bus: , Topic: heartbeat/listeneragent-3.3_1, Headers: {'TimeStamp': '2022-03-04T19:12:46.460096+00:00', 'min_compatible_version': '3.0', 'max_compatible_version': ''}, Message: 'GOOD' + ... + + +.. code-block:: bash + + # example output (error) + 2022-03-04 13:16:05,469 (listeneragent-3.3 3233) volttron.platform.vip.agent.core ERROR: No response to hello message after 10 seconds. + 2022-03-04 13:16:05,469 (listeneragent-3.3 3233) volttron.platform.vip.agent.core ERROR: Type of message bus used zmq + 2022-03-04 13:16:05,469 (listeneragent-3.3 3233) volttron.platform.vip.agent.core ERROR: A common reason for this is a conflicting VIP IDENTITY. + 2022-03-04 13:16:05,469 (listeneragent-3.3 3233) volttron.platform.vip.agent.core ERROR: Another common reason is not having an auth entry onthe target instance. + 2022-03-04 13:16:05,469 (listeneragent-3.3 3233) volttron.platform.vip.agent.core ERROR: Shutting down agent. + ... + +5. Stop VOLTTRON (Optional) +------------------------------ + +To stop VOLTTRON, use the following command: + +.. code-block:: bash + + $ ./stop-volttron + +.. code-block:: bash + + # expected output similar to this + Shutting down VOLTTRON + +After stopping the platform, check the status again to verify the VOLTTRON platform is shut down. + +.. code-block:: bash + + $ vctl status + +.. code-block:: bash + + # expected output similar to this + VOLTTRON is not running. This command requires VOLTTRON platform to be running + +Clean up (Optional) +============================== + +If for some reason you would like to clean up VOLTTRON, here is the guide to remove the whole VOLTTRON package + +- remove the code folder (e.g., :code:`~/volttron/`) +- remove the :code:`.volttron/` folder at :code:`VOLTTRON_HOME/.volttron` (e.g., by default at :code:`~/.volttron`) + +Summary +============================== + +This short tutorial for VOLTTRON first-time users. We covered the following topics. + +- VOLTTRON platform installation. (e.g., on a Virtual Machine.) +- VOLTTRON components. (e.g., agent, message bus, tag.) +- VOLTTRON commands. (e.g., :code:`start-volttron`, :code:`vctl status`.) + + +Next Steps +============================== + +There are several walk-throughs and detailed explanations of platform features to explore additional aspects of the +platform: + +* :ref:`Agent Framework ` +* :ref:`Driver Framework ` +* Demonstration of the :ref:`management UI ` +* :ref:`RabbitMQ setup ` with Federation and Shovel plugins + diff --git a/docs/source/index.rst b/docs/source/index.rst index 4b196a82c0..83b1b4b283 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -56,38 +56,17 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla introduction/definitions introduction/license - -.. toctree:: - :caption: Developing in VOLTTRON - :hidden: - :titlesonly: - :maxdepth: 1 - - developing-volttron/community - developing-volttron/development-environment/index - developing-volttron/developing-agents/agent-development - developing-volttron/developing-drivers/driver-development - developing-volttron/contributing-code - developing-volttron/contributing-documentation - developing-volttron/jupyter/jupyter-notebooks - developing-volttron/python-for-matlab-users - - .. toctree:: - :caption: Deploying VOLTTRON + :caption: Platform Features :hidden: :titlesonly: :maxdepth: 1 - deploying-volttron/bootstrap-process - deploying-volttron/platform-configuration - deploying-volttron/deployment-planning-options - deploying-volttron/single-machine - deploying-volttron/multi-platform/index - deploying-volttron/secure-deployment-considerations - deploying-volttron/linux-system-hardening - deploying-volttron/recipe-deployment - + platform-features/message-bus/index + platform-features/control/index + platform-features/config-store/configuration-store + platform-features/security/volttron-security + platform-features/web-api/introduction .. toctree:: :caption: Agent Framework @@ -98,45 +77,42 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla agent-framework/agents-overview agent-framework/core-service-agents/index agent-framework/operations-agents/index + agent-framework/driver-framework/drivers-overview agent-framework/historian-agents/historian-framework agent-framework/web-framework agent-framework/integrating-simulations/index agent-framework/platform-service-standardization agent-framework/third-party-agents - .. toctree:: - :caption: Driver Framework + :caption: Developing in VOLTTRON :hidden: :titlesonly: :maxdepth: 1 - driver-framework/drivers-overview - driver-framework/platform-driver/platform-driver - driver-framework/actuator/actuator-agent - driver-framework/fake-driver/fake-driver - driver-framework/bacnet/bacnet-driver - driver-framework/chargepoint/chargepoint-driver - driver-framework/dnp3-driver/dnp3-driver - driver-framework/ecobee/ecobee-web-driver - driver-framework/ieee-2030_5/ieee-2030_5-driver - driver-framework/modbus/modbus-driver - driver-framework/modbus/modbus-tk-driver - driver-framework/obix/obix - driver-framework/ted-driver/the-energy-detective-driver - + developing-volttron/community + developing-volttron/development-environment/index + developing-volttron/developing-agents/agent-development + developing-volttron/developing-drivers/driver-development + developing-volttron/contributing-code + developing-volttron/contributing-documentation + developing-volttron/jupyter/jupyter-notebooks + developing-volttron/python-for-matlab-users .. toctree:: - :caption: Platform Features + :caption: Deploying VOLTTRON :hidden: :titlesonly: :maxdepth: 1 - platform-features/message-bus/index - platform-features/control/index - platform-features/config-store/configuration-store - platform-features/security/volttron-security - + deploying-volttron/bootstrap-process + deploying-volttron/platform-configuration + deploying-volttron/deployment-planning-options + deploying-volttron/single-machine + deploying-volttron/multi-platform/index + deploying-volttron/secure-deployment-considerations + deploying-volttron/linux-system-hardening + deploying-volttron/recipe-deployment .. toctree:: :caption: VOLTTRON Core Service Agents @@ -147,7 +123,6 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla volttron-api/services/*/modules - .. toctree:: :caption: VOLTTRON Core Operations Agents :hidden: @@ -157,7 +132,6 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla volttron-api/ops/*/modules - .. toctree:: :caption: VOLTTRON Topics :hidden: @@ -167,6 +141,23 @@ at our bi-weekly office-hours and on Slack. To be invited to office-hours or sla Releases volttron-topics/troubleshooting/index volttron-topics/volttron-applications/index + +.. toctree:: + :caption: Tutorials + :hidden: + :titlesonly: + :maxdepth: 1 + + tutorials/quick-start + + +.. toctree:: + :caption: Tutorials + :hidden: + :titlesonly: + :maxdepth: 1 + + tutorials/quick-start Indices and tables diff --git a/docs/source/introduction/license.rst b/docs/source/introduction/license.rst index ccbc5801e1..4a08e6d14a 100644 --- a/docs/source/introduction/license.rst +++ b/docs/source/introduction/license.rst @@ -4,7 +4,7 @@ License ======= -Copyright 2019, Battelle Memorial Institute. +Copyright 2020, Battelle Memorial Institute. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at diff --git a/docs/source/introduction/platform-install.rst b/docs/source/introduction/platform-install.rst index 878d9c5a22..5b77d8ae82 100644 --- a/docs/source/introduction/platform-install.rst +++ b/docs/source/introduction/platform-install.rst @@ -10,7 +10,7 @@ Installing the Platform VOLTTRON is written in Python 3.6+ and runs on Linux Operating Systems. For users unfamiliar with those technologies, the following resources are recommended: -- `Python 3.6 Tutorial `_ +- `Python 3.6 Tutorial `_ - `Linux Tutorial `_ This guide will specify commands to use to successfully install the platform on supported Linux distributions, but a @@ -194,44 +194,40 @@ Step 2 - Install Erlang packages For RabbitMQ based VOLTTRON, some of the RabbitMQ specific software packages have to be installed. +Install Erlang pre-requisites ++++++++++++++++++++++++++++++ +.. code-block:: bash + sudo apt-get update + sudo apt-get install -y gnupg apt-transport-https libsctp1 libncurses5 -On Debian based systems and CentOS 6/7 -"""""""""""""""""""""""""""""""""""""" - -If you are running a Debian or CentOS system, you can install the RabbitMQ dependencies by running the -"rabbit_dependencies.sh" script, passing in the OS name and appropriate distribution as parameters. The -following are supported: - -* `debian bionic` (for Ubuntu 18.04) +Please note there could be other pre-requisites that erlang requires based on the version of Erlang and OS. If there are other pre-requisites required, install of erlang should fail with appropriate error message. -* `debian xenial` (for Ubuntu 16.04 or Linux Mint 18.04) +Purge previous versions of Erlang ++++++++++++++++++++++++++++++++++ -* `debian stretch` (for Debian Stretch) +.. code-block:: bash -* `debian buster` (for Debian Buster) + sudo apt-get purge -yf erlang-base -* `raspbian buster` (for Raspbian/Raspberry Pi OS Buster) +Install Erlang +++++++++++++++ -Example command: +Download and install ErlangOTP from [Erlang Solutions](https://www.erlang-solutions.com/downloads/). +RMQ uses components - ssl, public_key, asn1, and crypto. These are by default included in the OTP +RabbitMQ 3.9.29 is compatible with Erlang versions 24.3.4.2 to 25.2. VOLTTRON was tested with Erlang version 25.2-1 +Example: Ubuntu 22.04 .. code-block:: bash - ./scripts/rabbit_dependencies.sh debian xenial + wget https://binaries2.erlang-solutions.com/ubuntu/pool/contrib/e/esl-erlang/esl-erlang_25.2-1~ubuntu~jammy_amd64.deb + sudo dpkg -i esl-erlang_25.2-1~ubuntu~jammy_amd64.deb -Alternatively -""""""""""""" - -You can download and install Erlang from `Erlang Solutions `_. -Please include OTP/components - ssl, public_key, asn1, and crypto. -Also lock your version of Erlang using the `yum-plugin-versionlock `_. +Example: Ubuntu 20.04 +.. code-block:: bash -.. note:: - Currently VOLTTRON only officially supports specific versions of Erlang for each operating system: - * 1:22.1.8.1-1 for Debian - * 1:21.2.6+dfsg-1 for Raspbian - * Specific Erlang 21.x versions correspond to CentOS versions 6, 7, and 8, these can be found - `here `_ + wget https://binaries2.erlang-solutions.com/ubuntu/pool/contrib/e/esl-erlang/esl-erlang_25.2-1~ubuntu~focal_amd64.deb + sudo dpkg -i esl-erlang_25.2-1~ubuntu~focal_amd64.deb Step 3 - Configure hostname @@ -253,16 +249,20 @@ to connect to empd (port 4369) on ." Step 4 - Bootstrap the environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Remove older version of rabbitmq_server directory if you are upgrading from a older version. Defaults to +/rabbitmq_server/rabbitmq_server-3.9.7 +Run the rabbitmq bootstrap step from within an VOLTTRON activated environment. .. code-block:: bash cd volttron + source env/bin/activate python3 bootstrap.py --rabbitmq [optional install directory. defaults to /rabbitmq_server] This will build the platform and create a virtual Python environment and dependencies for RabbitMQ. It also installs RabbitMQ server as the current user. If an install path is provided, that path should exist and the user should have -write permissions. RabbitMQ will be installed under `/rabbitmq_server-3.7.7`. The rest of the -documentation refers to the directory `/rabbitmq_server-3.7.7` as `$RABBITMQ_HOME`. +write permissions. RabbitMQ will be installed under `/rabbitmq_server-`. The rest of the +documentation refers to the directory `/rabbitmq_server-` as `$RABBITMQ_HOME`. .. note:: @@ -279,11 +279,11 @@ Thus, you can use $RABBITMQ_HOME to see if the RabbitMQ server is installed by c .. note:: The `RABBITMQ_HOME` environment variable can be set in ~/.bashrc. If doing so, it needs to be set to the RabbitMQ - installation directory (default path is `/rabbitmq_server/rabbitmq_server-3.7.7`) + installation directory (default path is `/rabbitmq_server/rabbitmq_server-3.9.29`) .. code-block:: bash - echo 'export RABBITMQ_HOME=$HOME/rabbitmq_server/rabbitmq_server-3.7.7'|sudo tee --append ~/.bashrc + echo 'export RABBITMQ_HOME=$HOME/rabbitmq_server/rabbitmq_server-3.9.29'|sudo tee --append ~/.bashrc source ~/.bashrc $RABBITMQ_HOME/sbin/rabbitmqctl status @@ -341,7 +341,7 @@ prompts for necessary details. Is this the volttron you are attempting to setup? [Y]: Creating rmq config yml - RabbitMQ server home: [/home/vdev/rabbitmq_server/rabbitmq_server-3.7.7]: + RabbitMQ server home: [/home/vdev/rabbitmq_server/rabbitmq_server-3.9.29]: Fully qualified domain name of the system: [cs_cbox.pnl.gov]: Enable SSL Authentication: [Y]: @@ -361,7 +361,7 @@ prompts for necessary details. https port for the RabbitMQ management plugin: [15671]: INFO:rmq_setup.pyc:Starting rabbitmq server Warning: PID file not written; -detached was passed. - INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.7.7 + INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.9.29 INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost @@ -375,7 +375,7 @@ prompts for necessary details. INFO:requests.packages.urllib3.connectionpool:Starting new HTTP connection (1): localhost INFO:rmq_setup.pyc:**Stopped rmq server Warning: PID file not written; -detached was passed. - INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.7.7 + INFO:rmq_setup.pyc:**Started rmq server at /home/vdev/rabbitmq_server/rabbitmq_server-3.9.29 INFO:rmq_setup.pyc: ####################### diff --git a/docs/source/platform-features/config-store/agent-configuration-store.rst b/docs/source/platform-features/config-store/agent-configuration-store.rst index fbf55ecd3c..aea72e2ac6 100644 --- a/docs/source/platform-features/config-store/agent-configuration-store.rst +++ b/docs/source/platform-features/config-store/agent-configuration-store.rst @@ -312,36 +312,66 @@ Platform RPC Methods -------------------- -Methods for Agents -^^^^^^^^^^^^^^^^^^ - -Agent methods that change configurations do not trigger any callbacks unless trigger_callback is True. - -**set_config(config_name, contents, trigger_callback=False)** - Change/create a configuration file on the platform. +**set_config(identity, config_name, contents, config_type="raw", trigger_callback=True, send_update=True)** - +Change/create a configuration on the platform for an agent with the specified identity. Requires the +authorization capability 'edit_config_store'. By default agents have access to edit only their own config store entries. + +**manage_store(identity, config_name, contents, config_type="raw", trigger_callback=True, send_update=True)** - +Deprecated method. Please use set_config instead. Will be removed in VOLTTRON version 10. +Change/create a configuration on the platform for an agent with the specified identity. Requires the +authorization capability 'edit_config_store'. By default agents have access to edit only their own config store entries. + +**delete_config(identity, config_name, trigger_callback=True, send_update=True)** - Delete a configuration for an +agent with the specified identity. Requires the authorization capability 'edit_config_store'. By default agents have +access to edit only their own config store entries. + +**manage_delete_config(identity, config_name, trigger_callback=True, send_update=True)** - +Deprecated method. Please use delete_config instead. Will be removed in VOLTTRON version 10. +Delete a configuration for an agent with the specified identity. Requires the authorization capability +'edit_config_store'. By default agents have access to edit only their own config store entries. + +**delete_store(identity)** - Delete all configurations for an agent with the specified identity. Requires the +authorization capability 'edit_config_store'. By default agents have access to edit only their own config store entries. +Calls the agent's update_config with the action `DELETE_ALL` and no configuration name. -**get_configs()** - Get all of the configurations for an Agent. +**manage_delete_store(identity)** - +Deprecated method. Please use delete_store instead. Will be removed in VOLTTRON version 10. +Delete all configurations for an agent with the specified identity. Requires the +authorization capability 'edit_config_store'. By default agents have access to edit only their own config store entries. +Calls the agent's update_config with the action `DELETE_ALL` and no configuration name. -**delete_config(config_name, trigger_callback=False)** - Delete a configuration. +**list_configs(identity)** - Get a list of configurations for an agent with the specified identity. +**manage_list_configs(identity)** - +Deprecated method. Please use list_configs instead. Will be removed in VOLTTRON version 10. +Get a list of configurations for an agent with the specified identity. -Methods for Management -^^^^^^^^^^^^^^^^^^^^^^ +**list_stores()** - Get a list of all the agents with configurations. -**manage_store_config(identity, config_name, contents, config_type="raw")** - Change/create a configuration on the -platform for an agent with the specified identity +**manage_list_stores()** - +Deprecated method. Please use list_stores instead. Will be removed in VOLTTRON version 10. +Get a list of all the agents with configurations. -**manage_delete_config(identity, config_name)** - Delete a configuration for an agent with the specified identity. -Calls the agent's update_config with the action `DELETE_ALL` and no configuration name. -**manage_delete_store(identity)** - Delete all configurations for a :term:`VIP Identity`. +**get_config(identity, config_name, raw=True)** - Get the contents of a configuration file. If raw is set to +`True` this function will return the original file, otherwise it will return the parsed representation of the file. -**manage_list_config(identity)** - Get a list of configurations for an agent with the specified identity. +**manage_get_config(identity, config_name, raw=True)** - +Deprecated method. Please use get_config instead. Will be removed in VOLTTRON version 10. +Get the contents of a configuration file. If raw is set to `True` this function will return the original file, +otherwise it will return the parsed representation of the file. -**manage_get_config(identity, config_name, raw=True)** - Get the contents of a configuration file. If raw is set to -`True` this function will return the original file, otherwise it will return the parsed representation of the file. +**initialize_configs(identity)** - Called by an Agent at startup to trigger initial configuration state push. +Requires the authorization capability 'edit_config_store'. By default agents have access to edit only their own +config store entries. -**manage_list_stores()** - Get a list of all the agents with configurations. +**get_metadata(identity, config_name)** - Get the metadata of configuration named *config_name* of agent +identified by *identity*. Returns the type(json, csv, raw) of the configuration, modified date and actual content +**manage_get_metadata(identity, config_name)** - +Deprecated method. Please use get_metadata instead. Will be removed in VOLTTRON version 10. +Get the metadata of configuration named *config_name* of agent +identified by *identity*. Returns the type(json, csv, raw) of the configuration, modified date and actual content Direct Call Methods ^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/platform-features/control/platform-commands.rst b/docs/source/platform-features/control/platform-commands.rst index 8075b81bac..bae8a4f7a8 100644 --- a/docs/source/platform-features/control/platform-commands.rst +++ b/docs/source/platform-features/control/platform-commands.rst @@ -113,12 +113,12 @@ Agent Options instance - **--agent-monitor-frequency AGENT_MONITOR_FREQUENCY** - How often should the platform check for crashed agents and attempt to restart. Units=seconds. Default=600 -- **--secure-agent-users SECURE_AGENT_USERS** - Require that agents run with their own users (this requires running +- **--agent-isolation-mode AGENT_ISOLATION_MODE** - Require that agents run with their own users (this requires running scripts/secure_user_permissions.sh as sudo) .. warning:: - Certain options alter some basic behaviors of the platform, such as `--secure-agent-users` which causes the platform + Certain options alter some basic behaviors of the platform, such as `--agent-isolation-mode` which causes the platform to run each agent using its own Unix user to spawn the process. Please view the documentation for each feature to understand its implications before choosing to run the platform in that fashion. @@ -338,7 +338,7 @@ vcfg Optional Arguments vcfg --rabbitmq single|federation|shovel [rabbitmq config file] -- **--secure-agent-users** Require that agents run with their own users (this requires running +- **--agent-isolation-mode** Require that agents run with their own users (this requires running scripts/secure_user_permissions.sh as sudo) .. warning:: diff --git a/docs/source/platform-features/message-bus/rabbitmq/rabbitmq-ssl-auth.rst b/docs/source/platform-features/message-bus/rabbitmq/rabbitmq-ssl-auth.rst index 06b9694157..37fc7efdc3 100644 --- a/docs/source/platform-features/message-bus/rabbitmq/rabbitmq-ssl-auth.rst +++ b/docs/source/platform-features/message-bus/rabbitmq/rabbitmq-ssl-auth.rst @@ -14,7 +14,7 @@ configurations can be seen by running the following command: .. code-block:: bash - cat ~/rabbitmq_server/rabbitmq_server-3.7.7/etc/rabbitmq/rabbitmq.conf + cat ~/rabbitmq_server/rabbitmq_server-3.9.29/etc/rabbitmq/rabbitmq.conf The configurations required to enable SSL: @@ -78,8 +78,8 @@ To configure RabbitMQ-VOLTTRON to use SSL based authentication, we need to add S # defaults to true ssl: 'true' - # defaults to ~/rabbitmq_server/rabbbitmq_server-3.7.7 - rmq-home: "~/rabbitmq_server/rabbitmq_server-3.7.7" + # defaults to ~/rabbitmq_server/rabbbitmq_server-3.9.29 + rmq-home: "~/rabbitmq_server/rabbitmq_server-3.9.29" The parameters of interest for SSL based configuration are diff --git a/docs/source/platform-features/message-bus/rabbitmq/rabbitmq-volttron.rst b/docs/source/platform-features/message-bus/rabbitmq/rabbitmq-volttron.rst index 5e224c8137..5d29152ee1 100644 --- a/docs/source/platform-features/message-bus/rabbitmq/rabbitmq-volttron.rst +++ b/docs/source/platform-features/message-bus/rabbitmq/rabbitmq-volttron.rst @@ -56,8 +56,8 @@ Path: `$VOLTTRON_HOME/rabbitmq_config.yml` # defaults to true ssl: 'true' - # defaults to ~/rabbitmq_server/rabbbitmq_server-3.7.7 - rmq-home: "~/rabbitmq_server/rabbitmq_server-3.7.7" + # defaults to ~/rabbitmq_server/rabbbitmq_server-3.9.29 + rmq-home: "~/rabbitmq_server/rabbitmq_server-3.9.29" Each VOLTTRON instance resides within a RabbitMQ virtual host. The name of the virtual host needs to be unique per VOLTTRON instance if there are multiple virtual instances within a single host/machine. The hostname needs to be able @@ -67,7 +67,7 @@ These needs to be set appropriately if the default ports are not used. The 'ssl' flag indicates if SSL based authentication is required or not. If set to `True`, information regarding SSL certificates needs to be also provided. SSL based authentication is described in detail in -`Authentication And Authorization With RabbitMQ Message Bus `_. +:ref:`Authentication And Authorization With RabbitMQ Message Bus `. To configure the VOLTTRON instance to use RabbitMQ message bus, run the following command: diff --git a/docs/source/platform-features/message-bus/vip/vip-authentication.rst b/docs/source/platform-features/message-bus/vip/vip-authentication.rst index b31feda632..56240d5cd8 100644 --- a/docs/source/platform-features/message-bus/vip/vip-authentication.rst +++ b/docs/source/platform-features/message-bus/vip/vip-authentication.rst @@ -32,6 +32,27 @@ For example:: (volttron)[user@home]$ volttron-ctl auth serverkey FSG7LHhy3v8tdNz3gK35G6-oxUcyln54pYRKu5fBJzU +Authentication Implementation +----------------------------- + +Authentication is handled on the server and client sides. +The server authentication handles the setup and monitoring of incoming connection authentication. +This verifies that the authentication method matches the appropriate auth protocol that has been implemented. +The client authentication creates a connection address that contains all the necessary authentication meta data to +allow an agent to connect to a server. While the default authentication implementation is for Zap ZMQ, it is possible to +develop new authentication client and server designs. The server implementation is handled by the auth service, and client +authentication is handled by the agent core. + +Client Authentication: +* create_authenticated_address + +Server Authentication: +* setup_authentication +* handle_authentication +* stop_authentication +* unbind_authentication + + Peer Authentication ------------------- diff --git a/docs/source/platform-features/message-bus/vip/vip-authorization.rst b/docs/source/platform-features/message-bus/vip/vip-authorization.rst index 97386ff798..61beaf3e60 100644 --- a/docs/source/platform-features/message-bus/vip/vip-authorization.rst +++ b/docs/source/platform-features/message-bus/vip/vip-authorization.rst @@ -9,6 +9,31 @@ VOLTTRON platform that agent proves its identity to the platform. Once authenti the :ref:`message bus `. VIP authorization is about giving a platform owner the ability to limit the capabilities of authenticated agents. +Authorization Implementation +---------------------------- + +Authorization has client and server implementations. On the client side, +authorization is used to connect the auth subsystem to the auth service. +This provides the connection needed to manage capabilites and protected topics. +The server authorization is used to manage pending credentials and certificates that +are handled by authentication, as well as managing capabilties and protected topics. + +BaseServerAuthorization: +* approve_authorization +* deny_authorization +* delete_authorization +* get_authorization +* get_authorization_status +* get_pending_authorizations +* get_approved_authorizations +* get_denied_authorizations +* update_user_capabilites +* load_protected_topics +* update_protected_topics + +BaseClientAuthorization +* connect_remote_platform + There are two parts to authorization: #. Required capabilities (specified in agent's code) diff --git a/docs/source/platform-features/security/running-agent-as-user.rst b/docs/source/platform-features/security/agent-isolation-mode.rst similarity index 84% rename from docs/source/platform-features/security/running-agent-as-user.rst rename to docs/source/platform-features/security/agent-isolation-mode.rst index 24be716ab0..af987a65ae 100644 --- a/docs/source/platform-features/security/running-agent-as-user.rst +++ b/docs/source/platform-features/security/agent-isolation-mode.rst @@ -1,8 +1,8 @@ -.. _Running-Agents-as-Unix-User: +.. _Agent-Isolation-Mode: -============================ -Running Agents as Unix Users -============================ +==================== +Agent Isolation Mode +==================== This VOLTTRON feature will cause the platform to create a new, unique Unix user(agent users) on the host machine for each agent installed on the platform. This user will have restricted permissions for the file system, and will be used @@ -19,9 +19,9 @@ VOLTTRON platform. All files and folder created by the VOLTTRON process in this mode would not have any access to others by default. Permission for Unix group others would be provided to specific files and folder based on VOLTTRON process requirement. -It is recommended that you use a new :term:`VOLTTRON_HOME` to run VOLTTRON in secure mode. Converting a existing -VOLTTRON instance to secure mode is also possible but would involve some manual changes. Please see the section -`Porting existing volttron home to secure mode`_. +It is recommended that you use a new :term:`VOLTTRON_HOME` to run VOLTTRON in agent isolation mode. Converting a existing +VOLTTRON instance to agent isolation mode is also possible but would involve some manual changes. Please see the section +`Porting existing volttron home to agent isolation mode`_. .. note:: @@ -86,13 +86,13 @@ Setup agents to run using unique users Creating new Agents =================== -In this secure mode, agents will only have read write access to the agent-data directory under the agent install +In agent isolation mode, agents will only have read write access to the agent-data directory under the agent install directory - `VOLTTRON_HOME/agents///.agent-data`. Attempting to write in any other folder under `VOLTTRON_HOME` **will result in permission errors**. -Changes to existing agents in secure mode -========================================= +Changes to existing agents in agent isolation mode +================================================== Due to the above change, **SQL historian has been modified to create its database by default under its agent-data directory** if no path is given in the config file. If providing a path to the database in the config file, please @@ -100,17 +100,18 @@ provide a directory where agent will have write access. This can be an external (`recorded in VOLTTRON_HOME/agents//USER_ID`) has *read, write, and execute* access. -Porting existing VOLTTRON home to secure mode -============================================= +Porting existing VOLTTRON home to agent isolation mode +====================================================== When running `scripts/secure_users_permissions.sh` you will be prompted for a `VOLTTRON_HOME` directory. If this directory exists and contains a volttron config file, the script will update the file locations and permissions of existing VOLTTRON files including installed directories. However this step has the following limitations: -#. **You will NOT be able to revert to insecure mode once the changes are done.** - Once setup is complete, changing the - config file manually to make parameter `secure-agent-users` to `False`, may result inconsistent VOLTTRON behavior +#. **You will NOT be able to revert from agent isolation mode once the changes are done.** - Once setup is complete, + changing the config file manually to make parameter `agent-isolation-mode` to `False`, may result inconsistent + VOLTTRON behavior #. The VOLTTRON process and all agents have to be restarted to take effect -#. **Agents can only to write to its own agent-data dir.** - If your agents writes to any directory outside +#. **Agents can only write to its own agent-data dir.** - If your agents writes to any directory outside `$VOLTTRON_HOME/agents///agent-name.agent-data` move existing files and update the agent configuration such that the agent writes to the `agent-name.agent-data` dir. For example, if you have a `SQLHistorian` which writes a `.sqlite` file to a subdirectory under `VOLTTRON_HOME` that is not diff --git a/docs/source/platform-features/security/non-auth-mode.rst b/docs/source/platform-features/security/non-auth-mode.rst new file mode 100644 index 0000000000..5aaf514c01 --- /dev/null +++ b/docs/source/platform-features/security/non-auth-mode.rst @@ -0,0 +1,33 @@ +.. _non-auth-mode: + +==================================== +Disabling Authentication in VOLTTRON +==================================== + + +There may be some use-cases, such as simulating deployments or agent development, where security is not a consideration. +In these cases, it is possible to disable VOLTTRON's authentication and authorization, stripping away the security +layer from the VIP messagebus and simplifying agent connection and RPC communication. + +Since this is not ideal for any deployment, this can only be done by manually modifying the volttron configuration file. +Within the config file located within VOLTTRON_HOME, the allow-auth option must be added and set to False. + +.. code-block:: console + + [volttron] + message-bus = zmq + vip-address = tcp://127.0.0.1:22916 + instance-name = volttron1 + allow-auth = False + +In simulation environments where multiple volttron instances are used, it is important to ensure that auth settings are +the same across all the instances. + +**Important things to consider:** + + 1. This feature is recommended only for use with simulations and instances that are within a restrictive and + secure network. + 2. When authentication is disabled, there will be no server-key generated for the server and hence the server would + not have any access restrictions + 3. You can still use ssl (https) for your web access + 4. Non auth mode is currently available only for ZMQ diff --git a/docs/source/platform-features/security/volttron-security.rst b/docs/source/platform-features/security/volttron-security.rst index a9175536f8..6109a9c20d 100644 --- a/docs/source/platform-features/security/volttron-security.rst +++ b/docs/source/platform-features/security/volttron-security.rst @@ -29,4 +29,5 @@ Additional documentation related to VIP authentication and authorization is avai key-stores known-hosts-file - running-agent-as-user + agent-isolation-mode + non-auth-mode diff --git a/docs/source/platform-features/web-api/authentication-endpoints.rst b/docs/source/platform-features/web-api/authentication-endpoints.rst new file mode 100644 index 0000000000..15d5e0fad5 --- /dev/null +++ b/docs/source/platform-features/web-api/authentication-endpoints.rst @@ -0,0 +1,113 @@ +.. _Authentication-Endpoints: + +======================== +Authentication EndPoints +======================== + +The VOLTTRON Web API requires the use of bearer tokens to access resources. These tokens +are JSON Web Tokens (JWT) and are provided by the two ``/authenticate`` endpoints (``POST`` +and ``PUT``). Two classes of token are provided to the user: + +- Refresh Tokens: + Refresh tokens are long-lived, and used can be used to obtain a new short-lived access + token. Refresh tokens are obtained by providing a valid username and password to the + ``POST /authenticate`` endpoint. The refresh token SHOULD be kept secure on the + client side, and SHOULD NOT be provided to API call other than to + ``PUT /authenticate``. + +- Access Tokens: + An access token are short-lived tokens required to obtain resources or services provided + by other API endpoints. The access token is obtained using the ``PUT /authenticate`` + endpoint. For convenience, an inital access token is provided by the + ``POST /authenticate`` endpoint as well, but as use the ``POST`` method requires + sending credentials to the server, this should only be used on the first call, with + ``PUT`` being used thereafter to obtain new access tokens until the refresh token has + also expired. + +.. note:: Authentication begins by obtaining a JWT bearer refresh token from the + ``POST /authenticate`` endpoint. An initial access token is provided by this endpoint + as well. Subsequent access tokens are then obtained by providing the refresh token to the + ``PUT /authenticate`` endpoint without resending of credentials. + +---------------------------------------------------------------------------- + +POST /authenticate +================== + +Provide authentication credentials to receive refresh token. + +The user provides a username and password in the request body. If authentication succeeds, +the endpoint will returna a JWT bearer refresh token with user’s claims. An initial access +token is also returned. The refresh token can then be provided to ``PUT /authenticate`` +to obtain new short-lived access tokens, as needed, without sending the username and +password again until the refresh token expires. + +Request: +-------- +- Content Type: ``application/json`` +- Body: + + .. code-block:: JSON + + { + "username": "", + "password": "" + } + +Response: +--------- + +* **With valid username and password:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "refresh_token": "", + "access_token": "" + } + +* **With invalid username and password:** ``401 Unauthorized`` + +-------------- + +PUT /authenticate +================= + +Renew access token. + +The user provides a valid refresh token (provided by ``POST /authenticate``) in the +Authorization header to obtain a fresh access token. A current access token MAY also +be provided, as needed, in the request body to keep open any existing subscriptions. +All subsequent requests to any endpoint should include the new token, and the old +access token should be discarded. + +Request: +-------- + +- Content Type: ``application/json`` +- Authorization: ``BEARER `` +- Body (optional): + + .. code-block:: JSON + + { + "current_access_token": "" + } + +Response: +--------- + +* **With valid refresh token:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "access_token": "" + } + +* **With invalid or mismatched username, password, or token:** + ``401 Unauthorized`` diff --git a/docs/source/platform-features/web-api/files/create_admin_user.png b/docs/source/platform-features/web-api/files/create_admin_user.png new file mode 100644 index 0000000000..1e2fe9425f Binary files /dev/null and b/docs/source/platform-features/web-api/files/create_admin_user.png differ diff --git a/docs/source/platform-features/web-api/files/path_structure.png b/docs/source/platform-features/web-api/files/path_structure.png new file mode 100644 index 0000000000..6f1ead881b Binary files /dev/null and b/docs/source/platform-features/web-api/files/path_structure.png differ diff --git a/docs/source/platform-features/web-api/introduction.rst b/docs/source/platform-features/web-api/introduction.rst new file mode 100644 index 0000000000..0713d0e895 --- /dev/null +++ b/docs/source/platform-features/web-api/introduction.rst @@ -0,0 +1,164 @@ +.. _Web-API: + +====================================== +RESTful Web Interface +====================================== + +The VOLTTRON User Interface API (VUI) is provided by the VOLTTRON Web Service, and is +intended to provide capabilities for building fully featured frontend applications. +The VUI is a RESTful HTTP API for communicating with components of the VOLTTRON system. + +Installation +------------ +The VUI is a built-in part of the VOLTTRON Web Service. To enable to VOLTTRON Web Service, +bootstrap VOLTTRON within the virtual environment using the `--web` option: + +.. code-block:: bash + + python boostrap.py --web + +Additionally, to enable the web service, it is necessary to add a `bind-web-address` key to the +``$VOLTTRON_HOME/config`` file. The value of this key represents address bound to by the platform web service for +handling HTTP(s) requests. Typical addresses would be ``https://:8443`` or +``http://:8080``. + +.. Note:: + If a hostname is used, it must be resolvable for the service to work as expected. + +HTTPS is strongly recommended. When https is used, however, it will also be necessary to include `web-ssl-cert` and +`web-ssl-key` entries in the ``$VOLTTRON_HOME/config`` file: + +.. code-block:: ini + + [volttron] + message-bus = zmq + instance-name = my_instance + vip-address = tcp://127.0.0.1:22916 + bind-web-address = https://:8443 + web-ssl-cert = /home/volttron/.volttron/certificates/certs/platform_web-server.crt + web-ssl-key = /home/volttron/.volttron/certificates/private/platform_web-server.pem + +The ``vcfg`` tool may be used to make the changes to the ``$VOLTTRON_HOME/config`` file by choosing "Y" for the "Is this +instance web enabled?" prompt. To use HTTPS, enter "https" when asked "What is the protocol for this instance?". +The ``vcfg`` tool will then give the option to generate the required SSL cert and key if these do not already exist. + +.. code-block:: console + + (volttron) [volttron@my_instance volttron]$ vcfg + + Your VOLTTRON_HOME currently set to: /home/volttron/.volttron + + Is this the volttron you are attempting to setup? [Y]: y + What type of message bus (rmq/zmq)? [zmq]: + What is the vip address? [tcp://127.0.0.1]: + What is the port for the vip address? [22916]: + What is the name of this instance? [my_instance]: + Is this instance web enabled? [N]: y + What is the protocol for this instance? [http]: https + Web address set to: https://127.0.0.1 + What is the port for this instance? [8080]: 8443 + Would you like to generate a new web certificate? [Y]: y + WARNING! CA certificate does not exist. + Create new root CA? [Y]: y + + Please enter the following details for web server certificate: + Country: [US]: + State: WA + Location: MyTown + Organization: My Organization + Organization Unit: + Created CA cert + Creating new web server certificate. + Is this an instance of volttron central? [N]: n + Will this instance be controlled by volttron central? [Y]: n + Would you like to install a platform historian? [N]: n + Would you like to install a platform driver? [N]: n + Would you like to install a listener agent? [N]: + Finished configuration! + + You can now start the volttron instance. + + If you need to change the instance configuration you can edit + the config file is at /home/volttron/.volttron/config + +Finally, a user must be configured in the ``$VOLTTRON_HOME/web-users.json`` file to allow authentication to the API. +This file can be generated, if it does not exist, by navigating to `bind-web-address`/admin in a web browser and +creating a user and password: + +.. image:: files/create_admin_user.png + +.. code-block:: json + + { + "my_user":{ + "hashed_password":"$argon2id$v=19$m=102400,t=2,p=8$tbb2PgdA6B3jnPOeUwrB+A$yGA2xYOXld+COq4opWbs3Q", + "groups":["admin", "vui"] + } + } + +Users with the "vui" claim in `groups` will now be able to use the API by sending requests +to endpoints with paths on `bind-web-address` beginning with `/vui`. For example, where `bind-web-address` has been +set to ``https://localhost:8443`` the following HTTP request (with a proper +:ref:`HTTP Authorization Header `) may be used to retrieve the root endpoint of the API: + +:: + + GET https://localhost:8443/vui/ + +Access to the API may be disabled by removing "vui" from the list of groups in ``$VOLTTRON_HOME/web-users.json`` for any user which should not have access +to the API. + +Path Structure +--------------- + + +Paths to endpoints consist of alternating constant and variable segments, and are designed +to be readable and discoverable: + +.. image:: files/path_structure.png + +Get requests to non-leaf nodes typically return a `route-options` JSON object which gives additional possible paths +within the API. For instance, a GET request send to the path `/vui` will return: + +.. code-block:: json + + { + "route_options": { + "platforms": "/vui/platforms" + } + } + +Available Endpoints +------------------- + + +Endpoints which are currently provided by the API are described in detail in the +following sections: + +- :ref:`Authentication `: Endpoints for authenticating to the the API. +- :ref:`Platforms `: Endpoints for working with a particular platform. + - :ref:`Agents `: Endpoints for working with agents on the platform. + - :ref:`Configs `: Endpoints for managing the configuration store for agents + on the platform. + - :ref:`Enabled `: Endpoints for enabling, disabling, and setting the + start priority of agents on the platform. + - :ref:`RPC `: Endpoints allowing, discovery, inspection, and calling of + remote procedure calls to agents running on the platform. + - :ref:`Running `: Endpoints for starting and stopping agents on the + platform. + - :ref:`Status `: Endpoints for determining status information for agents + running on the platform. + - :ref:`Tag `: Endpoints for getting, setting, and deleting the tag of agents. + - :ref:`Devices `: Endpoints for getting, setting, and resetting devices on the + platform. + - :ref:`Historians `: Endpoints for querying data from historians on the platform. + - :ref:`Pubsub `: Endpoints for subscribing and publishing to the message bus on the + platform. + - :ref:`Status `: Endpoints for determining and clearing the status of all agents on + the platform. + + .. toctree:: + :hidden: + + Authentication + Platforms diff --git a/docs/source/platform-features/web-api/platform-endpoints.rst b/docs/source/platform-features/web-api/platform-endpoints.rst new file mode 100644 index 0000000000..f9458e2834 --- /dev/null +++ b/docs/source/platform-features/web-api/platform-endpoints.rst @@ -0,0 +1,131 @@ +.. _Platforms-Endpoints: + +=================== +Platforms Endpoints +=================== + + +Platforms endpoints expose functionality associated with specific +VOLTTRON platforms. + +As all functionality of VOLTTRON is the purview of one or another +platform, the /platforms tree forms the core of the VOLTTRON User +Interface API. Other top level partitions of the API consist of +convenience methods which refer to endpoints within /platforms. + +The platforms tree currently provides access to four major categories of endpoint, each of which are described in detail +through the following links: + +* :ref:`Agents `: Endpoints pertaining to a specific agent (e.g. RPC) +* :ref:`Devices `: Endpoints for discovering, getting, and setting data about the current + state of devices on the platform. +* :ref:`Historians `: Endpoints for querying data from historians. +* :ref:`PubSub `: Endpoints for subscription and publication to message bus topics. +* :ref:`Status `: Endpoints for retrieving and clearing status of all agents on the + platform. + +.. attention:: + All endpoints in this tree require authorization using a JWT bearer + token provided by the ``POST /authenticate`` or ``PUT /authenticate`` + endpoints. +-------------------------------------------------------------------------------- + +GET /platforms +============== + +Obtain routes for connected platforms. + +A ``GET`` request to the ``/platforms`` endpoint will return a JSON object containing routes to available platforms. +Available routes are included in a "route_options" object. The keys of the "route_options" object are the name of each +platform which is currently reachable through the API, and the values contain a route to an endpoint for the platform. + +Request: +-------- + +- Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "route_options": { + "": "/platforms/", + "": "/platforms/" + } + } + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` +--------------------------------------------------------------------------------------------------------------------- + +GET /platforms/:platform +======================== + +Obtain routes available for a specific platform. + +A ``GET`` request to the ``/platforms/:platform`` endpoint (where ``:platform`` is the instance name of a specific +platform) will return a JSON object containing routes to endpoints which are available for the requested platform. +Available routes are included in a "route_options" object. The keys of the "route_options" object are the name of each +endpoint which the platform supports, and the values contain a route to that endpoint for this platform. The currently +implemented possibilities include: :ref:`agents `, +:ref:`devices `, :ref:`historians `, +:ref:`pubsub ` and :ref:`status `. + +Request: +-------- + +- Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "route_options": { + "": "/platforms/:platform/", + "": "/platforms/:platform/" + } + } + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +.. toctree:: + :hidden: + + Agents + Configs + Devices + Health + Historians + Pubsub + Status diff --git a/docs/source/platform-features/web-api/platforms/agent-endpoints.rst b/docs/source/platform-features/web-api/platforms/agent-endpoints.rst new file mode 100644 index 0000000000..879e13c04d --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/agent-endpoints.rst @@ -0,0 +1,124 @@ +.. _Platforms-Agents-Endpoints: + +========================== +Platforms Agents Endpoints +========================== + +Platforms Agents endpoints expose functionality associated with applications +running on a VOLTTRON platform. + +Platforms Agents endpoints currently include: + * :ref:`Configs `: Endpoints for managing the configuration store for agents + on the platform. + * :ref:`Enabled `: Endpoints for enabling, disabling, and setting the start + priority of agents on the platform. + * :ref:`Running `: Endpoints for starting and stopping agents on the platform. + * :ref:`RPC `: Endpoints allowing, discovery, inspection, and calling of remote + procedure calls to agents running on the platform. + * :ref:`Status `: Endpoints for determining the status information for an agent + running on the platform. + * :ref:`Tag `: Endpoints for getting, setting, and deleting the tag of agents. + +.. attention:: + All Platforms Agents endpoints require a JWT bearer token obtained through the + ``POST /authenticate`` or ``PUT /authenticate`` endpoints. + +-------------- + +GET /platforms/:platform/agents +=============================== + +Return routes for the agents installed on the platform. + +Accepts a two query parameters: + +* ``agent-state`` accepts one of three string values: + - *"running"* (default): Returns only those agents which are currently running. + - *"installed"*: Returns all installed agents. + - *"packaged"*: Returns filenames of packaged agents on the platform which can be installed. +* ``include-hidden`` (default=False): When True, includes system agents which would not normally be displayed by vctl status. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "route_options": { + "": "/platforms/:platform/agents/:vip_identity", + "": "/platforms/:platform/agents/:vip_identity" + } + } + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +------------------------------------------------------------------------------------------ + +GET /platforms/:platform/agents/:vip-identity +============================================= + +Return routes for the supported endpoints for an agent installed on the platform. +Currently implemented endpoints include :ref:`RPC `. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "route_options": { + "": "/platforms/:platform/agents/:vip_identity/", + "": "/platforms/:platform/agents/:vip_identity/" + } + } + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +.. toctree:: + :hidden: + + ConfigStore + Enabled + Health + RPC + Running + Status + Tag diff --git a/docs/source/platform-features/web-api/platforms/agents/config-endpoints.rst b/docs/source/platform-features/web-api/platforms/agents/config-endpoints.rst new file mode 100644 index 0000000000..ebe03d8e10 --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/agents/config-endpoints.rst @@ -0,0 +1,233 @@ +.. _Platforms-Agents-Configs-Endpoints: + +================================== +Platforms Agents Configs Endpoints +================================== + +Platforms Agents Configs endpoints expose functionality associated with agent configurations stored in the +VOLTTRON Configuration Store. + +.. attention:: + All Platforms Configs endpoints require a JWT bearer token obtained through the + ``POST /authenticate`` or ``PUT /authenticate`` endpoints. + +-------------- + +GET /platforms/:platform/agents/:vip_identity/configs +===================================================== + +Get routes to available configuration files for the specified agent. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response +-------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "route_options": { + ":config_file_name": "/platforms/:platform/agents/:vip_identity/configs/:config_file_name", + "": "/platforms/:platform/agents/:vip_identity/configs/:config_file_name", + } + } + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +-------------------------------------------------------------------------------------------------- + +POST /platforms/:platform/agents/:vip_identity/configs/ +======================================================= + +Save a new configuration file to the config store. + +The file name should be passed in the query parameter file-name. + +The file should match the content type and contents which the VOLTTRON configuration store expects. +The configuration store currently accepts only JSON, CSV, or RAW files. The content type header should match the type +of file being sent (``application/json``, ``text/csv``, or ``text/plain`` respectively). This endpoint +will return 409 Conflict if the configuration file already exists. In this case, the user should use +``PUT /platforms/:platform/agents/:vip_identity/configs/:file_name`` if modification of the existing file is truly +intended. + +Request: +-------- + +* Authorization: ``BEARER `` +* Content Type: ``application/json``, ``text/csv``, or ``text/plain`` +* Body: Contents of configuration file. + +Response +-------- + +* **With valid BEARER token on success:** ``201 Created`` + - Location: /platforms/:platform/agents/:vip_identity/configs/:file_name + - Content Type: ``application/json``, ``text/csv``, or ``text/plain`` + - Body: Contents of the configuration file. + +* **With valid BEARER token on failure:** ``400 Bad Request`` or ``409 Conflict`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +----------------------------------------------------------------------------------------- + +DELETE /platforms/:platform/agents/:vip_identity/configs/ +===================================================================== + +Remove the configuration store for an agent. This endpoint will return ``409 Conflict`` if +the store for this agent is not empty. To remove all existing configurations for an agent from the config store +and delete the store, ``true`` must be passed to the ``all`` query parameter. + +Request: +-------- + +* Authorization: ``BEARER `` +* Query Parameters: + * ``all``: Boolean (default ``false``) + +Response +-------- + +* **With valid BEARER token on success:** ``204 No Content`` + +* **With valid BEARER token on failure:** ``400 Bad Request`` or ``409 Conflict`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +------------------------------------------------------------------------------------------------ + +GET /platforms/:platform/agents/:vip_identity/configs/:config_name +================================================================== + +Get a configuration file for the agent from the config store. + +The configuration store can currently return JSON, CSV, or RAW files. If the Accept header is not set, +the configuration store will return JSON by default. If the client wishes to restrict the type of file received, +it should set the Accept header to the correct MIME type(s) (``application/json``, ``text/csv``, or ``text/plain`` +respectively). + +Request: +-------- + +* Authorization: ``BEARER `` + +Response +-------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json``, ``text/csv``, or ``text/plain`` + - Body: Contents of the configuration file. + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +----------------------------------------------------------------------------------------- + +PUT /platforms/:platform/agents/:vip_identity/configs/:config_name +================================================================== + +Overwrite a configuration file already in the config store. + +The file should match the content type and contents which the VOLTTRON configuration store expects. +The configuration store currently accepts only JSON, CSV, or RAW files. The content type header should match the type +of file being sent (``application/json``, ``text/csv``, or ``text/plain`` respectively). + +Request: +-------- + +* Authorization: ``BEARER `` +* Content Type: ``application/json``, ``text/csv``, or ``text/plain`` +* Body: Contents of configuration file. + +Response +-------- + +* **With valid BEARER token on success:** ``204 No Content`` + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +----------------------------------------------------------------------------------------- + +DELETE /platforms/:platform/agents/:vip_identity/configs/:config_name +===================================================================== + +Remove an existing configuration file for the agent from the config store. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response +-------- + +* **With valid BEARER token on success:** ``204 No Content`` + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +.. toctree:: + :hidden: + + self diff --git a/docs/source/platform-features/web-api/platforms/agents/enabled-endpoints.rst b/docs/source/platform-features/web-api/platforms/agents/enabled-endpoints.rst new file mode 100644 index 0000000000..f104f0ee2e --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/agents/enabled-endpoints.rst @@ -0,0 +1,119 @@ +.. _Platforms-Agents-Enabled-Endpoints: + +================================== +Platforms Agents Enabled Endpoints +================================== + +Platforms Agents Enabled endpoints expose functionality associated with the enabled status of agents on the platform. +This includes determining whether an agent is enabled (and with what start priority), as well as enabling and disabling +the agent. + +.. attention:: + All Platforms Agents Enabled endpoints require a JWT bearer token obtained through the + ``POST /authenticate`` or ``PUT /authenticate`` endpoints. + +-------------- + +GET /platforms/:platform/agents/:agent/enabled +============================================== + +Retrieve the enabled status and priority of the specified agent. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: application/json + - Body: + + .. code-block:: json + + { + "status": true|false, + "priority": int + } +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + + +-------------- + +PUT /platforms/:platform/agents/:agent/enabled +============================================== + +Enable the specified agent. + +Accepts the ``priority`` query parameter to set the start priority of the agent. Allowable prioirties range from 0 to +99. If the priority is not given, the agent will be enabled with a priority of 50. + +Request: +-------- + +* Authorization: ``BEARER `` + + +Response: +--------- + +* **With valid BEARER token on success:** ``204 No Content`` + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +-------------- + +DELETE /platforms/:platform/agents/:agent/enabled +================================================= + +Disable the specified agent. + +Request: +-------- + +* Authorization: ``BEARER `` + + +Response: +--------- + +* **With valid BEARER token on success:** ``204 No Content`` + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +.. toctree:: + :hidden: + + self diff --git a/docs/source/platform-features/web-api/platforms/agents/health-endpoints.rst b/docs/source/platform-features/web-api/platforms/agents/health-endpoints.rst new file mode 100644 index 0000000000..b99195e5da --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/agents/health-endpoints.rst @@ -0,0 +1,55 @@ +.. _Platforms-Agents-Health-Endpoints: + +================================== +Platforms Agents Health Endpoints +================================== + +Platforms Agents Health endpoints expose functionality associated with getting health information for +a single agent running on a VOLTTRON platform. + +.. attention:: + All Platforms Agents Health endpoints require a JWT bearer token obtained through the + ``POST /authenticate`` or ``PUT /authenticate`` endpoints. + +-------------- + +GET /platforms/:platform/agents/:agent/health +============================================== + +Retrieve health information for the specified agent. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: application/json + - Body: + + .. code-block:: json + + { + "status": "", + "context": {"": "agent_specific_values>"}, + "last_updated": "" + } +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +.. toctree:: + :hidden: + + self diff --git a/docs/source/platform-features/web-api/platforms/agents/rpc-endpoints.rst b/docs/source/platform-features/web-api/platforms/agents/rpc-endpoints.rst new file mode 100644 index 0000000000..eb69c6b459 --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/agents/rpc-endpoints.rst @@ -0,0 +1,164 @@ +.. _Platforms-Agents-Rpc-Endpoints: + +============================== +Platforms Agents RPC Endpoints +============================== + + +RPC endpoints expose functionality associated with remote procedure calls to agents running on a VOLTTRON platform. + + +.. attention:: + All RPC endpoints require a JWT bearer token obtained through the ``POST /authenticate`` + or ``PUT /authenticate`` endpoints. + +-------------- + +GET /platforms/:platform/agents/:vip_identity/rpc +================================================= + +Get available remote procedure call endpoints for the specified agent. + +Success will yield a JSON object with available RPC methods as keys and routes for these as values. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "route_options": { + "": "/platforms/:platform/agents/:vip_identity/rpc/:function_name", + "": "/platforms/:platform/agents/:vip_identity/rpc/:function_name" + } + } + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +-------------- + +GET /platforms/:platform/agents/:vip_identity/rpc/:function_name +================================================================ + +Inspect a remote procedure call method. + +.. note:: + + The information for this endpoint is provided by the `inspect` module. Not all information is available for all + RPC methods. If the data is not available, the key will be absent from the response. + + ``kind`` is an enumeration where the values may be `POSITIONAL_OR_KEYWORD`, `POSITIONAL_ONLY`, or + `KEYWORD_ONLY`. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "params": { + "param_name_1": { + "kind": "POSITIONAL_OR_KEYWORD", + "default": "" + }, + "param_name_2": { + "kind": "KEYWORD_ONLY", + "default": null + } + }, + "doc": "Docstring from the method, if available.", + "source": { + "file": "", + "line_number": "" + } + "return": "" + } + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +-------------- + +POST /platforms/:platform/agents/:vip_identity/rpc/:function_name +================================================================= + + +Send an remote procedure call to an agent running on a VOLTTRON platform. + +Parameters provided in the request body are passed as arguments to the RPC method. The return value of an RPC call is +defined by the agent, so this may be a scalar value or another JSON object, for instance a list, dictionary, etc. + +Request: +-------- + +* Content Type: ``application/json`` +* Authorization: ``BEARER `` +* Body: + + .. code-block:: JSON + + { + "": "", + "": "" + } + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json`` + - Body: Any, as defined by the RPC method. + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +.. toctree:: + :hidden: + + self diff --git a/docs/source/platform-features/web-api/platforms/agents/running-endpoints.rst b/docs/source/platform-features/web-api/platforms/agents/running-endpoints.rst new file mode 100644 index 0000000000..fcb76d2584 --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/agents/running-endpoints.rst @@ -0,0 +1,117 @@ +.. _Platforms-Agents-Running-Endpoints: + +================================== +Platforms Agents Running Endpoints +================================== + +Platforms Agents Running endpoints expose functionality associated with the running status of agents on the platform. +This includes determining whether an agent is running as well as starting and stopping the agent. + +.. attention:: + All Platforms Agents Running endpoints require a JWT bearer token obtained through the + ``POST /authenticate`` or ``PUT /authenticate`` endpoints. + +-------------- + +GET /platforms/:platform/agents/:agent/running +============================================== + +Retrieve the running status of the specified agent. + +Request: +-------- + + - Authorization: ``BEARER `` + +Response: +--------- + + * **With valid BEARER token on success:** ``200 OK`` + - Content Type: application/json + - Body: + + .. code-block:: json + + { + "status": true|false + } + * **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + + * **With invalid BEARER token:** ``401 Unauthorized`` + + +-------------- + +PUT /platforms/:platform/agents/:agent/running +============================================== + +Start the specified agent. + +Accepts the ``restart`` query parameter to restart an agent. If the agent is already running, an error will be returned +if the restart parameter is not "true". + +Request: +-------- + + - Authorization: ``BEARER `` + + +Response: +--------- + + * **With valid BEARER token on success:** ``204 No Content`` + + * **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + + * **With invalid BEARER token:** ``401 Unauthorized`` + +-------------- + +DELETE /platforms/:platform/agents/:agent/running +================================================= + +Stop the specified agent. + +Request: +-------- + + - Authorization: ``BEARER `` + + +Response: +--------- + + * **With valid BEARER token on success:** ``204 No Content`` + + * **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + + * **With invalid BEARER token:** ``401 Unauthorized`` + +.. toctree:: + :hidden: + + self diff --git a/docs/source/platform-features/web-api/platforms/agents/status-endpoints.rst b/docs/source/platform-features/web-api/platforms/agents/status-endpoints.rst new file mode 100644 index 0000000000..a2294ed96b --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/agents/status-endpoints.rst @@ -0,0 +1,61 @@ +.. _Platforms-Agents-Status-Endpoints: + +================================= +Platforms Agents Status Endpoints +================================= + +Platforms Agents Status endpoints expose functionality associated with getting status for +a single agent running on a VOLTTRON platform. Only a GET method is currently implemented. + +.. attention:: + All Platforms Agents Status endpoints require a JWT bearer token obtained through the + ``POST /authenticate`` or ``PUT /authenticate`` endpoints. + +-------------- + +GET /platforms/:platform/agents/:agent/status +=============================== + +Get status for a specific agent on the platform. + +Request: +-------- + + - Authorization: ``BEARER `` + +Response: +--------- + + * **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: json + + { + "name": "", + "uuid": "", + "tag": "", + "priority": "", + "running": , + "enabled": , + "pid": , + "exit_code": + } + + * **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + + * **With invalid BEARER token:** ``401 Unauthorized`` + +.. toctree:: + :hidden: + + self diff --git a/docs/source/platform-features/web-api/platforms/agents/tag-endpoints.rst b/docs/source/platform-features/web-api/platforms/agents/tag-endpoints.rst new file mode 100644 index 0000000000..95747de5fa --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/agents/tag-endpoints.rst @@ -0,0 +1,121 @@ +.. _Platforms-Agents-Tag-Endpoints: + +============================== +Platforms Agents Tag Endpoints +============================== + +Platforms Agents Tag endpoints expose functionality associated with tags given to agents on the platform. +Agent tags provide a short name which can be used to identify an agent. + +.. attention:: + All Platforms Agents Tag endpoints require a JWT bearer token obtained through the + ``POST /authenticate`` or ``PUT /authenticate`` endpoints. + +-------------- + +GET /platforms/:platform/agents/:agent/tag +========================================== + +Retrieve the tag of the specified agent. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: application/json + - Body: + + .. code-block:: json + + { + "tag": "" + } +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + + +-------------- + +PUT /platforms/:platform/agents/:agent/tag +========================================== + +Set the tag to an agent installed on the platform. + +Request: +-------- + +* Authorization: ``BEARER `` +* Content Type: ``application/json`` +* Body: + + .. code-block:: json + + { + "tag": "" + } + + +Response: +--------- + +* **With valid BEARER token on success:** ``204 No Content`` +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +-------------- + +DELETE /platforms/:platform/agents/:agent/tag +============================================= + +Remove the tag from an agent installed on a VOLTTRON platform. + +Request: +-------- + +* Authorization: ``BEARER `` + + +Response: +--------- + +* **With valid BEARER token on success:** ``204 No Content`` + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +.. toctree:: + :hidden: + + self diff --git a/docs/source/platform-features/web-api/platforms/config-endpoints.rst b/docs/source/platform-features/web-api/platforms/config-endpoints.rst new file mode 100644 index 0000000000..c42c2ad722 --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/config-endpoints.rst @@ -0,0 +1,249 @@ +.. _Platforms-Configs-Endpoints: + +========================== +Platforms Configs Endpoints +========================== + +Platforms Configs endpoints expose functionality associated with platform configuration files. +These endpoints are for platform-level configurations. Agent configurations are managed by +the :ref:`Platforms Agents Configs ` endpoints. + +.. attention:: + All Platforms Configs endpoints require a JWT bearer token obtained through the + ``POST /authenticate`` or ``PUT /authenticate`` endpoints. + +-------------- + +GET /platforms/:platform/configs +================================ + +Get routes to available configuration files for the specified platform. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "route_options": { + "": "/platforms/:platform/configs/:config_name", + "": "/platforms/:platform/configs/:config_name" + } + } + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +--------------------------------------------------------------- + +POST /platforms/:platform/configs +================================ + +Save a new platform configuration file. + +The file name should be passed in the query parameter file-name. + +The platform configuration files are currently either JSON or INI files. The MIME type of the request will be either +``application/json`` or ``text/plain`` in the case of INI files. This endpoint will return an error if the file already exists. +To update an existing file, use the PUT /platforms/:platform/configs/:file_name endpoint. + +.. warning:: + + Editing platform configuration files can affect the ability of the platform to restart. It is not currently possible + to repair an unstartable platform from the API. Fixing mistakes will require direct access to the device or SSH. + +Request: +-------- + +* Authorization: ``BEARER `` +* Content Type: ``application/json`` or ``text/plain`` +* Query Parameters: + * file-name: The name of the file. If the file will be saved in a subdirectory, ``file-name`` should be a + URL-encoded path to the location of the file relative to the ``VOLTTRON_HOME`` directory. Paths outside of + ``VOLTTRON_HOME`` will be disallowed. +* Body (shown for JSON): + + .. code-block:: + + { + "": , + "": , + } + +Response: +--------- + +* **With valid BEARER token on success:** ``201 Created`` + * Location: ``/platforms/:platform/configs/:file_name`` + * Content Type: ``application/json`` + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +--------------------------------------------------------------- + +GET /platforms/:platform/configs/:config_name +============================================= + +Get a configuration file for the platform (not for an individual agent). + +The platform configuration files are currently either JSON or INI files. The MIME type of the response will be either +``applciation/json`` or ``text/plain`` in the case of INI files. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - `JSON file:` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "": , + "": , + } + + - `INI file:` + - Content Type: ``text/plain`` + - Body: + + .. code-block:: INI + + [section_name] + key1=value1 + key2=value2 + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +--------------------------------------------------------------- + +PUT /platforms/:platform/configs/:config_name +============================================== + +Replace an existing platform configuration file. + +The platform configuration files are currently either JSON, INI files. The MIME type of the response will be either +``applciation/json`` or ``text/plain`` in the case of INI files. This endpoint will return an error if the file does not +already exist. To create a new file, use the ``POST /platforms/:platform/configs`` endpoint. + +If the file is located in a subdirectory, ``:config_name`` should be a URL-encoded path to the location of the file +relative to the ``VOLTTRON_HOME`` directory. Paths outside of ``VOLTTRON_HOME`` will be disallowed. + +.. warning:: + + Editing platform configuration files can affect the ability of the platform to restart. It is not currently possible + to repair an unstartable platform from the API. Fixing mistakes will require direct access to the device or SSH. + +Request: +-------- + +* Authorization: ``BEARER `` +* Content Type: ``application/json`` or ``text/plain`` +* Body (shown for JSON): + + .. code-block:: + + { + "": , + "": , + } + +Response: +--------- + +* **With valid BEARER token on success:** ``201 Created`` + * Location: ``/platforms/:platform/configs/:file_name`` + * Content Type: ``application/json`` + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + +--------------------------------------------------------------- + +DELETE /platforms/:platform/configs/:config_name +================================================ + +Delete an existing platform configuration file. + +If the file is located in a subdirectory, ``:config_name`` should be a URL-encoded path to the location of the file +relative to the ``VOLTTRON_HOME`` directory. Paths outside of ``VOLTTRON_HOME`` will be disallowed. + +.. warning:: + + Editing platform configuration files can affect the ability of the platform to restart. It is not currently possible + to repair an unstartable platform from the API. Fixing mistakes will require direct access to the device or SSH. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``204 No Content`` + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` diff --git a/docs/source/platform-features/web-api/platforms/device-endpoints.rst b/docs/source/platform-features/web-api/platforms/device-endpoints.rst new file mode 100644 index 0000000000..3cc60693e7 --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/device-endpoints.rst @@ -0,0 +1,314 @@ +.. _Platforms-Devices-Endpoints: + +============================ +Platforms Devices Endpoints +============================ + + +Platform Devices endpoints expose functionality associated with devices managed by a VOLTTRON +platform. An optional topic portion of the route path may be used to select specific devices within +the platform. The selection of devices may then be further refined through the use of query parameters, +as described in *Use of Topics*. + +.. admonition:: Use of Topics + + There is no special meaning to most segments of a topic in the VOLTTRON platform. In the context of + devices, however, the final segment in a full topic denotes a point which can be read or actuated. + Partial topics denote some collection of these point resources. For instance, in the caes of a topic hierarchy + organized as ``:campus/:building/:device/:point``, a topic which is complete up to the ``:device`` level would + yield a single device containing one or more points. A topic complete to the ``:building`` level would include a + set of devices, each containing some set of points. The response to any request containing a full topic will + therefore perform a get or set operation, while a partial topic will typically return a list of routes to + further sub-topics (unless it is explicitly requested that an operation be performed on multiple + points). + + Several methods are available to refine the list of topics: + + Topic Wildcards: + The ``-`` character may be used alone to represent any value for a segment: ``/:campus/-/:device`` + will match all devices with the name :device on the :campus in any building. It is not possible to + use ``-`` as a wildcard within a segment containing other characters: ``/campus/foo-bar/:device`` + will only match a building called “foo-bar”, not one called “foobazbar”. + + Topic-Filtering Query Parameters: + - ``tag`` (default=null): + Filter the result by the provided tag. (This requires that the tagging service be + running and configured.) + - ``regex`` (default=null): + Filter the result by the provided regular expression. The raw regular expression + should follow python re syntax, but must be url-encoded within the query-string. + +.. attention:: + All endpoints in this tree require authorization using a JWT bearer access token provided by the + ``POST /authenticate`` or ``PUT /authenticate`` endpoints. +-------------- + +GET /platforms/:platform/devices/:topic +======================================= +Returns a collection of device points and values for a given device topic. + +If no topic, or a parital topic is provided, the output will be a JSON object containing routes to +additional sub-topics matching the provided partial topic. If a full topic is provided, or if the +``read-all`` query parameter is passed, the response will contain data and/or metadata about any +points indicated by the topic. In addition to the ``tag`` and ``regex`` query parameters described +in the *Use of Topics* section above, the following query parameters are accepted: + +* ``read-all`` (default=false): + If true, the response will return entries for every point. These will be a set of JSON objects + with `route`, `writability`, and `value` unless the result is further filtered by the + corresponding query parameters. +* ``routes`` (default=true): + If true, the result will include the route to the points. +* ``writability`` (default=true): + If true, the result will include the writability of the points. +* ``values`` (default=true): + If true, the result will include the value of the points. +* ``config`` (default=false): + If true, the result will include information about the configuration of the point. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: ``application/json`` + - Body: + + + For partial topics, where the ``read-all`` query parameter is false: + This example shows a partial topic, structured as `campus/building/device/point`, + where two segments were provided (the topic provided was `MyCampus/Building1`. + Devices within the building are returned: + + .. code-block:: JSON + + { + "route_options": { + "": "/platforms/:platform/devices/MyCampus/Building1/", + "": "/platforms/:platform/devices/MyCampus/Building1/" + } + } + + + For full topics, or where a partial topic is provided and the ``read-all`` query parameter is true: + This example shows the result of a topic: `MyCampus/Building1/-/Point4`. Note that + the wildcard selects all devices in `Building1` with a point called `Point4`. + ``read-all`` does not need to be ``true`` for this case to get data, as a point segment was provided. + Other query parameters were not provided or were set to their default values. + + .. code-block:: JSON + + { + "MyCampus/Building1/Device1/Point4": { + "route": "/platform/:platform/devices/MyCampus/Building1/Device1/Point4", + "writable": true, + "value": 42 + }, + { + "MyCampus/Building1/Device2/Point4": { + "route": "/platform/:platform/devices/MyCampus/Building1/Device2/Point4", + "writable": false, + "value": 23 + } + } + +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` + + +PUT /platforms/:platform/devices/:topic/ +======================================== + +Sets the value of the specified point and returns its new value and meta-data. In addition to the tag and regex query +parameters described in the Use of Topics section above, the following query parameters are accepted: + +* ``write-all`` (default=false): + If true, the response will write the given value to all points matching the topic. It is *always* necessary to + set write-all=true if more than one point is intended to be written in response to the request. +* ``confirm-values`` (default=false): + If true, the current value of any written points will be read and returned after the write. + +.. warning:: + If an attempt is made to set a point which is not writable, or if multiple points are selected + using a partial topic and/or query parameters and the ``write-all`` query parameter is not set + to ``true``, the response will be ``405 Method Not Allowed``. + +Request: +-------- + +* Authorization: ``BEARER `` + +* Content Type: ``application/json`` + +* Body: + + .. code-block:: JSON + + { + "value": + } + +Response: +--------- + +- **With valid BEARER token on success (confirm-values=false):** ``200 OK`` + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "": { + "route": "/vui/platforms/:platform/devices/:topic", + "set_error": , + "writable": + } + } + +- **With valid BEARER token on success (confirm-values=true):** ``200 OK`` + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "": { + "route": "/vui/platforms/:platform/devices/:topic", + "set_error": , + "writable": , + "value": , + "value_check_error": + } + } + +- **With valid BEARER token if any point is not writable:** + ``405 Method Not Allowed``: + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +- **With valid BEARER token on any other failure:** ``400 Bad Request`` + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +- **With invalid BEARER token:** ``401 Unauthorized`` + +-------------- + +DELETE /platforms/:platform/devices/:topic/ +=========================================== + +Resets the value of the specified point and returns its new value andmeta-data.In addition to the tag and regex query +parameters described in the Use of Topics section above, the following query parameters are accepted: + + * ``write-all`` (default=false): + If true, the response will write the given value to all points matching the topic. It is *always* necessary to + set write-all=true if more than one point is intended to be written in response to the request. + * ``confirm-values`` (default=false): + If true, the current value of any written points will be read and returned after the write. + +.. warning:: + If an attempt is made to set a point which is not writable, or if multiple points are selected + using a partial topic and/or query parameters and the ``write-all`` query parameter is not set + to ``true``, the response will be ``405 Method Not Allowed``. + +.. warning:: + The request will also fail unless all writes are successful, and any points which would otherwise be set will be + reverted to their previous value. + +Request: +-------- + +- Authorization: ``BEARER `` + +Response: +--------- + +- **With valid BEARER token on success (confirm-values=false):** ``200 OK`` + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "": { + "route": "/vui/platforms/:platform/devices/:topic", + "writable": + } + } + +- **With valid BEARER token on success (confirm-values=true):** ``200 OK`` + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "": { + "route": "/vui/platforms/:platform/devices/:topic", + "writable": , + "value": , + "value_check_error": + } + } + +- **With valid BEARER token if any point is not writable:** + ``405 Method Not Allowed``: + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +- **With valid BEARER token on any other failure:** ``400 Bad Request`` + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +- **With invalid BEARER token:** ``401 Unauthorized`` diff --git a/docs/source/platform-features/web-api/platforms/health-endpoints.rst b/docs/source/platform-features/web-api/platforms/health-endpoints.rst new file mode 100644 index 0000000000..6677f81e24 --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/health-endpoints.rst @@ -0,0 +1,55 @@ +.. _Platforms-Health-Endpoints: + +========================== +Platforms Health Endpoints +========================== + +Platforms Health endpoints expose functionality associated with getting health information for +all agents running on a VOLTTRON platform. + +.. attention:: + All Platforms Agents Health endpoints require a JWT bearer token obtained through the + ``POST /authenticate`` or ``PUT /authenticate`` endpoints. + +-------------- + +GET /platforms/:platform/health +============================================== + +Retrieve health information for all agents on the platform. + +Request: +-------- + +* Authorization: ``BEARER `` + +Response: +--------- + +* **With valid BEARER token on success:** ``200 OK`` + - Content Type: application/json + - Body: + + .. code-block:: json + + { + "": { + "peer": "", + "service_agent": true|false, + "connected": "", + "last_heartbeat": "", + "message": "" + }, + ... + } +* **With valid BEARER token on failure:** ``400 Bad Request`` + - Content Type: ``application/json`` + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +* **With invalid BEARER token:** ``401 Unauthorized`` diff --git a/docs/source/platform-features/web-api/platforms/historian-endpoints.rst b/docs/source/platform-features/web-api/platforms/historian-endpoints.rst new file mode 100644 index 0000000000..70b5abd433 --- /dev/null +++ b/docs/source/platform-features/web-api/platforms/historian-endpoints.rst @@ -0,0 +1,240 @@ +.. _Platforms-Historians-Endpoints: + +============================== +Platforms Historians Endpoints +============================== + +Platform Historian endpoints expose functionality related to historians +running on a VOLTTRON platform. + +.. admonition:: Use of Topics + + There is no special meaning to most segments of a topic in the VOLTTRON platform. In the context of + devices, however, the final segment in a full topic denotes a point which can be read or actuated. + Partial topics denote some collection of these point resources. For instance, in the caes of a topic hierarchy + organized as ``:campus/:building/:device/:point``, a topic which is complete up to the ``:device`` level would + yield a single device containing one or more points. A topic complete to the ``:building`` level would include a + set of devices, each containing some set of points. The response to any request containing a full topic will + therefore perform a get or set operation, while a partial topic will typically return a list of routes to + further sub-topics (unless it is explicitly requested that an operation be performed on multiple + points). + + Several methods are available to refine the list of topics: + + Topic Wildcards: + The ``-`` character may be used alone to represent any value for a segment: ``/:campus/-/:device`` + will match all devices with the name :device on the :campus in any building. It is not possible to + use ``-`` as a wildcard within a segment containing other characters: ``/campus/foo-bar/:device`` + will only match a building called “foo-bar”, not one called “foobazbar”. + + Topic-Filtering Query Parameters: + - ``tag`` (default=null): + Filter the result by the provided tag. (This requires that the tagging service be + running and configured.) + - ``regex`` (default=null): + Filter the result by the provided regular expression. The raw regular expression + should follow python re syntax, but must be url-encoded within the query-string. + +.. attention:: + All endpoints in this tree require authorization using a JWT bearer access token provided by the + ``POST /authenticate`` or ``PUT /authenticate`` endpoints. +-------------- + +GET /platforms/:platform/historians +=================================== + +Retrieve routes to historians on the platform, where each historian is listed by its VIP identity. + +Request: +-------- + +- Authorization: ``BEARER `` + +Response: +--------- + +- **With valid BEARER token on success:** ``200 OK`` + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "": "/vui/platforms/:platform/historians/:historian", + "": "/vui/platforms/:platform/historians/:historian" + } + +- **With valid BEARER token on failure:** ``400 Bad Request`` + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +- **With invalid BEARER token:** ``401 Unauthorized`` + +-------------- + +GET /platforms/:platform/historians/:historian +============================================== + +Retrieve routes for an historian. The only currently supported result is the "topics" endpoint. + +Request: +-------- + +- Authorization: ``BEARER `` + +Response: +--------- + +- **With valid BEARER token on success:** ``200 OK`` + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "topics": "/vui/platforms/:platform/historians/:historian/topics" + } + +- **With valid BEARER token on failure:** ``400 Bad Request`` + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "error": "" + } + +- **With invalid BEARER token:** ``401 Unauthorized`` + +-------------- + +GET /platforms/:platform/historians/:historian/topics/:topic +============================================================ + +Query data for a topic. If no topic, or a parital topic is provided, the output will be a JSON object containing routes +to additional sub-topics matching the provided partial topic. If a full topic is provided, or if the read-all query +parameter is passed, the response will contain data and/or metadata about any points indicated by the topic. +In addition to the tag and regex query parameters described in the Use of Topics section above, the following query +parameters are accepted: + +- ``read-all`` (default=false): + If true, the response will return entries for every point. These will be a set of JSON objects + with `route`, `writability`, and `value` unless the result is further filtered by the + corresponding query parameters. + +- ``routes`` (default=true): + If true, the result will include the route to the query. + +- ``values`` (default=true): + If true, the result will include the value of the query. + +Several query parameters may also be used to refine the results: + +- start (default=null): + Datetime of the start of the query. + +- end (default=null): + Datetime of the end of of the query. + +- skip (default=null): + Skip this number of results (for pagination). + +- count (default=null): + Return at maximum this number of results (for pagination). + +- order (default=null): + “FIRST_TO_LAST” for ascending time stamps, “LAST_TO_FIRST” for + descending time stamps. + +.. attention:: + Due to current limitations of the VOLTTRON historian, meta-data about the queried data is only returned when a + single topic has been queried. Where multiple topics are selected, the meta-data field will not be present in the + result. + +Request: +-------- + +- Authorization: ``BEARER `` + +Response: +--------- + +- **With valid BEARER token on success (single topic):** ``200 OK`` + + - Content Type: ``application/json`` + + - Body: + + .. code-block:: JSON + + { + "Campus/Building1/Fake2/SampleWritableFloat1": { + "value": [ + ["", ], + ["", ], + ["", ] + ], + "metadata": { + "units": "", + "type": "", + "tz": "