From 17769272c8c29470056ce943af231ff1c0a2654c Mon Sep 17 00:00:00 2001 From: Romain Hugonnet Date: Mon, 2 Oct 2023 13:13:50 -0800 Subject: [PATCH 1/5] Example codespell pre-commit --- .github/workflows/pre-commit.yml | 15 +++++++++++++++ .pre-commit-config.yaml | 13 +++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 .github/workflows/pre-commit.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 000000000..c57fa18fd --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,15 @@ +name: Linting and formatting (pre-commit) + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - uses: pre-commit/action@v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..04d22cfb6 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: + # Fix common spelling mistakes + - repo: https://github.com/codespell-project/codespell + rev: v2.2.1 + hooks: + - id: codespell + args: [ +# '--ignore-words-list', 'nd,alos,inout', +# '--ignore-regex', '\bhist\b', + '--' + ] + types_or: [python, rst, markdown] + files: ^(clients|docs|packages|platforms|plugins|scripts|targets)/ \ No newline at end of file From dd0fdcdca69740068798c7c420dcd07906f71528 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Tue, 3 Oct 2023 10:56:21 +0000 Subject: [PATCH 2/5] added debug message when getting registered nodes failed --- clients/python/sliderule/sliderule.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/python/sliderule/sliderule.py b/clients/python/sliderule/sliderule.py index ed3db6f6e..ccef286e3 100644 --- a/clients/python/sliderule/sliderule.py +++ b/clients/python/sliderule/sliderule.py @@ -931,7 +931,8 @@ def update_available_servers (desired_nodes=None, time_to_live=None): try: rsps = source("status", parm={"service":"sliderule"}, path="/discovery", silence=True) available_servers = rsps["nodes"] - except FatalError: + except FatalError as e: + logger.debug("Failed to retrieve number of nodes registered: {}".format(e)) available_servers = 0 return available_servers, requested_nodes From b1abaf96516a5ba3c5c1e4a6add0b184386e8d9e Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Tue, 3 Oct 2023 12:05:36 +0000 Subject: [PATCH 3/5] fixed spelling mistakes caught by pre-commit hook --- clients/python/sliderule/earthdata.py | 2 +- clients/python/sliderule/icesat2.py | 18 ---- clients/python/sliderule/ipxapi.py | 18 ++-- clients/python/sliderule/sliderule.py | 2 +- docs/README.md | 6 +- docs/rtd/source/archive/gdal_vrt_benchmark.md | 8 +- docs/rtd/source/getting_started/Install.rst | 2 +- docs/rtd/source/getting_started/SlideRule.rst | 2 +- docs/rtd/source/index.rst | 3 +- .../source/release_notes/release-v1-0-6.md | 2 +- .../source/release_notes/release-v1-4-0.md | 4 +- .../source/release_notes/release-v1-4-2.md | 2 +- .../source/release_notes/release-v1-5-x.md | 2 +- .../source/release_notes/release-v2-1-0.md | 4 +- .../source/release_notes/release-v3-0-2.md | 2 +- .../source/release_notes/release-v4.0.0.md | 6 +- .../tutorials/developer/aws_cli_tips.md | 4 +- .../developer/certbot_instructions.md | 2 +- .../source/tutorials/developer/pub_to_pypi.md | 2 +- .../source/tutorials/user/first_request.md | 2 +- docs/rtd/source/user_guide/GeoParquet.md | 2 +- docs/rtd/source/user_guide/GeoRaster.md | 2 +- docs/rtd/source/user_guide/ICESat-2.rst | 6 +- docs/rtd/source/user_guide/SlideRule.rst | 14 +-- docs/rtd/source/user_guide/Under-the-Hood.rst | 2 +- docs/rtd/source/user_guide/prov-sys.rst | 54 ----------- .../prov-sys/blacklist_refresh_token.md | 78 ---------------- .../source/user_guide/prov-sys/desired_onn.md | 67 ------------- .../user_guide/prov-sys/desired_onnttl.md | 66 ------------- .../source/user_guide/prov-sys/membership.md | 81 ---------------- docs/rtd/source/user_guide/prov-sys/org_nn.md | 57 ------------ .../source/user_guide/prov-sys/org_token.md | 76 --------------- .../user_guide/prov-sys/refresh_token.md | 93 ------------------- .../source/user_guide/prov-sys/remove_onn.md | 59 ------------ packages/arrow/README.md | 2 +- packages/aws/README.md | 2 +- packages/core/LuaEngine.md | 2 +- packages/core/MsgQ.md | 22 ++--- plugins/atlas/README.md | 2 +- plugins/atlas/apps/dfcview.py | 2 +- plugins/atlas/apps/sigview.py | 2 +- 41 files changed, 64 insertions(+), 720 deletions(-) delete mode 100644 docs/rtd/source/user_guide/prov-sys.rst delete mode 100644 docs/rtd/source/user_guide/prov-sys/blacklist_refresh_token.md delete mode 100644 docs/rtd/source/user_guide/prov-sys/desired_onn.md delete mode 100644 docs/rtd/source/user_guide/prov-sys/desired_onnttl.md delete mode 100644 docs/rtd/source/user_guide/prov-sys/membership.md delete mode 100644 docs/rtd/source/user_guide/prov-sys/org_nn.md delete mode 100644 docs/rtd/source/user_guide/prov-sys/org_token.md delete mode 100644 docs/rtd/source/user_guide/prov-sys/refresh_token.md delete mode 100644 docs/rtd/source/user_guide/prov-sys/remove_onn.md diff --git a/clients/python/sliderule/earthdata.py b/clients/python/sliderule/earthdata.py index 7fc9d7184..51bde2006 100644 --- a/clients/python/sliderule/earthdata.py +++ b/clients/python/sliderule/earthdata.py @@ -701,7 +701,7 @@ def tnm(short_name, polygon=None, time_start=None, time_end=datetime.utcnow().st >>> geojson {'type': 'FeatureCollection', 'features': [{'type': 'Feature', 'id': '5eaa4a0582cefae35a21ee8c', 'geometry': {'type': 'Polygon'... ''' - # Flatten polygon (the list must be formated as 'x y, x y, x y, x y, x y', the documentation is incorrect) + # Flatten polygon (the list must be formatted as 'x y, x y, x y, x y, x y', the documentation is incorrect) coord_list = [] for coord in polygon: coord_list.append('{} {}'.format(coord["lon"], coord["lat"])) diff --git a/clients/python/sliderule/icesat2.py b/clients/python/sliderule/icesat2.py index 36edad91e..cd68fdc75 100644 --- a/clients/python/sliderule/icesat2.py +++ b/clients/python/sliderule/icesat2.py @@ -306,8 +306,6 @@ def atl06 (parm, resource): parameters used to configure ATL06-SR algorithm processing (see `Parameters `_) resource: str ATL03 HDF5 filename - asset: str - data source asset (see `Assets `_) Returns ------- @@ -336,10 +334,6 @@ def atl06p(parm, callbacks={}, resources=None, keep_id=False, as_numpy_array=Fal ---------- parms: dict parameters used to configure ATL06-SR algorithm processing (see `Parameters `_) - asset: str - data source asset (see `Assets `_) - version: str - the version of the ATL03 data to use for processing callbacks: dictionary a callback function that is called for each result record resources: list @@ -414,8 +408,6 @@ def atl03s (parm, resource): parameters used to configure ATL03 subsetting (see `Parameters `_) resource: str ATL03 HDF5 filename - asset: str - data source asset (see `Assets `_) Returns ------- @@ -443,10 +435,6 @@ def atl03sp(parm, callbacks={}, resources=None, keep_id=False, height_key=None): ---------- parms: dict parameters used to configure ATL03 subsetting (see `Parameters `_) - asset: str - data source asset (see `Assets `_) - version: str - the version of the ATL03 data to return callbacks: dictionary a callback function that is called for each result record resources: list @@ -584,8 +572,6 @@ def atl08 (parm, resource): parameters used to configure ATL06-SR algorithm processing (see `Parameters `_) resource: str ATL03 HDF5 filename - asset: str - data source asset (see `Assets `_) Returns ------- @@ -614,10 +600,6 @@ def atl08p(parm, callbacks={}, resources=None, keep_id=False, as_numpy_array=Fal ---------- parms: dict parameters used to configure ATL06-SR algorithm processing (see `Parameters `_) - asset: str - data source asset (see `Assets `_) - version: str - the version of the ATL03 data to use for processing callbacks: dictionary a callback function that is called for each result record resources: list diff --git a/clients/python/sliderule/ipxapi.py b/clients/python/sliderule/ipxapi.py index fceb0abf2..821973229 100644 --- a/clients/python/sliderule/ipxapi.py +++ b/clients/python/sliderule/ipxapi.py @@ -44,7 +44,7 @@ # # ICEPYX ATL06 # -def atl06p(ipx_region, parm, asset=icesat2.DEFAULT_ASSET): +def atl06p(ipx_region, parm): """ Performs ATL06-SR processing in parallel on ATL03 data and returns geolocated elevations. The list of granules to be processed is identified by the ipx_region object. @@ -56,8 +56,6 @@ def atl06p(ipx_region, parm, asset=icesat2.DEFAULT_ASSET): icepyx region object defining the query of granules to be processed parm: dict parameters used to configure ATL06-SR algorithm processing (see `Parameters <../user_guide/ICESat-2.html#parameters>`_) - asset: str - data source asset (see `Assets <../user_guide/ICESat-2.html#assets>`_) Returns ------- @@ -65,7 +63,6 @@ def atl06p(ipx_region, parm, asset=icesat2.DEFAULT_ASSET): geolocated elevations (see `Elevations <../user_guide/ICESat-2.html#elevations>`_) """ try: - version = ipx_region.product_version resources = ipx_region.avail_granules(ids=True)[0] except: logger.critical("must supply an icepyx query as region") @@ -74,12 +71,12 @@ def atl06p(ipx_region, parm, asset=icesat2.DEFAULT_ASSET): if ipx_region.extent_type in ('bbox','polygon'): parm.update({'poly': to_region(ipx_region)}) - return icesat2.atl06p(parm, asset, version=version, resources=resources) + return icesat2.atl06p(parm, resources=resources) # # ICEPYX ATL03 # -def atl03sp(ipx_region, parm, asset=icesat2.DEFAULT_ASSET): +def atl03sp(ipx_region, parm): """ Performs ATL03 subsetting in parallel on ATL03 data and returns photon segment data. @@ -89,10 +86,8 @@ def atl03sp(ipx_region, parm, asset=icesat2.DEFAULT_ASSET): ---------- ipx_region: Query icepyx region object defining the query of granules to be processed - parms: dict + parm: dict parameters used to configure ATL03 subsetting (see `Parameters <../user_guide/ICESat-2.html#parameters>`_) - asset: str - data source asset (see `Assets <../user_guide/ICESat-2.html#assets>`_) Returns ------- @@ -100,7 +95,6 @@ def atl03sp(ipx_region, parm, asset=icesat2.DEFAULT_ASSET): ATL03 segments (see `Photon Segments <../user_guide/ICESat-2.html#segmented-photon-data>`_) """ try: - version = ipx_region.product_version resources = ipx_region.avail_granules(ids=True)[0] except: logger.critical("must supply an icepyx query as region") @@ -109,7 +103,7 @@ def atl03sp(ipx_region, parm, asset=icesat2.DEFAULT_ASSET): if ipx_region.extent_type in ('bbox','polygon'): parm.update({'poly': to_region(ipx_region)}) - return icesat2.atl03sp(parm, asset, version=version, resources=resources) + return icesat2.atl03sp(parm, resources=resources) def to_region(ipx_region): """ @@ -123,7 +117,7 @@ def to_region(ipx_region): Returns ------- list - polygon definining region of interest (can be passed into `icesat2` api functions) + polygon defining region of interest (can be passed into `icesat2` api functions) """ if (ipx_region.extent_type == 'bbox'): diff --git a/clients/python/sliderule/sliderule.py b/clients/python/sliderule/sliderule.py index ccef286e3..1b722f47c 100644 --- a/clients/python/sliderule/sliderule.py +++ b/clients/python/sliderule/sliderule.py @@ -233,7 +233,7 @@ def __decode_native(rectype, rawdata): if "PTR" in flags: continue - # get endianess + # get endianness if "LE" in flags: endian = '<' else: diff --git a/docs/README.md b/docs/README.md index d13e1c44b..cca0e1c1b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,7 +32,7 @@ The SlideRule **website** can be built and hosted locally for development purpos $ gem install jekyll bundler ``` - Then go into the `jekyll` directory and install all depedencies. + Then go into the `jekyll` directory and install all dependencies. ```bash $ bundle install @@ -58,7 +58,7 @@ The SlideRule **website** can be built and hosted locally for development purpos ``` Note: docutils version 0.17.x breaks certain formatting in Sphinx (e.g. lists). Therefore it is recommended that docutils version 0.16 be installed. -2. chruby using [Homebrew](https://brew.sh/) (install if neccessary) +2. chruby using [Homebrew](https://brew.sh/) (install if necessary) ```bash $ brew install chruby ruby-install @@ -83,7 +83,7 @@ The SlideRule **website** can be built and hosted locally for development purpos $ gem install jekyll ``` - Then go into the `jekyll` directory and install all depedencies. + Then go into the `jekyll` directory and install all dependencies. ```bash $ bundle install diff --git a/docs/rtd/source/archive/gdal_vrt_benchmark.md b/docs/rtd/source/archive/gdal_vrt_benchmark.md index c123eb58c..54ddbf7b0 100644 --- a/docs/rtd/source/archive/gdal_vrt_benchmark.md +++ b/docs/rtd/source/archive/gdal_vrt_benchmark.md @@ -8,7 +8,7 @@ Test reads elevation value from ArcticDem. POI is lon: -74.60 lat: 82.86 Method used vrt file created from mosaic rasters, version 3.0, 2m, hosted on AWS. The mosaic.vrt file is stored locally on aws dev server at /data/ArcticDem/mosaic.vrt -The actuall raster containing the elevation for POI is: +The actual raster containing the elevation for POI is: /vsis3/pgc-opendata-dems/arcticdem/mosaics/v3.0/2m/34_37/34_37_1_1_2m_v3.0_reg_dem.tif The elevation can be read from the aws file directly: @@ -19,13 +19,13 @@ gdallocationinfo -wgs84 /data/ArcticDem/mosaic.vrt -valonly -74.6 82.86 The test reads the same POI one million times. Gdal library should access the proper raster tile from S3 bucket and use it locally -for all remining elevation reads. +for all remaining elevation reads. In the first implementation the mosaic.vrt is opened, the vrtdataset and vrtband are used to do a direct read via: vrtband->RasterIO(GF_Read, col, row, 1, 1, &elevation, 1, 1, GDT_Float32, 0, 0, 0); col and row are calculated for the mosaic.vrt -This aproach works correctly but performance is very poor. It took almost 170 seconds to do one million reads. Each 100k reads takes almost 17 seconds. +This approach works correctly but performance is very poor. It took almost 170 seconds to do one million reads. Each 100k reads takes almost 17 seconds. Points read: 100000 16988.167999778 Points read: 200000 16991.582000162 @@ -60,7 +60,7 @@ Points read: 1000000 66.580000100657 1000000 points read time: 665.83800013177 msecs -The test was also reapeated with POI beigng different each time read is executed but always within the same raster file. This caused +The test was also repeated with POI beigng different each time read is executed but always within the same raster file. This caused more S3 transactions for GDAL to read many tiles. The performance degraded where the second 'fast' method took almost 2 seconds to read a million POIs which the first method took almost the same amount of time. diff --git a/docs/rtd/source/getting_started/Install.rst b/docs/rtd/source/getting_started/Install.rst index 5f31b7008..91e1dfda8 100644 --- a/docs/rtd/source/getting_started/Install.rst +++ b/docs/rtd/source/getting_started/Install.rst @@ -25,7 +25,7 @@ To install and setup JupyterLab to run the provided example notebooks, you must conda install -c conda-forge jupyterlab Then make sure the conda environment with the `sliderule python` client installed in it is available to use as one of the Python kernels. -To gaurantee that JuypterLab is using the correct Python kernel, you can start JupyterLab from the conda environment with `sliderule python` installed. +To guarantee that JuypterLab is using the correct Python kernel, you can start JupyterLab from the conda environment with `sliderule python` installed. .. code-block:: bash diff --git a/docs/rtd/source/getting_started/SlideRule.rst b/docs/rtd/source/getting_started/SlideRule.rst index f3de305d6..66ceb143c 100644 --- a/docs/rtd/source/getting_started/SlideRule.rst +++ b/docs/rtd/source/getting_started/SlideRule.rst @@ -72,4 +72,4 @@ SlideRule is openly developed on GitHub at https://github.com/ICESat2-SlideRule. Project Information ------------------- -The SlideRule project is funded by NASA's ICESat-2 program and is led by the University of Washington in collaboration with NASA Goddard Space Flight Center. The first public release of SlideRule ocurred in April 2021. Since then we've continued to add new services, new algorithms, and new datasets, while also making improvements to our processing architecture. Looking to the future, we hope to make SlideRule an indespensible component in the analysis of a broad array of Earth Science datasets that help us better understand the planet we call home. +The SlideRule project is funded by NASA's ICESat-2 program and is led by the University of Washington in collaboration with NASA Goddard Space Flight Center. The first public release of SlideRule occurred in April 2021. Since then we've continued to add new services, new algorithms, and new datasets, while also making improvements to our processing architecture. Looking to the future, we hope to make SlideRule an indispensable component in the analysis of a broad array of Earth Science datasets that help us better understand the planet we call home. diff --git a/docs/rtd/source/index.rst b/docs/rtd/source/index.rst index 651c32f85..eb7f27432 100644 --- a/docs/rtd/source/index.rst +++ b/docs/rtd/source/index.rst @@ -72,7 +72,7 @@ SlideRule is openly developed on GitHub at https://github.com/ICESat2-SlideRule. Project Information ------------------- -The SlideRule project is funded by NASA's ICESat-2 program and is led by the University of Washington in collaboration with NASA Goddard Space Flight Center. The first public release of SlideRule ocurred in April 2021. Since then we've continued to add new services, new algorithms, and new datasets, while also making improvements to our processing architecture. Looking to the future, we hope to make SlideRule an indespensible component in the analysis of a broad array of Earth Science datasets that help us better understand the planet we call home. +The SlideRule project is funded by NASA's ICESat-2 program and is led by the University of Washington in collaboration with NASA Goddard Space Flight Center. The first public release of SlideRule occurred in April 2021. Since then we've continued to add new services, new algorithms, and new datasets, while also making improvements to our processing architecture. Looking to the future, we hope to make SlideRule an indispensable component in the analysis of a broad array of Earth Science datasets that help us better understand the planet we call home. @@ -96,7 +96,6 @@ The SlideRule project is funded by NASA's ICESat-2 program and is led by the Uni user_guide/SlideRule.rst user_guide/ICESat-2.rst user_guide/GEDI.rst - user_guide/prov-sys.rst user_guide/H5Coro.md user_guide/GeoParquet.md user_guide/GeoRaster.md diff --git a/docs/rtd/source/release_notes/release-v1-0-6.md b/docs/rtd/source/release_notes/release-v1-0-6.md index f91918d44..d9f5eff99 100644 --- a/docs/rtd/source/release_notes/release-v1-0-6.md +++ b/docs/rtd/source/release_notes/release-v1-0-6.md @@ -23,7 +23,7 @@ Version description of the v1.0.6 release of ICESat-2 SlideRule ## Known Issues -1. Consul-Exit is not working and therefore a node does not dissappear from the service group when it goes down. +1. Consul-Exit is not working and therefore a node does not disappear from the service group when it goes down. ## Getting this release diff --git a/docs/rtd/source/release_notes/release-v1-4-0.md b/docs/rtd/source/release_notes/release-v1-4-0.md index a6ddaf926..7349a6d53 100644 --- a/docs/rtd/source/release_notes/release-v1-4-0.md +++ b/docs/rtd/source/release_notes/release-v1-4-0.md @@ -25,7 +25,7 @@ $ python3 setup.py install * User scripts that use the Python client need to make the following updates: - The `track` keyword argument of **atl03sp**, **atl03s**, **atl06p**, and **atl06** has moved to the `parm` dictionary - - The `block` keywork argument of **atl06p** and **atl03sp** has been removed + - The `block` keyword argument of **atl06p** and **atl03sp** has been removed * User scripts that use the Python client should make the following updates due to deprecated functionality: - The object returned from the **icesat2.toregion** function is now a dictionary instead of a list; the polygon should be accessed via `["poly"]` instead of with a numerical index. If the region of interest contains multiple polygons, the convex hull of those polygons is returned. For compatibility, for this version only, the returned polygon is also accessible at the `[0]` index. @@ -59,7 +59,7 @@ $ python3 setup.py install - Added `rgt`, `cycle`, and `region` request parameters [sliderule-python#27](https://github.com/ICESat2-SlideRule/sliderule-python/issues/27) -- Moved `track` request parameter from being a keyward argument to a member of the `parm` dictionary +- Moved `track` request parameter from being a keyword argument to a member of the `parm` dictionary - The `cnf` parameter default changed to 2; the `maxi` parameter default changed to 5 diff --git a/docs/rtd/source/release_notes/release-v1-4-2.md b/docs/rtd/source/release_notes/release-v1-4-2.md index 17ae7bec0..f70d706ae 100644 --- a/docs/rtd/source/release_notes/release-v1-4-2.md +++ b/docs/rtd/source/release_notes/release-v1-4-2.md @@ -10,7 +10,7 @@ Version description of the v1.4.2 release of ICESat-2 SlideRule. - Updated YAPC algorithm to Jeff's latest specification (05-04-22). The new algorithm is the default version that runs. If the previous algorithm is desired, there is a `version` parameter which is a part of the `yapc` parameter block that can be set to "1", and the original algorithm will execute. Note: the new algorithm runs about three times faster than the original one. -- Updated internal threading hueristic in the Python client: +- Updated internal threading heuristic in the Python client: * client will attempt to throttle the number of concurrent requests to any given processing node * the ***max_workers*** parameter in the `atl06p` and `atl03sp` APIs has been removed; if the calling application must change the number pending requests per node, then there is a new API `sliderule.set_max_pending` that can be called. diff --git a/docs/rtd/source/release_notes/release-v1-5-x.md b/docs/rtd/source/release_notes/release-v1-5-x.md index 1a5058539..e3a0bc4db 100644 --- a/docs/rtd/source/release_notes/release-v1-5-x.md +++ b/docs/rtd/source/release_notes/release-v1-5-x.md @@ -26,7 +26,7 @@ The SlideRule system underwent a gradual architectural shift from a single publi - Networking issues on one of the deployments [#6](https://github.com/ICESat2-SlideRule/sliderule-build-and-deploy/issues/6) -- Node restarted for no discernable reason [#8](https://github.com/ICESat2-SlideRule/sliderule-build-and-deploy/issues/8) +- Node restarted for no discernible reason [#8](https://github.com/ICESat2-SlideRule/sliderule-build-and-deploy/issues/8) - Application Load Balancer adds two seconds of latency [#2](https://github.com/ICESat2-SlideRule/sliderule-build-and-deploy/issues/2) diff --git a/docs/rtd/source/release_notes/release-v2-1-0.md b/docs/rtd/source/release_notes/release-v2-1-0.md index dd6a997b6..2fadb0e97 100644 --- a/docs/rtd/source/release_notes/release-v2-1-0.md +++ b/docs/rtd/source/release_notes/release-v2-1-0.md @@ -26,7 +26,7 @@ Version description of the v2.1.0 release of ICESat-2 SlideRule. - Raster sampling now supports including additional flags with each sample via the `with_flags` option -- Streamlined private cluster setup in the Python client; added `sliderule.scaleout` and udpated behavior of `sliderule.init` function to transparently handle starting and scaling a cluster if desired; [#126](https://github.com/ICESat2-SlideRule/sliderule-python/issues/126) +- Streamlined private cluster setup in the Python client; added `sliderule.scaleout` and updated behavior of `sliderule.init` function to transparently handle starting and scaling a cluster if desired; [#126](https://github.com/ICESat2-SlideRule/sliderule-python/issues/126) - Python client supports TIME8 fields from SlideRule @@ -54,7 +54,7 @@ Version description of the v2.1.0 release of ICESat-2 SlideRule. ## Known Issues -- PhoREAL processing includes some known bugs - the median ground height uses the relative heights instead of absolute heights, and the canopy openess calculation is incorrect +- PhoREAL processing includes some known bugs - the median ground height uses the relative heights instead of absolute heights, and the canopy openness calculation is incorrect ## Getting This Release diff --git a/docs/rtd/source/release_notes/release-v3-0-2.md b/docs/rtd/source/release_notes/release-v3-0-2.md index 94f400509..2d19e26b3 100644 --- a/docs/rtd/source/release_notes/release-v3-0-2.md +++ b/docs/rtd/source/release_notes/release-v3-0-2.md @@ -33,7 +33,7 @@ Version description of the v3.0.2 release of ICESat-2 SlideRule. ## Issues Resolved -- PhoREAL processing bug fixes: the median ground height now uses the absolute heights, and the canopy openess calculation is now correctly implements the standard deviation +- PhoREAL processing bug fixes: the median ground height now uses the absolute heights, and the canopy openness calculation is now correctly implements the standard deviation ## Known Issues diff --git a/docs/rtd/source/release_notes/release-v4.0.0.md b/docs/rtd/source/release_notes/release-v4.0.0.md index 0c7e4d7b5..a142535fe 100644 --- a/docs/rtd/source/release_notes/release-v4.0.0.md +++ b/docs/rtd/source/release_notes/release-v4.0.0.md @@ -29,7 +29,7 @@ This version contains a number of backward-incompatible changes, specifically to * Added new `opendata` plugin that supports sampling the ESA World Cover 10m dataset. -* The Python client supports returning a GeoDataFrame with 3D point geometry. The user must supply a "height" column name in the API call in order to receive back a GeoDataFrame with the 3D point gemetry; note there is a performance impact when selected. (See [#272](https://github.com/ICESat2-SlideRule/sliderule/pull/272)). +* The Python client supports returning a GeoDataFrame with 3D point geometry. The user must supply a "height" column name in the API call in order to receive back a GeoDataFrame with the 3D point geometry; note there is a performance impact when selected. (See [#272](https://github.com/ICESat2-SlideRule/sliderule/pull/272)). * User's can supply their own PROJ pipeline in raster sampling requests. To use this feature, add a "proj_pipeline" string to your request dictionary: ```Python @@ -74,7 +74,7 @@ parms = { "poly": region['poly'], * SlideRule Python bindings (srpy) have been updated to support building them inside a conda managed OS package environment. -* The Python bindings for H5Coro are superceded by the new pure Python implementation effort: [h5coro](https://github.com/ICESat2-SlideRule/h5coro) +* The Python bindings for H5Coro are superseded by the new pure Python implementation effort: [h5coro](https://github.com/ICESat2-SlideRule/h5coro) * Major rewrite of GDAL raster sampling code: new class structure has GdalRaster as a stand-alone class, with RasterObject parenting a GeoRaster and a GeoIndexedRaster for mosaic and strip datasets respectively. @@ -108,7 +108,7 @@ parms = { "poly": region['poly'], * [#293](https://github.com/ICESat2-SlideRule/sliderule/issues/293) - Add ESA WorldCover plugin -* [#295](https://github.com/ICESat2-SlideRule/sliderule/issues/295) - Reqests to TNM for 3DEP 1m is timing out +* [#295](https://github.com/ICESat2-SlideRule/sliderule/issues/295) - Requests to TNM for 3DEP 1m is timing out * [#299](https://github.com/ICESat2-SlideRule/sliderule/issues/299) - Errors reading ArcticDEM mosaic diff --git a/docs/rtd/source/tutorials/developer/aws_cli_tips.md b/docs/rtd/source/tutorials/developer/aws_cli_tips.md index ca04bb2c0..cb57e943d 100644 --- a/docs/rtd/source/tutorials/developer/aws_cli_tips.md +++ b/docs/rtd/source/tutorials/developer/aws_cli_tips.md @@ -16,7 +16,7 @@ When running from EC2, the credentials are automatically provided as a part of y When running from a local machine, there are a few steps you need to take before you can access your S3 bucket. ### Create an Access Key -1. Go to te **IAM** service on the AWS console (Integrated Access Management) +1. Go to the **IAM** service on the AWS console (Integrated Access Management) 2. Navigate to **Access Management --> Users** (using the navigation bar on the left) 3. Select the **Security credentials** tab 4. Click on the **Create access key** button @@ -47,7 +47,7 @@ When running from a local machine, there are a few steps you need to take before } } ``` -4. Using the values provided, export those credentials to your local shell environemnt: +4. Using the values provided, export those credentials to your local shell environment: ```bash $ export AWS_ACCESS_KEY_ID= $ export AWS_SECRET_ACCESS_KEY= diff --git a/docs/rtd/source/tutorials/developer/certbot_instructions.md b/docs/rtd/source/tutorials/developer/certbot_instructions.md index 90d46d6a4..14015dff6 100644 --- a/docs/rtd/source/tutorials/developer/certbot_instructions.md +++ b/docs/rtd/source/tutorials/developer/certbot_instructions.md @@ -2,7 +2,7 @@ 2023-01-18 -These instructions detail the steps neccessary to create the **pem** file needed by HAProxy to perform SSL termination - which means clients can access our web services via **https** instead of **http**. We will first request a certificate from Let's Encrypt for our domain, and then we will format that certificate into a format recognizable by HAProxy. Lastly, we will configure HAProxy to use the certificate to terminate SSL (i.e. support https on the frontend and pass along http to the backend). +These instructions detail the steps necessary to create the **pem** file needed by HAProxy to perform SSL termination - which means clients can access our web services via **https** instead of **http**. We will first request a certificate from Let's Encrypt for our domain, and then we will format that certificate into a format recognizable by HAProxy. Lastly, we will configure HAProxy to use the certificate to terminate SSL (i.e. support https on the frontend and pass along http to the backend). ## Setup workstation with proper credentials diff --git a/docs/rtd/source/tutorials/developer/pub_to_pypi.md b/docs/rtd/source/tutorials/developer/pub_to_pypi.md index dc8f8cc39..08ccfb80c 100644 --- a/docs/rtd/source/tutorials/developer/pub_to_pypi.md +++ b/docs/rtd/source/tutorials/developer/pub_to_pypi.md @@ -12,7 +12,7 @@ The ___sliderule___ PyPI package is located at https://pypi.org/project/sliderul ### 1. Clear Out Existing Outputs -Run the `./clean.sh` script in the base directory of the ___sliderule-python___ repository to remove all existing outputs. Later steps may inadvertantly use them. +Run the `./clean.sh` script in the base directory of the ___sliderule-python___ repository to remove all existing outputs. Later steps may inadvertently use them. ### 2. Build Binary Distribution diff --git a/docs/rtd/source/tutorials/user/first_request.md b/docs/rtd/source/tutorials/user/first_request.md index 0b7e9e5ce..30645e495 100644 --- a/docs/rtd/source/tutorials/user/first_request.md +++ b/docs/rtd/source/tutorials/user/first_request.md @@ -4,7 +4,7 @@ ## Overview -This tutorial walks you through the steps necessary to make your first request to SlideRule. By the end of this tutorial you will have used SlideRule to caclulate and plot elevations over Grand Mesa, Colorado, using ICESat-2 photon cloud data. +This tutorial walks you through the steps necessary to make your first request to SlideRule. By the end of this tutorial you will have used SlideRule to calculate and plot elevations over Grand Mesa, Colorado, using ICESat-2 photon cloud data. **Prerequisites**: This walk-through assumes you are comfortable using `git` and the `conda` Python packaging system. See the [installation](../../getting_started/Install.html) instructions in the reference documentation for details on other methods of installation. diff --git a/docs/rtd/source/user_guide/GeoParquet.md b/docs/rtd/source/user_guide/GeoParquet.md index bd9a7456a..49a098381 100644 --- a/docs/rtd/source/user_guide/GeoParquet.md +++ b/docs/rtd/source/user_guide/GeoParquet.md @@ -6,7 +6,7 @@ Apache Parquet is an open source, column-oriented data file format designed for efficient data storage and retrieval. It has strong support in Python/DataFrames and is a popular data format for big data analytics. See https://parquet.apache.org/ for more details. -GeoParquet is a meta data and file organization covention overlayed on top of the Parquet format that provides a standard way of storing geospatial information inside Parquet files. It has growing support in geospatial analysis tools, including Python/GeoDataFrames. See https://geoparquet.org/ for more details. +GeoParquet is a meta data and file organization convention overlaid on top of the Parquet format that provides a standard way of storing geospatial information inside Parquet files. It has growing support in geospatial analysis tools, including Python/GeoDataFrames. See https://geoparquet.org/ for more details. ## Overview diff --git a/docs/rtd/source/user_guide/GeoRaster.md b/docs/rtd/source/user_guide/GeoRaster.md index 00038cdc8..75a97f7d4 100644 --- a/docs/rtd/source/user_guide/GeoRaster.md +++ b/docs/rtd/source/user_guide/GeoRaster.md @@ -57,7 +57,7 @@ Sampling rasters is controlled by the `samples` field of a request's parameter d ``` The `mosaic` and `strips` dictionary keys are provided by the user to identify the data returned back to the user in the GeoDataFrame - they can be anything the user wants as long as it unique. The `asset` dictionary keys are used to identify which asset in SlideRule's asset directory is to be sampled. The dictionary keys that follow all control how the raster is to be sampled. -SlideRule supports various algorithms for sampling rasters: NearestNeighbour, Bilinear, Cubic, CubicSpline, Lanczos, Average, Mode, and Gauss. These are the algorithms supplied by GDAL and exposed to users through the SlideRule APIs. In addition to these algorithms, SlideRule also provides the ability to calculate zonal statistics for a given geolocation: count (number of pixels), minimum, maximum, mean, median, standard devation, and median absolute deviation. Lastly, depending on the dataset, SlideRule provides custom bands that can be derived using other bands in the raster file. For example, the HLS dataset (`landsat-hls`) provides the following three custom bands: "NDSI", "NDVI", "NDWI". +SlideRule supports various algorithms for sampling rasters: NearestNeighbour, Bilinear, Cubic, CubicSpline, Lanczos, Average, Mode, and Gauss. These are the algorithms supplied by GDAL and exposed to users through the SlideRule APIs. In addition to these algorithms, SlideRule also provides the ability to calculate zonal statistics for a given geolocation: count (number of pixels), minimum, maximum, mean, median, standard deviation, and median absolute deviation. Lastly, depending on the dataset, SlideRule provides custom bands that can be derived using other bands in the raster file. For example, the HLS dataset (`landsat-hls`) provides the following three custom bands: "NDSI", "NDVI", "NDWI". Prior to sampling a raster, SlideRule provides ways to select and filter which rasters are sampled in a dataset. These include filtering based on time, and file name pattern matching. diff --git a/docs/rtd/source/user_guide/ICESat-2.rst b/docs/rtd/source/user_guide/ICESat-2.rst index df3fcc093..95df2cb4b 100644 --- a/docs/rtd/source/user_guide/ICESat-2.rst +++ b/docs/rtd/source/user_guide/ICESat-2.rst @@ -135,9 +135,9 @@ The PhoREAL vegetation algorithm, developed at the University of Texas at Austin To enable PhoREAL functionality, the ``"phoreal"`` parameter must be populated in the request dictionary. * ``"phoreal"``: dictionary of rasters to sample - - ``"binsize"``: size of the veritical photon bin in meters + - ``"binsize"``: size of the vertical photon bin in meters - ``"geoloc"``: algorithm to use to calculate the geolocation (latitude, longitude, along-track distance, and time) of each custom length PhoREAL segment; "mean" - takes the average value across all photons in the segment; "median" - takes the median value across all photons in the segment; "center" - takes the halfway value calculated by the average of the first and last photon in the segment - - ``"use_abs_h"``: boolean whether the absoluate photon heights are used instead of the normalized heights + - ``"use_abs_h"``: boolean whether the absolute photon heights are used instead of the normalized heights - ``"send_waveform"``: boolean whether to send to the client the photon height histograms in addition to the vegetation statistics - ``"above_classifier"``: boolean whether to use the ABoVE photon classifier when determining top of canopy photons @@ -269,7 +269,7 @@ Most of the time, the photon extents are kept internal to SlideRule and not retu Even though this offloads processing on the server, the API calls can take longer since more data needs to be returned to the user, which can bottleneck over the network. Photon extents are returned as GeoDataFrames where each row is a photon. Each extent represents the data that the ATL06 algorithm uses to generate a single ATL06 elevation. -When the step size is shorter than the length of the extent, the extents returned overlap eachother which means that each photon is being returned multiple times and will be duplicated in the resulting GeoDataFrame. +When the step size is shorter than the length of the extent, the extents returned overlap each other which means that each photon is being returned multiple times and will be duplicated in the resulting GeoDataFrame. The GeoDataFrame for each photon extent has the following columns: diff --git a/docs/rtd/source/user_guide/SlideRule.rst b/docs/rtd/source/user_guide/SlideRule.rst index 973684857..fb13a0075 100644 --- a/docs/rtd/source/user_guide/SlideRule.rst +++ b/docs/rtd/source/user_guide/SlideRule.rst @@ -18,14 +18,14 @@ Using SlideRule means issuing HTTP requests to SlideRule's endpoints to perform #. Determining *which data* is available in that area of interest #. Describing *what processing* you want to perform #. Issuing the processing *request* -#. Parsing the *response* data into a useable data structure +#. Parsing the *response* data into a usable data structure The SlideRule Python Client helps make the above steps a lot easier by providing a user-friendly interface for accomplishing typical workflows. -#. The Python client supports an *area of interest* being defined in mulitple ways - as a dictionary of latitude and longitudes, a list of coordinates, a GeoJSON file, a Shapefile, or a GeoDataFrame - and then representing the area of interest in standard formats accepted by SlideRule and other NASA web services. +#. The Python client supports an *area of interest* being defined in multiple ways - as a dictionary of latitude and longitudes, a list of coordinates, a GeoJSON file, a Shapefile, or a GeoDataFrame - and then representing the area of interest in standard formats accepted by SlideRule and other NASA web services. #. The Python client queries NASA's Common Metadata Repository (CMR) system automatically for the user and populates the requests to SlideRule with the available science data pertinent to the user's request. #. The Python client allows users to define their processing parameters as Python dictionaries, and make requests to SlideRule using Python functions. -#. The Python client handles the HTTPS connection to the SlideRule servers as well as any necessary authentation requests to the SlideRule Provisioning System when private clusters are being used. +#. The Python client handles the HTTPS connection to the SlideRule servers as well as any necessary authentication requests to the SlideRule Provisioning System when private clusters are being used. #. The Python client parses the response data from SlideRule and presents a standard GeoDataFrame result back to the user. 3. De-serialization @@ -197,7 +197,7 @@ To request raster sampling, the ``"samples"`` parameter must be populated as a d - ``"algorithm"``: algorithm to use to sample the raster; the available algorithms for sampling rasters are: NearestNeighbour, Bilinear, Cubic, CubicSpline, Lanczos, Average, Mode, Gauss - ``"radius"``: the size of the kernel in meters when sampling a raster; the size of the region in meters for zonal statistics - ``"zonal_stats"``: boolean whether to calculate and return zonal statistics for the region around the location being sampled - - ``"with_flags"``: boolean whether to include auxillary information about the sampled pixel in the form of a 32-bit flag + - ``"with_flags"``: boolean whether to include auxiliary information about the sampled pixel in the form of a 32-bit flag - ``"t0"``: start time for filtering rasters to be sampled (format %Y-%m-%dT%H:%M:%SZ, e.g. 2018-10-13T00:00:00Z) - ``"t1"``: stop time for filtering rasters to be sampled (format %Y-%m-%dT%H:%M:%SZ, e.g. 2018-10-13T00:00:00Z) - ``"substr"``: substring filter for rasters to be sampled; the raster will only be sampled if the name of the raster includes the provided substring (useful for datasets that have multiple rasters for a given geolocation to be sampled) @@ -216,7 +216,7 @@ To request raster sampling, the ``"samples"`` parameter must be populated as a d } } -The output returned in the DataFrame can take two differnt forms depending on the nature of the data requested. +The output returned in the DataFrame can take two different forms depending on the nature of the data requested. (1) If the raster being sampled includes on a single value for each latitude and longitude, then the data returned will be of the form {key}.value, {key}.time, {key}.file_id, {key}.{zonal stat} where the zonal stats are only present if requested. @@ -226,7 +226,7 @@ The output returned in the DataFrame can take two differnt forms depending on th 5.6 Output parameters ------------------------ -By default, SlideRule returns all results in a native (i.e. custom to SlideRule) format that is streamed back to the client and used by the client to construct a GeoDataFrame that is returned to the user. Using the parameters below, SlideRule can build the entire DataFrame of the results on the servers, in one of a few different formats (currently, only GeoParquet `GeoParquet <./GeoParquet.html>`_ is supported), and return the results as a file that is streamed back to the client and reconstructed by the client. To control thie behavior, the ``"output"`` parameter is used. +By default, SlideRule returns all results in a native (i.e. custom to SlideRule) format that is streamed back to the client and used by the client to construct a GeoDataFrame that is returned to the user. Using the parameters below, SlideRule can build the entire DataFrame of the results on the servers, in one of a few different formats (currently, only GeoParquet `GeoParquet <./GeoParquet.html>`_ is supported), and return the results as a file that is streamed back to the client and reconstructed by the client. To control this behavior, the ``"output"`` parameter is used. Optionally, SlideRule supports writing the output to an S3 bucket instead of streaming the output back to the client. In order to enable this behavior, the ``"output.path"`` field must start with "s3://" followed by the bucket name and object key. For example, if you wanted the result to be written to a file named "grandmesa.parquet" in your S3 bucket "mybucket", in the subfolder "maps", then the output.path would be "s3://mybucket/maps/grandmesa.parquet". When writing to S3, it is required by the user to supply the necessary credentials. This can be done in one of two ways: (1) the user specifies an "asset" supported by SlideRule for which SlideRule already maintains credentials; (2) the user specifies their own set of temporary aws credentials. @@ -473,7 +473,7 @@ geo - polar y coordinate to project onto a spherical coordinate system - *optional* * - **span** - - a box defined by a lower left latitude/longitude pair, and an upper right lattitude/longitude pair + - a box defined by a lower left latitude/longitude pair, and an upper right latitude/longitude pair - *optional* * - **span1** - a span used for intersection with the span2 diff --git a/docs/rtd/source/user_guide/Under-the-Hood.rst b/docs/rtd/source/user_guide/Under-the-Hood.rst index 2f1f011c3..969621789 100644 --- a/docs/rtd/source/user_guide/Under-the-Hood.rst +++ b/docs/rtd/source/user_guide/Under-the-Hood.rst @@ -117,7 +117,7 @@ How Does SlideRule Work? Figure 2: SlideRule's Processing Workflow -The University of Washington’s deployment of SlideRule runs in AWS us-west-2 and consists of a set of public and prviate EC2 instances that have access to NASA’s Cumulus datasets in S3, and NASA’s CMR system. A user Python script can be running anywhere as long as it has access to the internet. When using SlideRule’s Python client, a processing request from a user script has three primary stages: +The University of Washington’s deployment of SlideRule runs in AWS us-west-2 and consists of a set of public and private EC2 instances that have access to NASA’s Cumulus datasets in S3, and NASA’s CMR system. A user Python script can be running anywhere as long as it has access to the internet. When using SlideRule’s Python client, a processing request from a user script has three primary stages: :1. Authentication: If accessing the public SlideRule service, authentication is unnecessary. But, if accessing a private cluster, the user authenticate themselves to the provisioning system which associates a profile with them that identifies what they can and cannot do. diff --git a/docs/rtd/source/user_guide/prov-sys.rst b/docs/rtd/source/user_guide/prov-sys.rst deleted file mode 100644 index e96a59244..000000000 --- a/docs/rtd/source/user_guide/prov-sys.rst +++ /dev/null @@ -1,54 +0,0 @@ - -Provisioning System -=================== -The provisioning system allows different organizations to share a given amazon web services account and to maintain independent clusters of sliderule nodes. The provisioning sytem provides the owners of organizations the ability to control access to their organization's cluster via a privilaged 'owner' account. - -Regular users of sliderule can create a regular account and request membership to organizations. The owner of the organization can accept the membership and make the user an *active* member or ignore the request. Owners can make members inactive to temporarily deny access to the cluster. Active members can obtain an access token that provides access to the system for 24 hours. Active members can request node capacity for the duration of the token lifetime or with a provided "time to live". - - -**Owner accounts:** - 1. deploy or shutdown the organization cluster - 2. accept users' membership requests - 3. activate/deactivate memberships - 4. specify the minimum and maximum number of nodes that can be deployed on the organization cluster - 5. can view organization's cluster account and budget forecast - -**Regular accounts:** - 1. can request membership to organizations - 2. can view cluster account balance and status - - -**Endpoints:** - -The provisioning system provides endpoints that allow regular users to request server resources to be allocated for their python client to use throughout a given session. Users share sliderule nodes. All requests for node capacity have an expiration. Requests from all users are combined so that each and every users' requests for the minimum number of nodes required are honored. When all the node capacity requests have expired the provisioning system will automatically reduce the number of nodes in the cluster to the minimum it is configured for. Organization cluster have two nodes (a load balancer and a monitor) that are always active even if the worker nodes is set to zero. The load balancer node can take several minutes to start. However, the organization cluster can be configured to destroy the overhead nodes if the minimum number of nodes is zero or to keep them active for faster deployment. The organization cluster can also be configured to deploy automatically (if the overhead nodes were configured to be destroyed) upon the first node capacity request. When the load balancer has to be started it will take longer to make the cluster completely available to the users' client. However this tradeoff can save money if the organization cluster is expected to be idle for long periods of time. - -All endpoints require some kind of authentication. - -.. toctree:: - :caption: Login: Obtain a token pair (requires username/password) - :maxdepth: 1 - - POST Obtain an access/refresh token pair - -These endpoints provide node capacity management as requested by a member. -An access token is provided in the header of the request: - -.. toctree:: - :caption: Manage Cluster node capacity (require an access token) - :maxdepth: 1 - - GET min-cur-max number of nodes - PUT minimum node capacity with token expiration - POST minimum node capacity with TTL expiration - PUT remove pending node capacity requests - GET membership status - -These endpoints require a refresh token. A refresh token is used to obtain a new access token without having to provide username/password credentials. It is a way to extend the session beyond the default 24 hours. - -.. toctree:: - :caption: Misc endpoints (require a refresh token) - :maxdepth: 1 - - POST Refresh an access token - POST Blacklist a refresh token - diff --git a/docs/rtd/source/user_guide/prov-sys/blacklist_refresh_token.md b/docs/rtd/source/user_guide/prov-sys/blacklist_refresh_token.md deleted file mode 100644 index 4a792e1dd..000000000 --- a/docs/rtd/source/user_guide/prov-sys/blacklist_refresh_token.md +++ /dev/null @@ -1,78 +0,0 @@ -# Blacklist a refresh token - -Make a given refresh token invalid - -**URL** : `ps.slideruleearth.io/api/token/blacklist/` - -**Method** : `POST` - -**Auth required** : Yes. A valid refresh token must be provided in the body - -**Permissions required** : None - -**Data constraints** - -A valid, unused, unexpired refresh token. - -**Curl example** - -``` -curl -X POST \ - -H Content-Type: application/json \ - -d {"refresh": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoicmVmcmVzaCIsImV4cCI6MTY3MDA3MzEyNSwiaWF0IjoxNjY3NDgxMTI112345678901234567890MzExZjE0M2VlODUyYjE4YWEyOWRkMzg3MCIsIm9yZ19uYW1lIjoiVW9mTURUZXN0IiwidXNlcl9uYW1lIjoiY2V1Z2FydGVibGFpciIsInVzZXJfaWQiOjN9.NW4mJUsm48oNK0iRqojNEiC0fyqm3GANAjcii3T41pE"} - https://ps.slideruleearth.io/api/token/blacklist/ -``` - -## Success Responses - -**Condition** : user in token claim is a valid active member of the organization, the refresh token in the body is a valid refresh token for organization. - -**Code** : `200 OK` - -**Data Response** - -```json -{} -``` -**Note**: -The response is a Null json string - -**Condition** : user in token claim is member but NOT an valid ***active*** member of the organization, the refresh token in the body is a valid token for the organization. - -**Code** : `200 OK` - -**Data Response** -```json -{"active":false} -``` -**Notes** - -## Error Responses - -**Condition** : typo in the URL, i.e. invalid url - -**Code** : `404 NOT FOUND` - -**Content** : `{The requested resource was not found on this server.}` - -**--- Or ---** - -**Condition** : The token is invalid - -**Code** : `403 Forbidden` - - **Content** : - ```json - "detail": "Given token not valid for any token type" - ``` - -**--- Or ---** - -**Condition** : The refresh token is blacklisted (i.e. has already been used) - -**Code** : `403 Forbidden` - - **Content** : - ```json - { "detail": "Token is blacklisted", "code": "token_not_valid" } - ``` \ No newline at end of file diff --git a/docs/rtd/source/user_guide/prov-sys/desired_onn.md b/docs/rtd/source/user_guide/prov-sys/desired_onn.md deleted file mode 100644 index 73c2f6d64..000000000 --- a/docs/rtd/source/user_guide/prov-sys/desired_onn.md +++ /dev/null @@ -1,67 +0,0 @@ -# Desired Org Number of Nodes - -Request that a minimum number of nodes be provisioned for the cluster of the organization specified in the url for the duration of the provided token. - -**URL** : `ps.slideruleearth.io/api/desired_org_num_nodes///` - -**Method** : `PUT` - -**Auth required** : Yes. A valid access token for the organization must be included in the Authorization Header Bearer field - -**Permissions required** : User is an active member of the organization (Note: the organization is specified as a claim in the provided token) - -**Data constraints** - -The desired num nodes must be greater than or equal the minimum and less than or equal to the max num nodes as specified by the cluster configuration (which is controlled by the owner of the cluster). - -**Curl example** - -``` -curl -X PUT \ - -H '"Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNjY4MTgxOTYzLCJpYXQiOjE2NjgwOTU1NjMsImp0aSI6ImJmYjIxMmExMzU0ZjQ4NGFhY2E2NmVjYWJmMmE3Mjg4Iiwib3JnX25hbWUiOiJVb2ZNRFRlc3QiLCJ1c2VyX25hbWUiOiJjZXVnYXJ0ZWJsYWlyIiwidXNlcl9pZCI6M30.nl1ACnWcoROhZ7K_HKOCOVfbqiDPBzmPdEPnAdb2vxk" \ -https://ps.slideruleearth.io/api/desired_org_num_nodes/sliderule/7/ -``` -## Success Responses - -**Condition** : user is an valid active member of the organization, the token in the Bearer field is a valid token for organization, the desired number of nodes is within the min/max range. The cluster is already deployed. - -**Code** : `200 OK` - -**Data Response** -```json -{"status":"QUEUED","msg":"Queued request no update yet current num_node reqs:\n[{org},{username},1,2022-11-29 13:03:08 UTC}]","error_msg":""} -``` - -**Notes** - -The response has a status field which indicated that the request was queued. The response has a msg field that inidicated that no update was initiated (because the cluster already had one node deployed). The msg field also indicates that list of queued requests. The queued requests contain the name of the cluster, the user that made the request, the number of nodes requested, and the expiration of the request. The response also has an error_msg field which is blank when the request is successful - -## Error Responses - -**Condition** : typo in the URL, i.e. invalid url - -**Code** : `404 NOT FOUND` - -**Content** : `{The requested resource was not found on this server.}` - -**--- Or ---** - -**Condition** : If a token for the wrong organization was provided - -**Code** : `400 BAD REQUEST` - -**Content** : -```json -{"status":"FAILED","error_msg":"Token with wrong organization"} -``` - **--- Or ---** - - **Condition** : desired number of nodes is out of range - -**Code** : `400 BAD REQUEST` - - **Content** : - ```json - {"status":"FAILED","msg":"","error_msg":"desired_num_nodes:10 greater than max:3"} - ``` - diff --git a/docs/rtd/source/user_guide/prov-sys/desired_onnttl.md b/docs/rtd/source/user_guide/prov-sys/desired_onnttl.md deleted file mode 100644 index 6c89417fc..000000000 --- a/docs/rtd/source/user_guide/prov-sys/desired_onnttl.md +++ /dev/null @@ -1,66 +0,0 @@ -# Desired Org Number of Nodes with Time to Live - -Request that a minimum number of nodes be provisioned for the cluster of the organization specified in the url for the the specified time to live (in minutes). - -**URL** : `ps.slideruleearth.io/api/desired_org_num_nodes////` - -**Method** : `POST` - -**Auth required** : Yes. A valid access token for the organization must be included in the Authorization Header Bearer field - -**Permissions required** : User is an active member of the organization (Note: the organization is specified as a claim in the provided token) - -**Data constraints** - -The desired num nodes must be greater than or equal to the minimum and less than or equal to the max num nodes as specified by the cluster configuration (which is controlled by the owner of the cluster). The TTL must be greater than or equal to 15 minutes - -**Curl example** -``` -curl -X PUT \ - -H '"Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNjY4MTgxOTYzLCJpYXQiOjE2NjgwOTU1NjMsImp0aSI6ImJmYjIxMmExMzU0ZjQ4NGFhY2E2NmVjYWJmMmE3Mjg4Iiwib3JnX25hbWUiOiJVb2ZNRFRlc3QiLCJ1c2VyX25hbWUiOiJjZXVnYXJ0ZWJsYWlyIiwidXNlcl9pZCI6M30.nl1ACnWcoROhZ7K_HKOCOVfbqiDPBzmPdEPnAdb2vxk" \ -https://ps.slideruleearth.io/api/desired_org_num_nodes/sliderule/7/15/ -``` -## Success Responses - -**Condition** : user is an valid active member of the organization, the token in the Bearer field is a valid token for organization, the desired number of nodes is within the min/max range. The cluster is already deployed with one node. - -**Code** : `200 OK` - -**Data Response** -```json -{"status":"QUEUED","msg":"Queued request, no update yet; current num_node reqs:[{org},{username},1,2022-11-28 14:09:30 UTC},{org},{username},1,2022-11-29 13:03:08 UTC}]","error_msg":""} -``` - -**Notes** - -The response has a status field which indicated that the request was queued. The response has a msg field that inidicated that no update was initiated (because the cluster already has the desired num nodes deployed). The msg field also indicates that list of queued requests. The queued requests contain the name of the cluster, the user that made the request, the number of nodes requested, and the expiration of the request. The response also has an error_msg field which is blank when the request is successful - -## Error Responses - -**Condition** : typo in the URL, i.e. invalid url - -**Code** : `404 NOT FOUND` - -**Content** : `{The requested resource was not found on this server.}` - -**--- Or ---** - -**Condition** : If a token for the wrong organization was provided - -**Code** : `400 BAD REQUEST` - -**Content** : -```json -{"status":"FAILED","error_msg":"Token with wrong organization"} -``` - **--- Or ---** - - **Condition** : desired number of nodes is out of range - -**Code** : `400 BAD REQUEST` - - **Content** : - ```json -{"status":"FAILED","msg":"","error_msg":"desired_num_nodes:10 greater than max:3"} -``` - diff --git a/docs/rtd/source/user_guide/prov-sys/membership.md b/docs/rtd/source/user_guide/prov-sys/membership.md deleted file mode 100644 index 2e6efd1d7..000000000 --- a/docs/rtd/source/user_guide/prov-sys/membership.md +++ /dev/null @@ -1,81 +0,0 @@ -# Obtain membership status of access token's user - -Request the membership status of the user and organization in the provided access token. - -**URL** : `ps.slideruleearth.io/api/membership_status//` - -**Method** : `GET` - -**Auth required** : Yes. A valid access token for the organization must be included in the Authorization Header Bearer field - -**Permissions required** : User is an active member of the organization (Note: the organization is specified as a claim in the provided token) - -**Data constraints** - -N/A - -**Curl Example** -``` -curl -X GET \ - -H '"Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNjY4MTgxOTYzLCJpYXQiOjE2NjgwOTU1NjMsImp0aSI6ImJmYjIxMmExMzU0ZjQ4NGFhY2E2NmVjYWJmMmE3Mjg4Iiwib3JnX25hbWUiOiJVb2ZNRFRlc3QiLCJ1c2VyX25hbWUiOiJjZXVnYXJ0ZWJsYWlyIiwidXNlcl9pZCI6M30.nl1ACnWcoROhZ7K_HKOCOVfbqiDPBzmPdEPnAdb2vxk" \ -https://ps.slideruleearth.io/api/membership_status/sliderule/ -``` -## Success Responses - -**Condition** : user in token claim is a valid active member of the organization, the token in the Bearer field is a valid token for organization. - -**Code** : `200 OK` - -**Data Response** -```json -{"active":true} -``` - -**Condition** : user in token claim is member but NOT an valid ***active*** member of the organization, the token in the Bearer field is a valid token for organization. - -**Code** : `200 OK` - -**Data Response** -```json -{"active":false} -``` -**Notes** - -## Error Responses: - -**Condition** : typo in the URL, i.e. invalid url - -**Code** : `404 NOT FOUND` - -**Content** : `{The requested resource was not found on this server.}` - -**--- Or ---** - -**Condition** : If a token for the wrong organization was provided - -**Code** : `400 BAD REQUEST` - -**Content** : -```json -{"status":"FAILED","error_msg":"Token with wrong organization"} -``` - **--- Or ---** - - **Condition** : The user in token claim is NOT a member of the organization - -**Code** : `403 Forbidden` - - **Content** : - ```json - {"detail": "{username} is NOT a member of {org}"} - ``` - -**Condition** : The token is invalid - -**Code** : `403 Forbidden` - - **Content** : - ```json - {"detail": "Given token not valid for any token type"} -``` - diff --git a/docs/rtd/source/user_guide/prov-sys/org_nn.md b/docs/rtd/source/user_guide/prov-sys/org_nn.md deleted file mode 100644 index b31bc0795..000000000 --- a/docs/rtd/source/user_guide/prov-sys/org_nn.md +++ /dev/null @@ -1,57 +0,0 @@ -# Org Number of Nodes - -Get the number of nodes currently deployed in the cluster for the organization along with the minimum and maximum allowed for the cluster - -**URL** : `ps.slideruleearth.io/api/org_num_nodes//` - -**Method** : `GET` - -**Auth required** : Yes. A valid access token for the organization must be included in the Authorization Header Bearer field - -**Permissions required** : User is an active member of the organization (Note: the organization is specified as a claim in the provided token) - -**Data constraints** - -N/A - -**Curl example** - -``` -curl -X GET \ - -H "Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNjY5NzI2OTg4LCJpYXQiOjE2Njk2NDA1ODgsImp0aSI6ImI2Njc2Zjg1YTBkNjRlNjY4ZTEyZWQxYjlmNTFhN2M0Iiwib3JnX25hbWUiOiJVb2ZNRFRlc3QiLCJ1c2VyX25hbWUiOiJjZXVnYXJ0ZWJsYWlyIiwidXNlcl9pZCI6M30.4Q2E76l4UGHbUlrN7hYPKIEa4FKMJ7UhFEwJTV6fqdk" https://ps.testsliderule.org/api/org_num_nodes/UofMDTest/ -``` - -## Success Responses - -**Condition** : user is an valid active member of the organization, the token in the Bearer field is a valid token for organization, the desired number of nodes is within the min/max range. The cluster is already deployed with one node. - -**Code** : `200 OK` - -**Data Response** - -```json -{"status":"SUCCESS","min_nodes":0,"current_nodes":1,"max_nodes":3} -``` -**Notes** - -The response has a status field which indicated that the request was successful and fields for the mimimum,maximum and current number of nodes - -## Error Responses - -**Condition** : typo in the URL, i.e. invalid url - -**Code** : `404 NOT FOUND` - -**Content** : `{The requested resource was not found on this server.}` - -**--- Or ---** - -**Condition** : If a token for the wrong organization was provided - -**Code** : `400 BAD REQUEST` - -**Content** : -```json -{"status":"FAILED","error_msg":"Token with wrong organization"} -``` - \ No newline at end of file diff --git a/docs/rtd/source/user_guide/prov-sys/org_token.md b/docs/rtd/source/user_guide/prov-sys/org_token.md deleted file mode 100644 index b7916d1c1..000000000 --- a/docs/rtd/source/user_guide/prov-sys/org_token.md +++ /dev/null @@ -1,76 +0,0 @@ -# Obtain an access/refresh token pair - -"Login" to a api session by obtaining an access bearer token to use to authorize subsequent requests - -**URL** : `ps.slideruleearth.io/api/org_token/` - -**Method** : `POST` - -**Auth required** : YES username and password are required - -**Permissions required** : Yes. User must be an active member of the organization for which the token is to be used - -**Data constraints** - -Provide username password and organization name. - -```json -{ - "username" : "{username}", - "password" : "******************", - "org_name" : "UofMDTest" -} -``` - - -**Curl Example** -``` -curl -X POST \ - -H "Content-Type: application/json" \ - -d '{ - "username" : "{username}", - "password" : "******************", - "org_name" : "UofMDTest" - }' \ -https://ps.slideruleearth.io/api/org_token/ -``` - - -## Success Response - -**Condition** : If the user is an active member of the organization - -**Code** : `200 Ok` - -```json -{ - "exp": "2022-11-29T13:03:08UTC", - "access_lifetime": "86400.0", - "refresh_lifetime": "2592000.0", - "refresh": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoicmVmcmVzaCIsImV4cCI6MTY3MjIzMjU4OCwiaWF0IjoxNjY5NjQwNTg4LCJqdGkiOiJh1234567890ZmN2U0YjAwOWNkOGI2ZTI1MzRiNTdkNyIsIm9yZ19uYW1lIjoiVW9mTU1234567890dXNlcl9uYW1lIjoiY2V1Z2FydGVibGFpciIsInVzZXJfaWQiOjN9.czx8dPA4msGC1234581K6iwVwpW815kzluk7Htw-gow", - "access": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwi12345joxNjY5NzI2OTg4LCJpYXQiOjE2Njk2NDA1ODgsImp0aSI6Im123452Zjg1YTBkNjRlNjY4ZTEyZWQxYjlmNTFhN2M0Iiwib3JnX25hbWUiOiJVb2ZNRFRlc3Qi12345678905hbWUiOiJjZXVnYXJ0ZWJsYWlyIiwidXNlcl9pZCI6M30.4Q2E76l4UGHbUlrN7hYPKIEa4FKMJ7UhFEwJTV6fqdk" -} -``` -**Notes**: exp is expiration of the access token. access_lifetime and refresh_lifetime are given is seconds. The refresh token can be used only once to obtain a new access token. Otherwise the user must use the /api/org_token/ endpoint and submit username and password. - -## Error Responses - -**Condition** : If organization is not valid. - -**Code** : `403 Forbidden` - -**Content** : -```json -{"detail": "{org} is NOT a valid organization name"} -``` - -**--- Or ---** - -**Condition** : If username or password are not valid. - -**Code** : `401 Unauthorized` - -**Content** : -```json -{"detail": "No active account found with the given credentials"} -``` diff --git a/docs/rtd/source/user_guide/prov-sys/refresh_token.md b/docs/rtd/source/user_guide/prov-sys/refresh_token.md deleted file mode 100644 index 7e2491b4f..000000000 --- a/docs/rtd/source/user_guide/prov-sys/refresh_token.md +++ /dev/null @@ -1,93 +0,0 @@ -# Refresh an access token - -Request a new access token using a refresh token - -**URL** : `ps.slideruleearth.io/api/org_token/refresh/` - -**Method** : `POST` - -**Auth required** : Yes. A valid refresh token must be provided in the body - -**Permissions required** : User in the refresh token is an active member of the organization (Note: the organization is specified as a claim in the provided token) - -**Data constraints** - -A valid, unused, unexpired refresh token. - -**Curl Example** - -``` -curl -X POST \ - -H Content-Type: application/json \ - -d {"refresh": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoicmVmcmVzaCIsImV4cCI6MTY3MDA3MzEyNSwiaWF0IjoxNjY3NDgxMTI112345678901234567890MzExZjE0M2VlODUyYjE4YWEyOWRkMzg3MCIsIm9yZ19uYW1lIjoiVW9mTURUZXN0IiwidXNlcl9uYW1lIjoiY2V1Z2FydGVibGFpciIsInVzZXJfaWQiOjN9.NW4mJUsm48oNK0iRqojNEiC0fyqm3GANAjcii3T41pE"} \ -https://ps.slideruleearth.io/api/org_token/refresh/ -``` - -## Success Responses - -**Condition** : user in token claim is a valid active member of the organization, the refresh token in the body is a valid refresh token for organization. - -**Code** : `200 OK` - -**Data Response** - -```json -{ - "exp": "2022-11-29T15:24:09UTC", - "access_lifetime": "86400.0", - "refresh_lifetime": "2592000.0", - "refresh": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoicmVmcmVzaCIsImV4cCI6MTY3MjI0MTA0OSwiaWF0IjoxNjY5NjQ5MDQ5LCJqdG123456789012345678901234567890ZTIwZTIyM2Q0OGZmMyIsIm9yZ19uYW1lIjoiVW9mTURUZXN0IiwidXNlcl9uYW1lIjoiY2V1Z2FydGVibGFpciIsInVzZXJfaWQiOjN9.hqbI18P8alQ1v2s0qun1NxOlou3QG0Ggd4vPmAotaZc", - "access": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNjY5NzM1NDQ5LCJpYXQiOjE2Njk2NDkwNDksImp0aSI6IjJmODcwYjlkM2E4ZjQyNWE4ZjgxNzI2YTQ3YWUxN2Q0Iiwib3JnX25hbWUiOiJVb2ZNRFRlc3QiLCJ1c2VyX25hbWUiOiJjZXVnYXJ0ZWJsYWlyIiwidXNlcl9pZCI6M30.NX396fsa-FotYMctDVe0CSVRDs17Q5lVuoBPCWPkRhM" -} -``` - -**Condition** : user in token claim is member but NOT an valid ***active*** member of the organization, the refresh token in the body is a valid token for the organization. - -**Code** : `200 OK` - -**Data Response** - -{"active":false} - -**Notes** - - -## Error Responses - -**Condition** : typo in the URL, i.e. invalid url - -**Code** : `404 NOT FOUND` - -**Content** : `{The requested resource was not found on this server.}` - - **--- Or ---** - - **Condition** : The user in refresh token claim is NOT a member of the organization - -**Code** : `403 Forbidden` - - **Content** : - ```json - "detail": "{user} is NOT a member of {organization}" - ``` -**--- Or ---** - -**Condition** : The access token is invalid - -**Code** : `403 Forbidden` - - **Content** : - ```json - "detail": "Given token not valid for any token type" - ``` - -**--- Or ---** - -**Condition** : The refresh token is blacklisted (i.e. has already been used) - -**Code** : `403 Forbidden` - - **Content** : - ```json - { "detail": "Token is blacklisted", "code": "token_not_valid" } - ``` \ No newline at end of file diff --git a/docs/rtd/source/user_guide/prov-sys/remove_onn.md b/docs/rtd/source/user_guide/prov-sys/remove_onn.md deleted file mode 100644 index a81dca19b..000000000 --- a/docs/rtd/source/user_guide/prov-sys/remove_onn.md +++ /dev/null @@ -1,59 +0,0 @@ -# Remove outstanding Org Number of Nodes requests - -Request to remove any outstanding org number of nodes requests for the organization specified and the authorized user in the bearer token - -**URL** : `ps.slideruleearth.io/api/remove_org_num_nodes//` - -**Method** : `PUT` - -**Auth required** : Yes. A valid access token for the organization must be included in the Authorization Header Bearer field - -**Permissions required** : User is an active member of the organization (Note: the organization is specified as a claim in the provided token) - -**Data constraints** - -N/A - -**Curl example** - -``` -curl -X PUT \ - -H "Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0b2tlbl90eXBlIjoiYWNjZXNzIiwiZXhwIjoxNjY4MTgxOTYzLCJpYXQiOjE2NjgwOTU1NjMsImp0aSI6ImJmYjIxMmExMzU0ZjQ4NGFhY2E2NmVjYWJmMmE3Mjg4Iiwib3JnX25hbWUiOiJVb2ZNRFRlc3QiLCJ1c2VyX25hbWUiOiJjZXVnYXJ0ZWJsYWlyIiwidXNlcl9pZCI6M30.nl1ACnWcoROhZ7K_HKOCOVfbqiDPBzmPdEPnAdb2vxk" \ https://ps.slideruleearth.io/api/remove_org_num_nodes_reqs/sliderule/ -``` - -## Success Responses - -**Condition** : user is an valid active member of the organization, the token in the Bearer field is a valid token for organization, the desired number of nodes is within the min/max range. The cluster is already deployed. - -**Code** : `200 OK` - -**Data Response** - -```json -{"status":"SUCCESS","msg":"cleaned org node reqs for sliderule {username}"} -``` - -**Notes** - -The response has a status field which indicated that the request was queued. The response has a msg field that inidicated that no update was initiated (because the cluster already had one node deployed). The msg field also indicates that list of queued requests. The queued requests contain the name of the cluster, the user that made the request, the number of nodes requested, and the expiration of the request. The response also has an error_msg field which is blank when the request is successful - -## Error Responses - -**Condition** : typo in the URL, i.e. invalid url - -**Code** : `404 NOT FOUND` - -**Content** : -`{The requested resource was not found on this server.}` - -**--- Or ---** - -**Condition** : If a token for the wrong organization was provided - -**Code** : `400 BAD REQUEST` - -**Content** : -```json -{"status":"FAILED","error_msg":"Token with wrong organization"} -``` - \ No newline at end of file diff --git a/packages/arrow/README.md b/packages/arrow/README.md index ed42617da..0e6160ea5 100644 --- a/packages/arrow/README.md +++ b/packages/arrow/README.md @@ -1,6 +1,6 @@ # Arrow Package -This package provides classses for building and using Apache Arrow in-memory and file formats. +This package provides classes for building and using Apache Arrow in-memory and file formats. ## Install Arrow Library diff --git a/packages/aws/README.md b/packages/aws/README.md index 820c776b2..5e7b26fc4 100644 --- a/packages/aws/README.md +++ b/packages/aws/README.md @@ -1,6 +1,6 @@ # AWS Package -This package provides classses to access data in S3 and to manage AWS credentials. +This package provides classes to access data in S3 and to manage AWS credentials. ## Install cURL Library diff --git a/packages/core/LuaEngine.md b/packages/core/LuaEngine.md index 36eee12e6..1ae867af0 100644 --- a/packages/core/LuaEngine.md +++ b/packages/core/LuaEngine.md @@ -3,7 +3,7 @@ Lua Engine SlideRule is a framework for developing high-performance distributed computing systems for analyzing science data in real-time. -The LuaEngine module encapsulates the Lua intepreter inside a C++ class. +The LuaEngine module encapsulates the Lua interpreter inside a C++ class. The engine has three modes of operation: file, stream, and interactive. **File mode** accepts a file or standard input and uses a ported version of the Lua command line client delivered with the Lua source in order to create an environment as close to possible to the delivered Lua interpreter. diff --git a/packages/core/MsgQ.md b/packages/core/MsgQ.md index 9d9bb85e4..95c4b0c2a 100644 --- a/packages/core/MsgQ.md +++ b/packages/core/MsgQ.md @@ -93,7 +93,7 @@ When a subscriber moves its pointer up the chain, it _dereferences_ the link it ##### MsgQ -`MsgQ::MsgQ (const char* name, MsgQ::free_func_t free_func=NULL, int depth=CFG_DEPTH_STANDARD, int data_size=CFG_SIZE_INFINITY)` : Base constructor for message queues. Except in rare cases when the characteristics of a message queue need to be established before any subsribers or publishers are created, this constructor should not be directly called. Instead the Subscriber or Publisher constructors should be called which will in turn call this constructor with the appropriate settings. +`MsgQ::MsgQ (const char* name, MsgQ::free_func_t free_func=NULL, int depth=CFG_DEPTH_STANDARD, int data_size=CFG_SIZE_INFINITY)` : Base constructor for message queues. Except in rare cases when the characteristics of a message queue need to be established before any subscribers or publishers are created, this constructor should not be directly called. Instead the Subscriber or Publisher constructors should be called which will in turn call this constructor with the appropriate settings. * **name** - the name of the message queue, null if a private queue * **free_func** - the function used to free queued objects after they have been fully dereferenced @@ -105,7 +105,7 @@ When a subscriber moves its pointer up the chain, it _dereferences_ the link it * **existing_q** - an existing message queue * **free_func** - the function used to free queued objects after they have been fully dereferenced -`MsgQ::~MsgQ (void)` : Base destructor for all message queues. Handles freeing resources associated with message queue once all subsribers and publishers have detached. +`MsgQ::~MsgQ (void)` : Base destructor for all message queues. Handles freeing resources associated with message queue once all subscribers and publishers have detached. ##### Static Routines @@ -159,7 +159,7 @@ When a subscriber moves its pointer up the chain, it _dereferences_ the link it * **data** - a pointer to the data being queued * **size** - the size of the data being queued, used in the post only to check against the maximum allowed data size. The size is later used in receive calls: returned in the receive by reference, or used in the receive by copy to copy the data contents out. -* **timeout** - the minimal amount of time, specified in milliseconds, to block the operation in order to wait for it to succeed. If IO_CHECK is supplied, then the operation will be non-blocking and immediately return. If IO_PEND is supplied, then the opperation will block forever until the operation succeeds. +* **timeout** - the minimal amount of time, specified in milliseconds, to block the operation in order to wait for it to succeed. If IO_CHECK is supplied, then the operation will be non-blocking and immediately return. If IO_PEND is supplied, then the operation will block forever until the operation succeeds. _Returns_ - the function will return the state of the message queue at the time of the post operation's attempt. See the STATE_* definitions above for details. If the operation succeeded, STATE_OKAY will be returned. Otherwise, one of the error codes will be returned indicating why the operation failed. This is different than the postCopy which returns the number of bytes copied on success. Given that the data is a pointer, there is no concept of bytes being queued, only that the post succeeded or failed for a given reason. @@ -169,21 +169,21 @@ _Returns_ - the function will return the state of the message queue at the time * **data** - a pointer to the data being queued * **size** - the size of the data being queued, used to copy the data into the queue and to check against the maximum allowed data size. -* **timeout** - the minimal amount of time, specified in milliseconds, to block the operation in order to wait for it to succeed. If IO_CHECK is supplied, then the operation will be non-blocking and immediately return. If IO_PEND is supplied, then the opperation will block forever until the operation succeeds. +* **timeout** - the minimal amount of time, specified in milliseconds, to block the operation in order to wait for it to succeed. If IO_CHECK is supplied, then the operation will be non-blocking and immediately return. If IO_PEND is supplied, then the operation will block forever until the operation succeeds. _Returns_ - the function will return the size of the data queued on success, or an error code on failure. See the STATE_* definitions above for details. -`int Publisher::postCopy (const void* header_data, int header_size, const void* payload_data, int payload_size, int timeout=IO_CHECK)` : This function behaves exactly like the above postCopy function except that it takes two sets of pointers and sizes to the data instead of one. The purpose of this function is to support the often employed paradigm where messages are encapsulated inside of a highler level protocol before being queued. For instance, if an image is being transfered over a SpaceWire network using RMAP (remote memory access protocol), the image data would need to be broken up into smaller chucks, have an RMAP header prepended to it, and then sent. Prepending the header requires that a new contiguous memory block be obtained of sufficient size to contain both the RMAP header and the image payload. Providing this API allows the allocation of that contiquous memory block and the subsequent copying of the data to only occur once - when the data is enqueued. Note that the data size check is performed against the sum total of the header and payload sizes. +`int Publisher::postCopy (const void* header_data, int header_size, const void* payload_data, int payload_size, int timeout=IO_CHECK)` : This function behaves exactly like the above postCopy function except that it takes two sets of pointers and sizes to the data instead of one. The purpose of this function is to support the often employed paradigm where messages are encapsulated inside of a highler level protocol before being queued. For instance, if an image is being transferred over a SpaceWire network using RMAP (remote memory access protocol), the image data would need to be broken up into smaller chucks, have an RMAP header prepended to it, and then sent. Prepending the header requires that a new contiguous memory block be obtained of sufficient size to contain both the RMAP header and the image payload. Providing this API allows the allocation of that contiquous memory block and the subsequent copying of the data to only occur once - when the data is enqueued. Note that the data size check is performed against the sum total of the header and payload sizes. * **header_data** - a pointer to the header of the message being queued * **header_size** - the size of the header * **payload_data** - a pointer to the payload of the message being queued * **payload_size** - the size of the payload -* **timeout** - the minimal amount of time, specified in milliseconds, to block the operation in order to wait for it to succeed. If IO_CHECK is supplied, then the operation will be non-blocking and immediately return. If IO_PEND is supplied, then the opperation will block forever until the operation succeeds. +* **timeout** - the minimal amount of time, specified in milliseconds, to block the operation in order to wait for it to succeed. If IO_CHECK is supplied, then the operation will be non-blocking and immediately return. If IO_PEND is supplied, then the operation will block forever until the operation succeeds. _Returns_ - the function will return the size of the total amount data queued on success, or an error code on failure. See the STATE_* definitions above for details. -`int Publisher::postString (const char* format_string, ...)` : Creates a formated ASCII string of data as specified by the format_string. Underneath, this function calls one of the above postCopy functions, and is provided here as a wrapper function because of how often such functionality is needed when working with text based messaging. The ASCII string that is ultimately enqueued is always null terminated, and the size of the object recorded in the message queue includes the null termination. +`int Publisher::postString (const char* format_string, ...)` : Creates a formatted ASCII string of data as specified by the format_string. Underneath, this function calls one of the above postCopy functions, and is provided here as a wrapper function because of how often such functionality is needed when working with text based messaging. The ASCII string that is ultimately enqueued is always null terminated, and the size of the object recorded in the message queue includes the null termination. * **format_string** - a string format specification analogous to the c printf specification. @@ -191,7 +191,7 @@ _Returns_ - the function will return the size of the string (including null term ##### Subscriber -`Subscriber::Subscriber (const char* name, subscriber_type_t type=SUBSCRIBER_OF_CONFIDENCE, int depth=CFG_DEPTH_STANDARD, int data_size=CFG_SIZE_INFINITY)` : Constructor for the Subscriber class. This will create a subscribing attachment to the specified message queue. If the queue does not exist, it will create the queue with the parameters specified. If the queue does exist, the parameters specified will be ignored and the attachment will proceed. Only data that is posted AFTER a subsriber has attached will be made available to that subscriber. In other words, any data that currently exists on the message queue when a subsriber attaches will not be visible or ever returned to the subsriber. +`Subscriber::Subscriber (const char* name, subscriber_type_t type=SUBSCRIBER_OF_CONFIDENCE, int depth=CFG_DEPTH_STANDARD, int data_size=CFG_SIZE_INFINITY)` : Constructor for the Subscriber class. This will create a subscribing attachment to the specified message queue. If the queue does not exist, it will create the queue with the parameters specified. If the queue does exist, the parameters specified will be ignored and the attachment will proceed. Only data that is posted AFTER a subscriber has attached will be made available to that subscriber. In other words, any data that currently exists on the message queue when a subscriber attaches will not be visible or ever returned to the subscriber. * **name** - the name of the message queue * **type** - the type of subscription, see SUBSCRIBER_OF_CONFIDENCE and SUBSCRIBER_OF_OPPORTUNITY definitions above. @@ -203,14 +203,14 @@ _Returns_ - the function will return the size of the string (including null term * **existing_q** - an existing message queue * **type** - the type of subscription, see SUBSCRIBER_OF_CONFIDENCE and SUBSCRIBER_OF_OPPORTUNITY definitions above. -`Subscriber::~Subscriber (void)` : Destructor for the Subscriber class. Detaches the subscriber from the message queue, dereferences all queued objects remaining on the message queue for this subsriber only, frees resources associated with the subscriber, and then calls the MsgQ destructor. +`Subscriber::~Subscriber (void)` : Destructor for the Subscriber class. Detaches the subscriber from the message queue, dereferences all queued objects remaining on the message queue for this subscriber only, frees resources associated with the subscriber, and then calls the MsgQ destructor. ##### Receive Reference `int Subscriber::receiveRef (msgRef_t& ref, int timeout)` : Receives a reference to the oldest subscribed-to object on the message queue that has yet to be received by the subscriber (i.e. the next object on the queue). No data is copied, instead only the data pointer and data size is placed in the returned reference structure. If this function is called, a subsequent call to _dereference_ must be made once the data is no longer needed. * **ref** - a reference to a msgRef_t structure. The contents of the structure are populated by the function. See its definition above in the [Types](#types) section. On both success and error the structure is populated, but only on success can the contents of the structure be trusted. -* **timeout** - the minimal amount of time, specified in milliseconds, to block the operation in order to wait for it to succeed. If IO_CHECK is supplied, then the operation will be non-blocking and immediately return. If IO_PEND is supplied, then the opperation will block forever until the operation succeeds. +* **timeout** - the minimal amount of time, specified in milliseconds, to block the operation in order to wait for it to succeed. If IO_CHECK is supplied, then the operation will be non-blocking and immediately return. If IO_PEND is supplied, then the operation will block forever until the operation succeeds. _Returns_ - the function will return the state of the message queue at the time of the receive operation's attempt. See the STATE_* definitions above for details. If the operation succeeded, STATE_OKAY will be returned. Otherwise, one of the error codes will be returned indicated why the operation failed. This is different than the receiveCopy which returns the number of bytes copied on success. Given that the data is a reference, there is no concept of bytes being dequeued, only that the receive succeeded or failed for a given reason. @@ -231,7 +231,7 @@ _Returns_ - true on success, false on failure. Currently, there is no condition * **data** - a pointer to a buffer that the dequeued data will be copied into * **size** - the size of the data buffer, and therefore the maximum size allowed for the dequeued data dequeued. -* **timeout** - the minimal amount of time, specified in milliseconds, to block the operation in order to wait for it to succeed. If IO_CHECK is supplied, then the operation will be non-blocking and immediately return. If IO_PEND is supplied, then the opperation will block forever until the operation succeeds. +* **timeout** - the minimal amount of time, specified in milliseconds, to block the operation in order to wait for it to succeed. If IO_CHECK is supplied, then the operation will be non-blocking and immediately return. If IO_PEND is supplied, then the operation will block forever until the operation succeeds. _Returns_ - the function will return the size of the data object dequeued and copied into the buffer on success, or an error code on failure. See the STATE_* definitions above for details. diff --git a/plugins/atlas/README.md b/plugins/atlas/README.md index bab3c71bd..95249a6b3 100644 --- a/plugins/atlas/README.md +++ b/plugins/atlas/README.md @@ -121,7 +121,7 @@ An example could be additional threads is often fine even when the results need to be ordered. 6. The key is used to create the index for each record. A typical index is the creation time - provided in the secondary header of the CCSDS telemetry packet. The following options are availble + provided in the secondary header of the CCSDS telemetry packet. The following options are available for creating the index: * RECEIPT_KEY - increments by one for every record in the dispatched stream * INCREMENTING_KEY - increments by one for every metric created (no gaps in index value) diff --git a/plugins/atlas/apps/dfcview.py b/plugins/atlas/apps/dfcview.py index 6b35ee16a..86b9906db 100644 --- a/plugins/atlas/apps/dfcview.py +++ b/plugins/atlas/apps/dfcview.py @@ -99,7 +99,7 @@ def hidePopup(self): self.updateText() def timerEvent(self, event): - # After timeout, kill timer, and reenable click on line edit + # After timeout, kill timer, and re-enable click on line edit self.killTimer(event.timerId()) self.closeOnLineEditClick = False diff --git a/plugins/atlas/apps/sigview.py b/plugins/atlas/apps/sigview.py index 9ad8bf363..5dbf19376 100644 --- a/plugins/atlas/apps/sigview.py +++ b/plugins/atlas/apps/sigview.py @@ -370,7 +370,7 @@ def on_click_filter(self): @pyqtSlot() def on_click_peak_align(self): - integration, okPressed = QInputDialog.getInt(self, "Histgram Integration Period","Major Frames:", self.histIntegration, 0, 500, 1) + integration, okPressed = QInputDialog.getInt(self, "Histogram Integration Period","Major Frames:", self.histIntegration, 0, 500, 1) if okPressed: self.histIntegration = integration self.integrateLabel.setText("{}".format(self.histIntegration)) From eea774dc40dd49b2de14f1ecc95b2f9b6f795132 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Tue, 3 Oct 2023 12:07:59 +0000 Subject: [PATCH 4/5] set pre commit spell check to ignore parm and parms --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04d22cfb6..b2ca0ea30 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: hooks: - id: codespell args: [ -# '--ignore-words-list', 'nd,alos,inout', + '--ignore-words-list', 'parm,parms', # '--ignore-regex', '\bhist\b', '--' ] From 101f684806b58382357727cd62b369e3692a7a72 Mon Sep 17 00:00:00 2001 From: JP Swinski Date: Tue, 3 Oct 2023 12:10:04 +0000 Subject: [PATCH 5/5] continued spelling fixes --- clients/python/utils/benchmark.py | 2 +- docs/rtd/source/archive/gdal_vrt_benchmark.md | 2 +- docs/rtd/source/release_notes/release-v4.0.0.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/clients/python/utils/benchmark.py b/clients/python/utils/benchmark.py index fa613107c..3931c345c 100644 --- a/clients/python/utils/benchmark.py +++ b/clients/python/utils/benchmark.py @@ -251,7 +251,7 @@ def atl03_rasterized_subset(): return icesat2.atl03sp(parms, resources=[args.granule03]) # ------------------------------------ -# Benchmark ATL03 Polgon Subset +# Benchmark ATL03 Polygon Subset # ------------------------------------ def atl03_polygon_subset(): parms = { diff --git a/docs/rtd/source/archive/gdal_vrt_benchmark.md b/docs/rtd/source/archive/gdal_vrt_benchmark.md index 54ddbf7b0..1f6e43312 100644 --- a/docs/rtd/source/archive/gdal_vrt_benchmark.md +++ b/docs/rtd/source/archive/gdal_vrt_benchmark.md @@ -41,7 +41,7 @@ Points read: 1000000 16998.240999877 1000000 points read time: 169940.77899982 msecs -In the second aproach the vrtdataset and vrtband are used ONLY to find the name of the raster containing POI. +In the second approach the vrtdataset and vrtband are used ONLY to find the name of the raster containing POI. The code is very similar to how gdallocationinfo tool is implemented using xml parsing in GDAL. The raster tif file is than opened and this raster's dataset and band are used to do a read using col, row calculated for that raster. This method is much more efficient. It only took 665 msecs to do one million reads. diff --git a/docs/rtd/source/release_notes/release-v4.0.0.md b/docs/rtd/source/release_notes/release-v4.0.0.md index a142535fe..e8922b96a 100644 --- a/docs/rtd/source/release_notes/release-v4.0.0.md +++ b/docs/rtd/source/release_notes/release-v4.0.0.md @@ -68,7 +68,7 @@ parms = { "poly": region['poly'], * The ATL03 subsetting code now operates on one spot per thread (it used to be one pair track per thread). This change is made in conjunction with a change in the base AWS instance type to an instance that has 8 cores. The additional subsetting threads will therefore be able to utilize the greater number of cores when subsetting and speed up processing requests when only one subsetting operation on a given node is being processed. -* When making raster sample requests, the user no longer needs to perform a separate earthdata.* call to generate a catalog for the 3DEP and Landsat raster datasets, but the client is not smart enouh to know to generate the catalog on its own if the user does not supply it. +* When making raster sample requests, the user no longer needs to perform a separate earthdata.* call to generate a catalog for the 3DEP and Landsat raster datasets, but the client is not smart enough to know to generate the catalog on its own if the user does not supply it. ## Development Updates