From 1880cb920d510f48a37f12a5245d65336d2144da Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Thu, 10 Oct 2024 14:28:44 -0700 Subject: [PATCH 01/13] Applying some fixes using pyright LSP guidance. --- dts/client.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dts/client.py b/dts/client.py index d84265e..f1fc625 100644 --- a/dts/client.py +++ b/dts/client.py @@ -11,6 +11,7 @@ from urllib.error import ( HTTPError, ) +from typing import Any logger = logging.getLogger('dts') api_version = 1 @@ -78,7 +79,7 @@ def disconnect(self): self.name = None self.version = None - def databases(self): + def databases(self) -> list[Database] | None: """`client.databases()` -> `list` of `Database` objects * Returns all databases available to the service, or `None` if an error occurs.""" @@ -106,7 +107,7 @@ def search(self, offset = 0, limit = None, specific = None, - ): + ) -> list[JsonResource] | None: """ `client.search(database = None, query = None, @@ -139,7 +140,7 @@ def search(self, raise RuntimeError('search: missing query.') if not isinstance(database, str): raise TypeError('search: database must be a string.') - params = { + params: dict[str, Any] = { 'database': database, 'query': query, } @@ -200,7 +201,7 @@ def fetch_metadata(self, raise RuntimeError('search: missing or invalid file IDs.') if not isinstance(database, str): raise TypeError('search: database must be a string.') - params = { + params: dict[str, Any] = { 'database': database, 'ids': ','.join(ids), } @@ -234,6 +235,7 @@ def transfer(self, source = None, destination = None, description = None, + instructions = None, timeout = None): """ `client.transfer(file_ids = None, From edae1ab3ddab804f7eea405a24b29bc3556ba21e Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Mon, 14 Oct 2024 08:44:17 -0700 Subject: [PATCH 02/13] Adding mkdocs for API documentation. --- requirements.txt | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index bc090c0..78c15c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,29 +1,48 @@ annotated-types==0.7.0 attrs==23.2.0 -certifi==2024.7.4 +babel==2.16.0 +certifi==2024.2.2 chardet==5.2.0 charset-normalizer==3.3.2 click==8.1.7 -coverage==7.6.0 +colorama==0.4.6 frictionless==5.17.0 +ghp-import==2.1.0 +griffe==1.4.1 humanize==4.9.0 idna==3.7 isodate==0.6.1 Jinja2==3.1.4 jsonschema==4.22.0 jsonschema-specifications==2023.12.1 +Markdown==3.7 markdown-it-py==3.0.0 marko==2.0.3 MarkupSafe==2.1.5 mdurl==0.1.2 +mergedeep==1.3.4 +mkdocs==1.6.1 +mkdocs-autorefs==1.2.0 +mkdocs-get-deps==0.2.0 +mkdocs-material==9.5.40 +mkdocs-material-extensions==1.3.1 +mkdocstrings==0.26.2 +mkdocstrings-python==1.12.1 +packaging==24.1 +paginate==0.5.7 +pathspec==0.12.1 petl==1.7.15 +platformdirs==4.3.6 pydantic==2.7.2 pydantic_core==2.18.3 Pygments==2.18.0 +pymdown-extensions==10.11.2 python-dateutil==2.9.0.post0 python-slugify==8.0.4 PyYAML==6.0.1 +pyyaml_env_tag==0.1 referencing==0.35.1 +regex==2024.9.11 requests==2.32.3 rfc3986==2.0.0 rich==13.7.1 @@ -36,6 +55,7 @@ tabulate==0.9.0 text-unidecode==1.3 typer==0.12.3 typing_extensions==4.12.0 -urllib3==2.2.2 +urllib3==2.2.1 uuid==1.30 validators==0.28.3 +watchdog==5.0.3 From fb720442271f17c56147a9d559e40f0e459c6f6a Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Mon, 14 Oct 2024 11:47:09 -0700 Subject: [PATCH 03/13] Added some Google-style docstrings and fixed some more type hints. --- dts/__init__.py | 11 +- dts/client.py | 262 +++++++++++++++++++++++++---------------- dts/database.py | 7 +- dts/transfer_status.py | 7 +- 4 files changed, 183 insertions(+), 104 deletions(-) diff --git a/dts/__init__.py b/dts/__init__.py index 51747b9..a1500e3 100644 --- a/dts/__init__.py +++ b/dts/__init__.py @@ -1,4 +1,13 @@ -"""dts: a client for the Data Transfer Service""" +"""dts: a Python client for the Data Transfer Service. + +The [Data Transfer System (DTS)](https://kbase.github.io/dts/) offers a federated +search capability for participating organizations in the DOE Biological and +Environmental Research program, and allows the transfer of related data and +metadata between these organizations. + +DTS API documentation is available [here](https://lb-dts.staging.kbase.us/docs#/). + +""" from .database import Database from .transfer_status import TransferStatus diff --git a/dts/client.py b/dts/client.py index f1fc625..b940c5f 100644 --- a/dts/client.py +++ b/dts/client.py @@ -18,7 +18,7 @@ class KBaseAuth(AuthBase): """Attaches a KBase-sensible Authorization header to the given Request object.""" - def __init__(self, api_key): + def __init__(self, api_key: str): self.api_key = api_key def __call__(self, request): @@ -28,14 +28,31 @@ def __call__(self, request): return request class Client: - """`Client`: A client for performing file transfers with the Data Transfer System""" + """`dts.Client`: A client for performing file transfers with the Data Transfer System (DTS). + +This type exposes the [DTS API](https://lb-dts.staging.kbase.us/docs#/) for use +in Python programs. +""" def __init__(self, - api_key = None, - server = None, - port = None): - """`Client(server = None, port = None, api_key = None)` -> DTS client. + api_key: str | None = None, + server: str | None = None, + port: int | None = None): + """Creates a DTS client that handles search and transfer requests via +a connected server. + +If no server is specified, you must call `connect` on the created client. -* If no `server` is given, you must call `connect` on the created client.""" +Args: + api_key: An unencoded KBase developer token. + server: The DTS server that handles the client's API requests. + port: The port to which the client connects with the server. + +Returns: + a `dts.Client` instance. + +Raises: + TypeError: an argument of improper type was specified. +""" if server: self.connect(server = server, port = port, api_key = api_key) else: @@ -44,13 +61,20 @@ def __init__(self, self.version = None def connect(self, - api_key = None, - server = None, - port = None): - """`client.connect(api_key = None, server = None, port = None)` + api_key: str | None = None, + server: str | None = None, + port: int | None = None) -> None: + """Connects the client to the given DTS server via the given port using the given +(unencoded) KBase developer token. -* Connects the client to the given DTS `server` via the given `port` using the given - (unencoded) `api_key`.""" +Args: + api_key: An unencoded KBase developer token. + server: The DTS server that handles the client's API requests. + port: The port to which the client connects with the server. + +Raises: + TypeError: an argument of improper type was specified. +""" if not isinstance(api_key, str): raise TypeError('api_key must be an unencoded API key.') if not isinstance(server, str): @@ -70,19 +94,22 @@ def connect(self, self.name = result['name'] self.version = result['version'] - def disconnect(self): - """`client.disconnect() -> None - -* disconnects the client from the server.""" + def disconnect(self) -> None: + """Disconnects the client from the server. +""" self.api_key = None self.uri = None self.name = None self.version = None - def databases(self) -> list[Database] | None: - """`client.databases()` -> `list` of `Database` objects + def databases(self) -> list[Database]: + """Returns all databases available to the service. + +Server-side errors are captured and logged. -* Returns all databases available to the service, or `None` if an error occurs.""" +Returns: + A list of Database objects containing information about available databases. +""" if not self.uri: raise RuntimeError('dts.Client: not connected.') try: @@ -90,10 +117,10 @@ def databases(self) -> list[Database] | None: response.raise_for_status() except HTTPError as http_err: logger.error(f'HTTP error occurred: {http_err}') - return None + return [] except Exception as err: logger.error(f'Other error occurred: {err}') - return None + return [] results = response.json() return [Database(id = r['id'], name = r['name'], @@ -101,41 +128,42 @@ def databases(self) -> list[Database] | None: url = r['url']) for r in results] def search(self, - database = None, - query = None, - status = None, + database: str, + query: str | int | float, + status: str | None, offset = 0, limit = None, specific = None, - ) -> list[JsonResource] | None: - """ -`client.search(database = None, - query = None, - status = None, - offset = 0, - limit = None, - specific = None) -> `list` of `frictionless.DataResource` objects - -* Performs a synchronous search of the database with the given name using the - given query string. -Optional arguments: - * query: a search string that is directly interpreted by the database - * status: filters for files based on their status: - * `"staged"` means "search only for files that are already in the source database staging area" - * `"unstaged"` means "search only for files that are not staged" - * offset: a 0-based index from which to start retrieving results (default: 0) - * limit: if given, the maximum number of results to retrieve - * specific: a dictionary mapping database-specific search parameters to their values + ) -> list[JsonResource]: + """Performs a synchronous search of the database with the given name using the given query string. + +This method searches the indicated database for files that can be transferred. + +Args: + database: A string containing the name of the database to search. + query: A search string that is directly interpreted by the database. + status: An optional string (`"staged"` or `"unstaged"`) indicating whether files are filtered based on their status. + offset: An optional 0-based pagination index indicating the first retrieved result (default: 0). + limit: An optional pagination parameter indicating the maximum number of results to retrieve. + specific: An optional dictionary mapping database-specific search parameters to their values. + +Returns: + A list of [frictionless DataResources](https://specs.frictionlessdata.io/data-resource/) + containing metadata for files matching the query. + +Raises: + RuntimeError: Indicates an issue with the DTS client and its connection to the server. + TypeError: Indicates that an argument passed to the client isn't of the proper type. + ValueError: Indicates that an argument passed to the client has an invalid value. """ if not self.uri: raise RuntimeError('dts.Client: not connected.') - if query: - if not isinstance(query, str): - # we also accept numeric values - if isinstance(query, int) or isinstance(query, float): - query = str(query) - else: - raise RuntimeError('search: query must be a string or a number.') + if not isinstance(query, str): + # we also accept numeric values + if isinstance(query, int) or isinstance(query, float): + query = str(query) + else: + raise TypeError('search: query must be a string or a number.') else: raise RuntimeError('search: missing query.') if not isinstance(database, str): @@ -171,29 +199,35 @@ def search(self, response.raise_for_status() except (HTTPError, requests.exceptions.HTTPError) as err: logger.error(f'HTTP error occurred: {err}') - return None + return [] except Exception as err: logger.error(f'Other error occurred: {err}') - return None + return [] return [JsonResource(r) for r in response.json()['resources']] def fetch_metadata(self, - database = None, - ids = None, - offset = 0, - limit = None, - ): - """ -`client.fetch_metadata(database = None, - ids = None, - offset = 0, - limit = None) -> `list` of `frictionless.DataResource` objects + database: str, + ids: list[str], + offset: int = 0, + limit: int | None = None) -> list[JsonResource]: + """Fetches metadata for the files with the specified IDs within the specified database. + +Server-side errors are intercepted and logged. -* Fetches metadata for the files with the specified IDs within the specified - database. -Optional arguments: - * offset: a 0-based index from which to start retrieving results (default: 0) - * limit: if given, the maximum number of results to retrieve +Args: + database: A string containing the name of the database to search. + ids: A list containing file identifiers for which metadata is retrieved. + offset: An optional 0-based pagination index from which to start retrieving results (default: 0). + limit: An optional pagination parameter indicating the maximum number of results to retrieve. + +Returns: + A list of [frictionless DataResources](https://specs.frictionlessdata.io/data-resource/) + containing metadata for files with the requested IDs. + +Raises: + RuntimeError: Indicates an issue with the DTS client and its connection to the server. + TypeError: Indicates that an argument passed to the client isn't of the proper type. + ValueError: Indicates that an argument passed to the client has an invalid value. """ if not self.uri: raise RuntimeError('dts.Client: not connected.') @@ -224,33 +258,39 @@ def fetch_metadata(self, response.raise_for_status() except (HTTPError, requests.exceptions.HTTPError) as err: logger.error(f'HTTP error occurred: {err}') - return None + return [] except Exception as err: logger.error(f'Other error occurred: {err}') - return None + return [] return [JsonResource(r) for r in response.json()['resources']] def transfer(self, - file_ids = None, - source = None, - destination = None, - description = None, - instructions = None, - timeout = None): - """ -`client.transfer(file_ids = None, - source = None, - destination = None, - description = None, - instructions = None, - timeout = None) -> UUID - -* Submits a request to transfer files from a source to a destination database. the - files in the source database are identified by a list of string file_ids. -Optional arguments: - * description: a string containing Markdown text describing the transfer - * instructions: a dict representing a JSON object containing instructions - for processing the payload at its destination + file_ids: list[str], + source: str, + destination: str, + description: str | None = None, + instructions: dict[str, Any] | None = None, + timeout: int | None = None) -> uuid.UUID | None: + """Submits a request to transfer files from a source to a destination database. + +Server-side errors are intercepted and logged. + +Args: + file_ids: A list of identifiers for files to be transferred. + source: The name of the database from which files are transferred. + destination: The name of the database to which files are transferred. + description: An optional string containing human-readable Markdown text describing the transfer. + instructions: An optional dict representing a JSON object containing instructions for processing the payload at its destination. + timeout: An optional integer indicating the number of seconds to wait for a response from the server. + +Returns: + A UUID uniquely identifying the file transfer that can be used to check its + status, or None if a server-side error is encountered. + +Raises: + RuntimeError: Indicates an issue with the DTS client and its connection to the server. + TypeError: Indicates that an argument passed to the client isn't of the proper type. + ValueError: Indicates that an argument passed to the client has an invalid value. """ if not self.uri: raise RuntimeError('dts.Client: not connected.') @@ -289,11 +329,18 @@ def transfer(self, return None return uuid.UUID(response.json()["id"]) - def transfer_status(self, id): - """`client.transfer_status(id)` -> TransferStatus + def transfer_status(self, id: uuid.UUID) -> TransferStatus | None: + """Returns status information for the transfer with the given identifier. -* Returns status information for the transfer with the given identifier. - Possible statuses are: + +Server-side errors are intercepted and logged. + +Arguments: + id: A UUID that uniquely identifies the transfer operation for which the status is requested. + +Returns: + A `TransferStatus` object whose contents indicate the status of the transfer, + or None if a server-side error occurs. Possible statuses are: * `'staging'`: the files requested for transfer are being copied to the staging area for the source database job * `'active'`: the files are being transferred from the source database to the @@ -302,7 +349,12 @@ def transfer_status(self, id): * `'inactive'`: the file transfer has been suspended * `'failed'`: the file transfer could not be completed because of a failure` * `'unknown'`: the status of the given transfer is unknown -* If an error is encountered, returns `None`.""" + +Raises: + RuntimeError: Indicates an issue with the DTS client and its connection to the server. + TypeError: Indicates that an argument passed to the client isn't of the proper type. + ValueError: Indicates that an argument passed to the client has an invalid value. +""" if not self.uri: raise RuntimeError('dts.Client: not connected.') try: @@ -310,7 +362,7 @@ def transfer_status(self, id): auth=self.auth) response.raise_for_status() except (HTTPError, requests.exceptions.HTTPError) as err: - logger.error(f'HTTP error occurred: {http_err}') + logger.error(f'HTTP error occurred: {err}') return None except Exception as err: logger.error(f'Other error occurred: {err}') @@ -324,11 +376,19 @@ def transfer_status(self, id): num_files_transferred = results.get('num_files_transferred'), ) - def cancel_transfer(self, id): - """ -`client.cancel_transfer(id) -> None + def cancel_transfer(self, id: uuid.UUID): + """Cancels a file transfer with the requested UUID. + +Status information for the cancelled transfer is retained for a time so its +cancellation can be seen. + +Args: + id: A UUID that uniquely identifies the transfer operation to be cancelled. -* Deletes a file transfer, canceling +Raises: + RuntimeError: Indicates an issue with the DTS client and its connection to the server. + TypeError: Indicates that an argument passed to the client isn't of the proper type. + ValueError: Indicates that an argument passed to the client has an invalid value. """ if not self.uri: raise RuntimeError('dts.Client: not connected.') @@ -337,7 +397,7 @@ def cancel_transfer(self, id): auth=self.auth) response.raise_for_status() except (HTTPError, requests.exceptions.HTTPError) as err: - logger.error(f'HTTP error occurred: {http_err}') + logger.error(f'HTTP error occurred: {err}') return None except Exception as err: logger.error(f'Other error occurred: {err}') diff --git a/dts/database.py b/dts/database.py index 5b29b12..6478edf 100644 --- a/dts/database.py +++ b/dts/database.py @@ -2,7 +2,12 @@ @dataclass(slots = True) class Database(object): - """`Database` - A database storing files that can be selected and transferred""" + """`Database` - A database storing files that can be selected and transferred. + +This type holds human-readable information about databases available to DTS. +Objects of this type are returned by calls to the DTS API, so it is not +necessary to construct them directly. +""" id: str name: str organization: str diff --git a/dts/transfer_status.py b/dts/transfer_status.py index c2dd588..e89422d 100644 --- a/dts/transfer_status.py +++ b/dts/transfer_status.py @@ -3,7 +3,12 @@ @dataclass(slots = True) class TransferStatus(object): - """`TransferStatus` - holds status information for a file transfer""" + """`TransferStatus` status information for a file transfer. + +This type holds information pertaining to the transfer of a payload initiated +via the DTS. Objects of this type are returned by calls to the DTS API, so it +is not necessary to create them directly. +""" id: str status: str message: Optional[str] From f035daedaf78b1bca3c09824dc924064572ab44e Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Mon, 14 Oct 2024 12:06:07 -0700 Subject: [PATCH 04/13] Added mkdocs documentation. --- docs/index.md | 2 ++ dts/client.py | 37 +++++++++++++++++-------------------- mkdocs.yml | 11 +++++++++++ 3 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 docs/index.md create mode 100644 mkdocs.yml diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..00359b3 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,2 @@ +# dtspy: a Python client for the [Data Transfer System](https://kbase.github.io/dts/) + diff --git a/dts/client.py b/dts/client.py index b940c5f..69ad395 100644 --- a/dts/client.py +++ b/dts/client.py @@ -131,9 +131,9 @@ def search(self, database: str, query: str | int | float, status: str | None, - offset = 0, - limit = None, - specific = None, + offset: int = 0, + limit: int | None = None, + specific: dict[str, Any] | None = None, ) -> list[JsonResource]: """Performs a synchronous search of the database with the given name using the given query string. @@ -148,8 +148,7 @@ def search(self, specific: An optional dictionary mapping database-specific search parameters to their values. Returns: - A list of [frictionless DataResources](https://specs.frictionlessdata.io/data-resource/) - containing metadata for files matching the query. + A list of [frictionless DataResources](https://specs.frictionlessdata.io/data-resource/) containing metadata for files matching the query. Raises: RuntimeError: Indicates an issue with the DTS client and its connection to the server. @@ -221,8 +220,7 @@ def fetch_metadata(self, limit: An optional pagination parameter indicating the maximum number of results to retrieve. Returns: - A list of [frictionless DataResources](https://specs.frictionlessdata.io/data-resource/) - containing metadata for files with the requested IDs. + A list of [frictionless DataResources](https://specs.frictionlessdata.io/data-resource/) containing metadata for files with the requested IDs. Raises: RuntimeError: Indicates an issue with the DTS client and its connection to the server. @@ -284,8 +282,7 @@ def transfer(self, timeout: An optional integer indicating the number of seconds to wait for a response from the server. Returns: - A UUID uniquely identifying the file transfer that can be used to check its - status, or None if a server-side error is encountered. + A UUID uniquely identifying the file transfer that can be used to check its status, or None if a server-side error is encountered. Raises: RuntimeError: Indicates an issue with the DTS client and its connection to the server. @@ -333,22 +330,22 @@ def transfer_status(self, id: uuid.UUID) -> TransferStatus | None: """Returns status information for the transfer with the given identifier. -Server-side errors are intercepted and logged. +Server-side errors are intercepted and logged. Possible transfer statuses are: + +* `'staging'`: The files requested for transfer are being copied to the staging + area for the source database job. +* `'active'`: The files are being transferred from the source database to the + destination database. +* `'finalizing'`: The files have been transferred and a manifest is being written. +* `'inactive'`: The file transfer has been suspended. +* `'failed'`: The file transfer could not be completed because of a failure. +* `'unknown'`: The status of the given transfer is unknown. Arguments: id: A UUID that uniquely identifies the transfer operation for which the status is requested. Returns: - A `TransferStatus` object whose contents indicate the status of the transfer, - or None if a server-side error occurs. Possible statuses are: - * `'staging'`: the files requested for transfer are being copied to the staging - area for the source database job - * `'active'`: the files are being transferred from the source database to the - destination database - * `'finalizing'`: the files have been transferred and a manifest is being written - * `'inactive'`: the file transfer has been suspended - * `'failed'`: the file transfer could not be completed because of a failure` - * `'unknown'`: the status of the given transfer is unknown + A `TransferStatus` object whose contents indicate the status of the transfer, or None if a server-side error occurs. Raises: RuntimeError: Indicates an issue with the DTS client and its connection to the server. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..92f10bd --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,11 @@ +site_name: dtspy - A Python client for the Data Transfer Service + +theme: + name: "material" + +plugins: + - mkdocstrings + +nav: + - dtspy: index.md + - API reference: api.md From 310109bbe94f46947d8ee03838745728f76da541 Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Mon, 14 Oct 2024 12:19:24 -0700 Subject: [PATCH 05/13] (Re?-)added coverage module to dependencies. --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 78c15c2..5d36b0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ chardet==5.2.0 charset-normalizer==3.3.2 click==8.1.7 colorama==0.4.6 +coverage==7.6.3 frictionless==5.17.0 ghp-import==2.1.0 griffe==1.4.1 From 4d7bd577e462cd4937fea0ca960b365ecc3f5db5 Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Mon, 14 Oct 2024 12:56:49 -0700 Subject: [PATCH 06/13] Making a search filter optional. --- dts/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dts/client.py b/dts/client.py index 69ad395..60cc76a 100644 --- a/dts/client.py +++ b/dts/client.py @@ -130,7 +130,7 @@ def databases(self) -> list[Database]: def search(self, database: str, query: str | int | float, - status: str | None, + status: str | None = None, offset: int = 0, limit: int | None = None, specific: dict[str, Any] | None = None, From 84caf3801781379fb42da8389f028c64ef812db6 Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Mon, 14 Oct 2024 12:59:31 -0700 Subject: [PATCH 07/13] Fixing another oversight. --- dts/client.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dts/client.py b/dts/client.py index 60cc76a..0dbfb76 100644 --- a/dts/client.py +++ b/dts/client.py @@ -163,8 +163,6 @@ def search(self, query = str(query) else: raise TypeError('search: query must be a string or a number.') - else: - raise RuntimeError('search: missing query.') if not isinstance(database, str): raise TypeError('search: database must be a string.') params: dict[str, Any] = { From c8de7731abc641a80d6ed5bd955f1a2ba62af852 Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Mon, 14 Oct 2024 13:35:46 -0700 Subject: [PATCH 08/13] Added GitHub Pages integration for PRs. --- .github/workflows/gh-pages.yml | 54 ++++++++++++++++++++++++++++++++++ mkdocs.yml | 2 +- 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/gh-pages.yml diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml new file mode 100644 index 0000000..6891211 --- /dev/null +++ b/.github/workflows/gh-pages.yml @@ -0,0 +1,54 @@ +name: Build and deploy gh-pages branch with Mkdocs + +on: + push: + # Runs only if documentation is changed + paths: + - 'mkdocs.yml' + - 'docs/**' + # Runs every time main branch is updated + branches: ["main"] + # Runs every time a PR is open against main + pull_request: + branches: ["main"] + workflow_dispatch: + +concurrency: + # Prevent 2+ copies of this workflow from running concurrently + group: dts-docs-action + +jobs: + Build-and-Deploy-docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + show-progress: false + fetch-depth: 0 # Needed, or else gh-pages won't be fetched, and push rejected + submodules: false # speeds up clone and not building anything in submodules + - name: Show action trigger + run: echo "= The job was automatically triggered by a ${{github.event_name}} event." + - name: Set up Python 3.10 + uses: actions/setup-python@v4.7.0 + with: + python-version: "3.10" + - name: Install python deps + run: python3 -m pip install mkdocs-material pymdown-extensions mkdocs-monorepo-plugin mdutils + # build every time (PR or push to main) + - name: Build + run: mkdocs build --strict --verbose + # deploy only when it is a push + - if: ${{ github.event_name == 'push' }} + name: GitHub Pages action + uses: JamesIves/github-pages-deploy-action@v4 + with: + # Do not remove existing pr-preview pages + clean-exclude: pr-preview + folder: ./site/ + # If it's a PR from within the same repo, deploy to a preview page + # For security reasons, PRs from forks cannot write into gh-pages for now + - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} + name: Preview docs + uses: rossjrw/pr-preview-action@v1 + with: + source-dir: ./site/ diff --git a/mkdocs.yml b/mkdocs.yml index 92f10bd..6f3daf8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,5 +7,5 @@ plugins: - mkdocstrings nav: - - dtspy: index.md + - Overview: index.md - API reference: api.md From 434a4a8f6d0ea73bb545680c55ab38c8a821a25d Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Mon, 14 Oct 2024 13:39:42 -0700 Subject: [PATCH 09/13] Fixing mkdocs deployment dependencies. --- .github/workflows/gh-pages.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 6891211..cbd0b45 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -26,18 +26,21 @@ jobs: show-progress: false fetch-depth: 0 # Needed, or else gh-pages won't be fetched, and push rejected submodules: false # speeds up clone and not building anything in submodules + - name: Show action trigger run: echo "= The job was automatically triggered by a ${{github.event_name}} event." - - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.0 + + - name: Setting up Python 3.12 + uses: actions/setup-python@v5 with: - python-version: "3.10" - - name: Install python deps - run: python3 -m pip install mkdocs-material pymdown-extensions mkdocs-monorepo-plugin mdutils - # build every time (PR or push to main) + python-version: "3.12" + + - name: Installing dtspy dependencies (${{ matrix.os }}) + run: python3 -m pip install -r requirements.txt + - name: Build run: mkdocs build --strict --verbose - # deploy only when it is a push + - if: ${{ github.event_name == 'push' }} name: GitHub Pages action uses: JamesIves/github-pages-deploy-action@v4 @@ -45,6 +48,7 @@ jobs: # Do not remove existing pr-preview pages clean-exclude: pr-preview folder: ./site/ + # If it's a PR from within the same repo, deploy to a preview page # For security reasons, PRs from forks cannot write into gh-pages for now - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }} From 3f9de7807d3d15b99b1aac0859f7a96f6249fa46 Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Mon, 14 Oct 2024 13:41:08 -0700 Subject: [PATCH 10/13] Added omitted documentation file. --- docs/api.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/api.md diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..8947612 --- /dev/null +++ b/docs/api.md @@ -0,0 +1 @@ +::: dts From 29af53d9bd41d129c176a9d95683312a33e8d970 Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Mon, 14 Oct 2024 14:41:39 -0700 Subject: [PATCH 11/13] Not building documentation with strict flags. --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index cbd0b45..74a3f45 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -39,7 +39,7 @@ jobs: run: python3 -m pip install -r requirements.txt - name: Build - run: mkdocs build --strict --verbose + run: mkdocs build --verbose - if: ${{ github.event_name == 'push' }} name: GitHub Pages action From 903333c9c635b3be9d66b3df0b82eee7e5079744 Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Wed, 16 Oct 2024 11:04:53 -0700 Subject: [PATCH 12/13] Type hinting fixes. --- dts/client.py | 27 +++++++++++++++------------ dts/database.py | 2 +- dts/transfer_status.py | 2 +- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/dts/client.py b/dts/client.py index 0dbfb76..2976609 100644 --- a/dts/client.py +++ b/dts/client.py @@ -18,10 +18,10 @@ class KBaseAuth(AuthBase): """Attaches a KBase-sensible Authorization header to the given Request object.""" - def __init__(self, api_key: str): + def __init__(self: "KBaseAuth", api_key: str): self.api_key = api_key - def __call__(self, request): + def __call__(self: "KBaseAuth", request): b64_token = base64.b64encode(bytes(self.api_key + '\n', 'utf-8')) token = b64_token.decode('utf-8') request.headers['Authorization'] = f'Bearer {token}' @@ -33,7 +33,7 @@ class Client: This type exposes the [DTS API](https://lb-dts.staging.kbase.us/docs#/) for use in Python programs. """ - def __init__(self, + def __init__(self: "Client", api_key: str | None = None, server: str | None = None, port: int | None = None): @@ -60,7 +60,7 @@ def __init__(self, self.name = None self.version = None - def connect(self, + def connect(self: "Client", api_key: str | None = None, server: str | None = None, port: int | None = None) -> None: @@ -74,6 +74,7 @@ def connect(self, Raises: TypeError: an argument of improper type was specified. + urllib3.exceptions.ConnectionError: the client was unable to connect to the DTS server. """ if not isinstance(api_key, str): raise TypeError('api_key must be an unencoded API key.') @@ -94,7 +95,7 @@ def connect(self, self.name = result['name'] self.version = result['version'] - def disconnect(self) -> None: + def disconnect(self: "Client") -> None: """Disconnects the client from the server. """ self.api_key = None @@ -102,7 +103,7 @@ def disconnect(self) -> None: self.name = None self.version = None - def databases(self) -> list[Database]: + def databases(self: "Client") -> list[Database]: """Returns all databases available to the service. Server-side errors are captured and logged. @@ -127,7 +128,7 @@ def databases(self) -> list[Database]: organization = r['organization'], url = r['url']) for r in results] - def search(self, + def search(self: "Client", database: str, query: str | int | float, status: str | None = None, @@ -202,7 +203,7 @@ def search(self, return [] return [JsonResource(r) for r in response.json()['resources']] - def fetch_metadata(self, + def fetch_metadata(self: "Client", database: str, ids: list[str], offset: int = 0, @@ -260,7 +261,7 @@ def fetch_metadata(self, return [] return [JsonResource(r) for r in response.json()['resources']] - def transfer(self, + def transfer(self: "Client", file_ids: list[str], source: str, destination: str, @@ -324,7 +325,8 @@ def transfer(self, return None return uuid.UUID(response.json()["id"]) - def transfer_status(self, id: uuid.UUID) -> TransferStatus | None: + def transfer_status(self: "Client", + id: uuid.UUID) -> TransferStatus | None: """Returns status information for the transfer with the given identifier. @@ -371,7 +373,8 @@ def transfer_status(self, id: uuid.UUID) -> TransferStatus | None: num_files_transferred = results.get('num_files_transferred'), ) - def cancel_transfer(self, id: uuid.UUID): + def cancel_transfer(self: "Client", + id: uuid.UUID): """Cancels a file transfer with the requested UUID. Status information for the cancelled transfer is retained for a time so its @@ -399,7 +402,7 @@ def cancel_transfer(self, id: uuid.UUID): return None return None - def __repr__(self): + def __repr__(self: "Client"): if self.uri: return f""" dts.Client(uri = {self.uri}, diff --git a/dts/database.py b/dts/database.py index 6478edf..255109b 100644 --- a/dts/database.py +++ b/dts/database.py @@ -1,7 +1,7 @@ from dataclasses import dataclass @dataclass(slots = True) -class Database(object): +class Database: """`Database` - A database storing files that can be selected and transferred. This type holds human-readable information about databases available to DTS. diff --git a/dts/transfer_status.py b/dts/transfer_status.py index e89422d..756ba23 100644 --- a/dts/transfer_status.py +++ b/dts/transfer_status.py @@ -2,7 +2,7 @@ from typing import Optional @dataclass(slots = True) -class TransferStatus(object): +class TransferStatus: """`TransferStatus` status information for a file transfer. This type holds information pertaining to the transfer of a payload initiated From a2fb12f2ebc75d4037d574574b9c1cabbff5358d Mon Sep 17 00:00:00 2001 From: "Jeffrey N. Johnson" Date: Wed, 16 Oct 2024 11:44:46 -0700 Subject: [PATCH 13/13] Adding a few more type hints. --- dts/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dts/client.py b/dts/client.py index 2976609..89eabf4 100644 --- a/dts/client.py +++ b/dts/client.py @@ -18,7 +18,7 @@ class KBaseAuth(AuthBase): """Attaches a KBase-sensible Authorization header to the given Request object.""" - def __init__(self: "KBaseAuth", api_key: str): + def __init__(self: "KBaseAuth", api_key: str) -> None: self.api_key = api_key def __call__(self: "KBaseAuth", request): @@ -36,7 +36,7 @@ class Client: def __init__(self: "Client", api_key: str | None = None, server: str | None = None, - port: int | None = None): + port: int | None = None) -> None: """Creates a DTS client that handles search and transfer requests via a connected server. @@ -374,7 +374,7 @@ def transfer_status(self: "Client", ) def cancel_transfer(self: "Client", - id: uuid.UUID): + id: uuid.UUID) -> None: """Cancels a file transfer with the requested UUID. Status information for the cancelled transfer is retained for a time so its @@ -402,7 +402,7 @@ def cancel_transfer(self: "Client", return None return None - def __repr__(self: "Client"): + def __repr__(self: "Client") -> str: if self.uri: return f""" dts.Client(uri = {self.uri},