From 4a4909b2dbd3c9e45a5127547d26d52f0970e06a Mon Sep 17 00:00:00 2001 From: saileshwar-skyflow Date: Mon, 21 Oct 2024 01:29:42 +0530 Subject: [PATCH] SK-1649: Added validations and logging --- skyflow/client/skyflow.py | 120 +++--- skyflow/error/__init__.py | 2 +- skyflow/error/_skyflow_error.py | 15 +- skyflow/service_account/__init__.py | 2 +- skyflow/service_account/_utils.py | 50 +-- skyflow/utils/__init__.py | 6 +- skyflow/utils/_log_helpers.py | 27 ++ skyflow/utils/_logger.py | 43 ++ skyflow/utils/_skyflow_messages.py | 160 ++++++- skyflow/utils/_utils.py | 244 ++++++++++- skyflow/utils/_version.py | 1 + skyflow/utils/enums/__init__.py | 7 +- skyflow/utils/enums/batch_method.py | 8 + skyflow/utils/enums/interface_name.py | 15 + skyflow/utils/enums/log_level.py | 10 +- skyflow/utils/enums/order_by.py | 6 + skyflow/utils/enums/redaction_type.py | 10 +- skyflow/utils/enums/token_strict.py | 6 + skyflow/utils/validations/__init__.py | 12 +- skyflow/utils/validations/_validations.py | 404 +++++++++++++++++- skyflow/vault/client/client.py | 36 +- .../connection/_invoke_connection_request.py | 13 +- .../connection/_invoke_connection_response.py | 12 +- skyflow/vault/controller/_connections.py | 32 +- skyflow/vault/controller/_vault.py | 296 ++++++++----- skyflow/vault/data/_delete_response.py | 7 +- skyflow/vault/data/_get_request.py | 16 +- skyflow/vault/data/_get_response.py | 14 +- skyflow/vault/data/_insert_request.py | 14 +- skyflow/vault/data/_insert_response.py | 27 +- skyflow/vault/data/_query_response.py | 13 +- skyflow/vault/data/_update_request.py | 9 +- skyflow/vault/data/_update_response.py | 11 +- skyflow/vault/tokens/_detokenize_request.py | 7 +- skyflow/vault/tokens/_detokenize_response.py | 25 +- skyflow/vault/tokens/_tokenize_response.py | 12 +- 36 files changed, 1321 insertions(+), 371 deletions(-) create mode 100644 skyflow/utils/_log_helpers.py create mode 100644 skyflow/utils/_logger.py create mode 100644 skyflow/utils/_version.py create mode 100644 skyflow/utils/enums/batch_method.py create mode 100644 skyflow/utils/enums/interface_name.py create mode 100644 skyflow/utils/enums/order_by.py create mode 100644 skyflow/utils/enums/token_strict.py diff --git a/skyflow/client/skyflow.py b/skyflow/client/skyflow.py index 626b649..ce1be52 100644 --- a/skyflow/client/skyflow.py +++ b/skyflow/client/skyflow.py @@ -1,7 +1,9 @@ from collections import OrderedDict from skyflow import LogLevel from skyflow.error import SkyflowError -from skyflow.utils.validations import validate_vault_config, validate_connection_config +from skyflow.utils import Logger, log_info, SkyflowMessages, log_error +from skyflow.utils.validations import validate_vault_config, validate_connection_config, validate_update_vault_config, \ + validate_update_connection_config, validate_credentials, validate_log_level from skyflow.vault.client.client import VaultClient from skyflow.vault.controller import Vault from skyflow.vault.controller import Connection @@ -9,6 +11,9 @@ class Skyflow: def __init__(self, builder): self.__builder = builder + log_info(SkyflowMessages.Info.CLIENT_INITIALIZED.value, + SkyflowMessages.InterfaceName.CLIENT.value, + self.__builder.get_logger()) @staticmethod def builder(): @@ -72,94 +77,107 @@ def connection(self, connection_id = None): class Builder: def __init__(self): self.__vault_configs = OrderedDict() + self.__vault_list = list() self.__connection_configs = OrderedDict() + self.__connection_list = list() self.__skyflow_credentials = None - self.__log_level = LogLevel.ERROR + self.__log_level = LogLevel.OFF + self.__logger = Logger(LogLevel.OFF) def add_vault_config(self, config): - if validate_vault_config(config) and config.get("vault_id") not in self.__vault_configs.keys(): - vault_id = config.get("vault_id") - vault_client = VaultClient(config) - self.__vault_configs[vault_id] = { - "vault_client": vault_client, - "controller": Vault(vault_client) - } - return self - else: - raise SkyflowError(f"Vault config with id {config['vault_id']} already exists") + self.__vault_list.append(config) + return self def remove_vault_config(self, vault_id): if vault_id in self.__vault_configs.keys(): self.__vault_configs.pop(vault_id) else: - raise SkyflowError(f"Vault config with id {vault_id} not found") + log_error(SkyflowMessages.Error.INVALID_VAULT_ID.value, + SkyflowMessages.ErrorCodes.INVALID_INPUT.value, + logger = self.__logger) def update_vault_config(self, config): + validate_update_vault_config(self.__logger, config) vault_id = config.get("vault_id") - if not vault_id: - raise SkyflowError("vault_id is required and cannot be None") - if vault_id in self.__vault_configs.keys(): - vault_config = self.__vault_configs[vault_id] - vault_config.get("vault_client").update_config(config) - else: - raise SkyflowError(f"Vault config with id {vault_id} not found") + vault_config = self.__vault_configs[vault_id] + vault_config.get("vault_client").update_config(config) def get_vault_config(self, vault_id): if vault_id in self.__vault_configs.keys(): vault_config = self.__vault_configs.get(vault_id) return vault_config - raise SkyflowError(f"Vault config with id {vault_id} not found") + raise SkyflowError(SkyflowMessages.Error.VAULT_ID_NOT_IN_CONFIG_LIST.value.format(vault_id), SkyflowMessages.ErrorCodes.INVALID_INPUT.value, logger = self.__logger, logger_method=log_error) def add_connection_config(self, config): - if validate_connection_config(config) and config["connection_id"] not in self.__connection_configs.keys(): - connection_id = config.get("connection_id") - vault_client = VaultClient(config) - self.__connection_configs[connection_id] = { - "vault_client": vault_client, - "controller": Connection(vault_client) - } - return self - else: - raise SkyflowError(f"Connection config with id {config['connection_id']} already exists") + self.__connection_list.append(config) + return self def remove_connection_config(self, connection_id): if connection_id in self.__connection_configs.keys(): self.__connection_configs.pop(connection_id) else: - raise SkyflowError(f"Connection config with id {connection_id} not found") + log_error(SkyflowMessages.Error.INVALID_CONNECTION_ID.value, + SkyflowMessages.ErrorCodes.INVALID_INPUT.value, + logger = self.__logger) def update_connection_config(self, config): + validate_update_connection_config(self.__logger, config) connection_id = config['connection_id'] - if not connection_id: - raise SkyflowError("connection_id is required and can not be empty") - - if connection_id in self.__connection_configs.keys(): - connection_config = self.__connection_configs[connection_id] - connection_config.get("vault_client").update_config(config) - else: - raise SkyflowError(f"Connection config with id {connection_id} not found") + connection_config = self.__connection_configs[connection_id] + connection_config.get("vault_client").update_config(config) def get_connection_config(self, connection_id): if connection_id in self.__connection_configs.keys(): connection_config = self.__connection_configs[connection_id] return connection_config - raise SkyflowError(f"Connection config with id {connection_id} not found") + raise SkyflowError(SkyflowMessages.Error.CONNECTION_ID_NOT_IN_CONFIG_LIST.value.format(connection_id), SkyflowMessages.ErrorCodes.INVALID_INPUT.value, logger = self.__logger, logger_method=log_error) def add_skyflow_credentials(self, credentials): - for vault_id, vault_config in self.__vault_configs.items(): - vault_config.get("vault_client").set_common_skyflow_credentials(credentials) - - for connection_id, connection_config in self.__connection_configs.items(): - connection_config.get("vault_client").set_common_skyflow_credentials(credentials) + self.__skyflow_credentials = credentials return self def set_log_level(self, log_level): + self.__log_level = log_level + return self + + def get_logger(self): + return self.__logger + + def build(self): + log_info(SkyflowMessages.Info.INITIALIZE_CLIENT.value, SkyflowMessages.InterfaceName.CLIENT.value, self.__logger) + validate_log_level(self.__logger, self.__log_level) + self.__logger.set_log_level(self.__log_level) + + for config in self.__vault_list: + validate_vault_config(self.__logger, config) + vault_id = config.get("vault_id") + vault_client = VaultClient(config) + self.__vault_configs[vault_id] = { + "vault_client": vault_client, + "controller": Vault(vault_client) + } + + for config in self.__connection_list: + validate_connection_config(self.__logger, config=config) + connection_id = config.get("connection_id") + vault_client = VaultClient(config) + self.__connection_configs[connection_id] = { + "vault_client": vault_client, + "controller": Connection(vault_client) + } + for vault_id, vault_config in self.__vault_configs.items(): - vault_config.get("vault_client").set_log_level(log_level) + vault_config.get("vault_client").set_logger(self.__log_level, self.__logger) for connection_id, connection_config in self.__connection_configs.items(): - connection_config.get("vault_client").set_log_level(log_level) - return self + connection_config.get("vault_client").set_logger(self.__log_level, self.__logger) - def build(self): - return Skyflow(self) \ No newline at end of file + if self.__skyflow_credentials is not None: + validate_credentials(self.__logger, self.__skyflow_credentials) + for vault_id, vault_config in self.__vault_configs.items(): + vault_config.get("vault_client").set_common_skyflow_credentials(self.__skyflow_credentials) + + for connection_id, connection_config in self.__connection_configs.items(): + connection_config.get("vault_client").set_common_skyflow_credentials(self.__skyflow_credentials) + + return Skyflow(self) diff --git a/skyflow/error/__init__.py b/skyflow/error/__init__.py index 45d7424..305c796 100644 --- a/skyflow/error/__init__.py +++ b/skyflow/error/__init__.py @@ -1 +1 @@ -from skyflow.error._skyflow_error import SkyflowError \ No newline at end of file +from ._skyflow_error import SkyflowError \ No newline at end of file diff --git a/skyflow/error/_skyflow_error.py b/skyflow/error/_skyflow_error.py index 8ea7e46..5c52f26 100644 --- a/skyflow/error/_skyflow_error.py +++ b/skyflow/error/_skyflow_error.py @@ -1,4 +1,13 @@ class SkyflowError(Exception): - def __init__(self, message): - super().__init__(message) - self.message = message \ No newline at end of file + def __init__(self, + message, + http_code, + request_id = None, + grpc_code = None, + http_status = None, + details = None, + logger = None, + logger_method = None): + + logger_method(message, http_code, request_id, grpc_code, http_status, details, logger) + super().__init__() \ No newline at end of file diff --git a/skyflow/service_account/__init__.py b/skyflow/service_account/__init__.py index 794ef18..b5c1919 100644 --- a/skyflow/service_account/__init__.py +++ b/skyflow/service_account/__init__.py @@ -1 +1 @@ -from ._utils import generate_bearer_token, generate_bearer_token_from_creds \ No newline at end of file +from ._utils import generate_bearer_token, generate_bearer_token_from_creds, is_expired \ No newline at end of file diff --git a/skyflow/service_account/_utils.py b/skyflow/service_account/_utils.py index 3d2a682..71141ea 100644 --- a/skyflow/service_account/_utils.py +++ b/skyflow/service_account/_utils.py @@ -5,9 +5,11 @@ from skyflow.error import SkyflowError from skyflow.generated.rest.models import V1GetAuthTokenRequest from skyflow.service_account.client.auth_client import AuthClient -from skyflow.utils import get_base_url, format_scope +from skyflow.utils import get_base_url, format_scope, SkyflowMessages, log_error -def is_expired(token): +invalid_input_error_code = SkyflowMessages.ErrorCodes.INVALID_INPUT.value + +def is_expired(token, logger = None): if len(token) == 0: return True @@ -18,54 +20,54 @@ def is_expired(token): return False except jwt.ExpiredSignatureError: return True - except Exception as e: - SkyflowError("Invalid token") + except Exception: + log_error(SkyflowMessages.Error.JWT_DECODE_ERROR.value, invalid_input_error_code, logger = logger) return True pass -def generate_bearer_token(credentials_file_path, options = None): +def generate_bearer_token(credentials_file_path, options = None, logger = None): try: credentials_file =open(credentials_file_path, 'r') except Exception: - raise SkyflowError("Invalid file path") + raise SkyflowError(SkyflowMessages.Error.INVALID_CREDENTIAL_FILE_PATH.value, invalid_input_error_code, logger = logger, logger_method=log_error) try: credentials = json.load(credentials_file) except Exception: - raise SkyflowError("Error in json parsing") + raise SkyflowError(SkyflowMessages.Error.FILE_INVALID_JSON.value.format(credentials_file_path), invalid_input_error_code, logger = logger, logger_method=log_error) finally: credentials_file.close() - result = get_service_account_token(credentials, options) + result = get_service_account_token(credentials, options, logger) return result -def generate_bearer_token_from_creds(credentials, options = None): +def generate_bearer_token_from_creds(credentials, options = None, logger = None): try: json_credentials = json.loads(credentials.replace('\n', '\\n')) except Exception as e: - raise SkyflowError(e) - result = get_service_account_token(json_credentials, options) + raise SkyflowError(SkyflowMessages.Error.FILE_INVALID_JSON.value, invalid_input_error_code, logger = logger, logger_method=log_error) + result = get_service_account_token(json_credentials, options, logger) return result -def get_service_account_token(credentials, options): +def get_service_account_token(credentials, options, logger): try: private_key = credentials["privateKey"] except: - raise SkyflowError("privateKey not found") + raise SkyflowError(SkyflowMessages.Error.MISSING_PRIVATE_KEY.value, invalid_input_error_code, logger = logger, logger_method=log_error) try: client_id = credentials["clientID"] except: - raise SkyflowError("clientID not found") + raise SkyflowError(SkyflowMessages.Error.MISSING_CLIENT_ID.value, invalid_input_error_code, logger = logger, logger_method=log_error) try: key_id = credentials["keyID"] except: - raise SkyflowError("keyID not found") + raise SkyflowError(SkyflowMessages.Error.MISSING_KEY_ID.value, invalid_input_error_code, logger = logger, logger_method=log_error) try: token_uri = credentials["tokenURI"] except: - raise SkyflowError("tokenURI not found") + raise SkyflowError(SkyflowMessages.Error.MISSING_TOKEN_URI.value, invalid_input_error_code, logger = logger, logger_method=log_error) - signed_token = get_signed_jwt(options, client_id, key_id, token_uri, private_key) + signed_token = get_signed_jwt(options, client_id, key_id, token_uri, private_key, logger) base_url = get_base_url(token_uri) auth_client = AuthClient(base_url) auth_api = auth_client.get_auth_api() @@ -80,7 +82,7 @@ def get_service_account_token(credentials, options): response = auth_api.authentication_service_get_auth_token(request) return response.access_token, response.token_type -def get_signed_jwt(options, client_id, key_id, token_uri, private_key): +def get_signed_jwt(options, client_id, key_id, token_uri, private_key, logger): payload = { "iss": client_id, "key": key_id, @@ -93,7 +95,7 @@ def get_signed_jwt(options, client_id, key_id, token_uri, private_key): try: return jwt.encode(payload=payload, key=private_key, algorithm="RS256") except Exception as e: - raise SkyflowError("") + raise SkyflowError(SkyflowMessages.Error.JWT_INVALID_FORMAT.value, invalid_input_error_code, logger = logger, logger_method=log_error) @@ -102,7 +104,7 @@ def get_signed_tokens(credentials, options): try: credentials_obj = json.loads(credentials) except: - raise SkyflowError("Invalid JSON") + raise ValueError("Invalid JSON") expiry_time = time.time() + options.get("time_to_live", 60) prefix = "signed_token_" @@ -130,16 +132,16 @@ def get_signed_tokens(credentials, options): return response_array except Exception as e: - raise SkyflowError(str(e)) + raise ValueError(str(e)) -def generate_signed_data_tokens(credentials_file_path, options): +def generate_signed_data_tokens(credentials_file_path, options, logger = None): try: credentials_file =open(credentials_file_path, 'r') except Exception: - raise SkyflowError("Invalid file path") + raise SkyflowError(SkyflowMessages.Error.INVALID_CREDENTIAL_FILE_PATH.value, invalid_input_error_code, logger = logger, logger_method=log_error) - return get_signed_tokens(credentials_file_path, options) + return get_signed_tokens(credentials_file, options) def generate_signed_data_tokens_from_creds(credentials, options): return get_signed_tokens(credentials, options) diff --git a/skyflow/utils/__init__.py b/skyflow/utils/__init__.py index 7bec711..3c8cfbf 100644 --- a/skyflow/utils/__init__.py +++ b/skyflow/utils/__init__.py @@ -1,2 +1,6 @@ from ..utils.enums import LogLevel, Env -from ._utils import get_credentials, get_vault_url, get_client_configuration, get_base_url, format_scope, get_redaction_type, construct_invoke_connection_request, build_field_records \ No newline at end of file +from ._skyflow_messages import SkyflowMessages +from ._version import SDK_VERSION +from ._logger import Logger +from ._log_helpers import log_error, log_info +from ._utils import get_credentials, get_vault_url, get_client_configuration, get_base_url, format_scope, get_redaction_type, construct_invoke_connection_request, get_metrics, parse_insert_response, handle_exception, parse_update_record_response, parse_delete_response, parse_detokenize_response, parse_tokenize_response, parse_query_response, parse_get_response \ No newline at end of file diff --git a/skyflow/utils/_log_helpers.py b/skyflow/utils/_log_helpers.py new file mode 100644 index 0000000..bf4b6af --- /dev/null +++ b/skyflow/utils/_log_helpers.py @@ -0,0 +1,27 @@ +from .enums import LogLevel +from . import Logger + + +def log_info(message, interface, logger = None): + formatted_message = '{} {}'.format(interface, message) + logger.info(formatted_message) + +def log_error(message, http_code, request_id=None, grpc_code=None, http_status=None, details=None, logger=None): + if not logger: + logger = Logger(LogLevel.ERROR) + + log_data = { + 'http_code': http_code, + 'message': message + } + + if grpc_code is not None: + log_data['grpc_code'] = grpc_code + if http_status is not None: + log_data['http_status'] = http_status + if request_id is not None: + log_data['request_id'] = request_id + if details is not None: + log_data['details'] = details + + logger.error(log_data) \ No newline at end of file diff --git a/skyflow/utils/_logger.py b/skyflow/utils/_logger.py new file mode 100644 index 0000000..0827abd --- /dev/null +++ b/skyflow/utils/_logger.py @@ -0,0 +1,43 @@ +import logging +from .enums.log_level import LogLevel + + +class Logger: + def __init__(self, level=LogLevel.ERROR): + self.current_level = level + self.logger = logging.getLogger('skyflow-python') + self.logger.propagate = False # Prevent logs from being handled by parent loggers + self.set_log_level(level) + + if not self.logger.hasHandlers(): + handler = logging.StreamHandler() + formatter = logging.Formatter('%(levelname)s - %(message)s') + handler.setFormatter(formatter) + self.logger.addHandler(handler) + + def set_log_level(self, level): + self.current_level = level + log_level_mapping = { + LogLevel.DEBUG: logging.DEBUG, + LogLevel.INFO: logging.INFO, + LogLevel.WARN: logging.WARNING, + LogLevel.ERROR: logging.ERROR, + LogLevel.OFF: logging.CRITICAL + 1 + } + self.logger.setLevel(log_level_mapping[level]) + + def debug(self, message): + if self.current_level.value <= LogLevel.DEBUG.value: + self.logger.debug(message) + + def info(self, message): + if self.current_level.value <= LogLevel.INFO.value: + self.logger.info(message) + + def warn(self, message): + if self.current_level.value <= LogLevel.WARN.value: + self.logger.warning(message) + + def error(self, message): + if self.current_level.value <= LogLevel.ERROR.value: + self.logger.error(message) \ No newline at end of file diff --git a/skyflow/utils/_skyflow_messages.py b/skyflow/utils/_skyflow_messages.py index 9d5b60c..efbdd8a 100644 --- a/skyflow/utils/_skyflow_messages.py +++ b/skyflow/utils/_skyflow_messages.py @@ -1,11 +1,167 @@ from enum import Enum +from ._version import SDK_VERSION + +error_prefix = f"Skyflow Node SDK {SDK_VERSION}" + class SkyflowMessages: + class ErrorCodes(Enum): + INVALID_INPUT = 400 + INVALID_INDEX = 404 + SERVER_ERROR = 500 + PARTIAL_SUCCESS = 500 + TOKENS_GET_COLUMN_NOT_SUPPORTED = 400 + REDACTION_WITH_TOKENS_NOT_SUPPORTED = 400 + class Error(Enum): - ERROR_MESSAGE = "ERROR MESSAGE" + EMPTY_VAULT_ID = f"{error_prefix} Initialization failed. Invalid vault Id. Specify a valid vault Id." + INVALID_VAULT_ID = f"{error_prefix} Initialization failed. Invalid vault Id. Specify a valid vault Id as a string." + EMPTY_CLUSTER_ID = f"{error_prefix} Initialization failed. Invalid cluster Id. Specify a valid cluster Id." + INVALID_CLUSTER_ID = f"{error_prefix} Initialization failed. Invalid cluster Id. Specify cluster Id as a string." + INVALID_ENV = f"{error_prefix} Initialization failed. Invalid env. Specify a valid env." + INVALID_KEY = f"{error_prefix} Initialization failed. Invalid {{}}. Specify a valid key" + VAULT_ID_NOT_IN_CONFIG_LIST = f"{error_prefix} Validation error. {{}} is missing from the config. Specify the vaultId's from config." + + EMPTY_CREDENTIALS = f"{error_prefix} Validation error. Invalid credentials. Credentials must not be empty." + INVALID_CREDENTIALS = f"{error_prefix} Validation error. Invalid credentials. Specify a valid credentials." + MULTIPLE_CREDENTIALS_PASSED = f"{error_prefix} Validation error. Multiple credentials provided. Please specify only one valid credential." + EMPTY_CREDENTIALS_STRING = f"{error_prefix} Validation error. Invalid credentials. Specify valid credentials." + INVALID_CREDENTIALS_STRING = f"{error_prefix} Validation error. Invalid credentials. Specify credentials as a string." + EMPTY_CREDENTIAL_FILE_PATH = f"{error_prefix} Initialization failed. Invalid credentials. Specify a valid file path." + INVALID_CREDENTIAL_FILE_PATH = f"{error_prefix} Initialization failed. Invalid credentials. Expected file path to be a string." + EMPTY_CREDENTIALS_TOKEN = f"{error_prefix} Initialization failed. Specify a valid credentials token." + INVALID_CREDENTIALS_TOKEN = f"{error_prefix} Initialization failed. Invalid credentials token. Expected token to be a string." + EXPIRED_TOKEN = f"${error_prefix} Initialization failed. Given token is expired. Specify a valid credentials token." + EMPTY_API_KEY = f"{error_prefix} Initialization failed. Specify a valid api key." + INVALID_API_KEY = f"{error_prefix} Initialization failed. Invalid api key. Expected api key to be a string." + INVALID_ROLES_KEY_TYPE = f"{error_prefix} Validation error. Invalid roles. Specify roles as an array." + EMPTY_ROLES = f"{error_prefix} Validation error. Invalid roles. Specify at least one role." + EMPTY_CONTEXT = f"{error_prefix} Initialization failed. Invalid context provided. Specify context as type Context." + INVALID_CONTEXT = f"{error_prefix} Initialization failed. Invalid context. Specify a valid context." + INVALID_LOG_LEVEL = f"{error_prefix} Initialization failed. Invalid log level. Specify a valid log level." + EMPTY_LOG_LEVEL = f"{error_prefix} Initialization failed. Specify a valid log level." + + EMPTY_CONNECTION_ID = f"{error_prefix} Initialization failed. Invalid connection Id. Specify a valid connection Id." + INVALID_CONNECTION_ID = f"{error_prefix} Initialization failed. Invalid connection Id. Specify connection Id as a string." + EMPTY_CONNECTION_URL = f"{error_prefix} Initialization failed. Invalid connection Url. Specify a valid connection Url." + INVALID_CONNECTION_URL = f"{error_prefix} Initialization failed. Invalid connection Url. Specify connection Url as a string." + CONNECTION_ID_NOT_IN_CONFIG_LIST = f"{error_prefix} Validation error. {{}} is missing from the config. Specify the connectionIds from config." + + MISSING_TABLE_NAME_IN_INSERT = f"{error_prefix} Validation error. Table name cannot be empty in insert request. Specify a table name." + INVALID_TABLE_NAME_IN_INSERT = f"{error_prefix} Validation error. Invalid table name in insert request. Specify a valid table name." + INVALID_TYPE_OF_DATA_IN_INSERT = f"{error_prefix} Validation error. Invalid type of data in insert request. Specify data as a object array." + EMPTY_DATA_IN_INSERT = f"{error_prefix} Validation error. Data array cannot be empty. Specify data in insert request." + INVALID_UPSERT_OPTIONS_TYPE = f"{error_prefix} Validation error. 'upsert' key cannot be empty in options. At least one object of table and column is required." + INVALID_HOMOGENEOUS_TYPE = f"{error_prefix} Validation error. Invalid type of homogeneous. Specify homogeneous as a string." + INVALID_TOKEN_STRICT_TYPE = f"{error_prefix} Validation error. Invalid type of token strict. Specify token strict as a enum." + INVALID_RETURN_TOKENS_TYPE = f"{error_prefix} Validation error. Invalid type of return tokens. Specify return tokens as a boolean." + INVALID_CONTINUE_ON_ERROR_TYPE = f"{error_prefix} Validation error. Invalid type of continue on error. Specify continue on error as a boolean." + TOKENS_PASSED_FOR_TOKEN_STRICT_DISABLE = f"{error_prefix} Validation error. 'token_strict' wasn't specified. Set 'token_strict' to 'ENABLE' to insert tokens." + INSUFFICIENT_TOKENS_PASSED_FOR_TOKEN_STRICT_ENABLE_STRICT = f"{error_prefix} Validation error. 'byot' is set to 'ENABLE_STRICT', but some fields are missing tokens. Specify tokens for all fields." + NO_TOKENS_IN_INSERT = f"{error_prefix} Validation error. Tokens weren't specified for records while 'token_Strict' was {{}}. Specify tokens." + BATCH_INSERT_FAILURE = f"{error_prefix} Insert operation failed." + GET_FAILURE = f"{error_prefix} Get operation failed." + + EMPTY_TABLE_VALUE = f"{error_prefix} Validation error. 'table' can't be empty. Specify a table." + INVALID_TABLE_VALUE = f"{error_prefix} Validation error. Invalid type of table. Specify table as a string" + EMPTY_RECORD_IDS_IN_DELETE = f"{error_prefix} Validation error. 'record ids' array can't be empty. Specify one or more record ids." + BULK_DELETE_FAILURE = f"{error_prefix} Delete operation failed." + + INVALID_QUERY_TYPE = f"{error_prefix} Validation error. Query parameter is of type {{}}. Specify as a string." + EMPTY_QUERY = f"{error_prefix} Validation error. Query parameter can't be empty. Specify as a string." + INVALID_QUERY_COMMAND = f"{error_prefix} Validation error. {{}} command was passed instead, but only SELECT commands are supported. Specify the SELECT command." + SERVER_ERROR = f"{error_prefix} Validation error. Check SkyflowError.data for details." + QUERY_FAILED = f"{error_prefix} Query operation failed." + DETOKENIZE_FIELD = f"{error_prefix} Detokenize operation failed." + UPDATE_FAILED = f"{error_prefix} Update operation failed." + TOKENIZE_FAILED = f"{error_prefix} Tokenize operation failed." + INVOKE_CONNECTION_FAILED = f"{error_prefix} Invoke Connection operation failed." + + INVALID_IDS_TYPE = f"{error_prefix} Validation error. 'ids' has a value of type {{}}. Specify 'ids' as list." + INVALID_REDACTION_TYPE = f"{error_prefix} Validation error. 'redaction' has a value of type {{}}. Specify 'redaction' as type Skyflow.Redaction." + INVALID_COLUMN_NAME = f"{error_prefix} Validation error. 'column' has a value of type {{}}. Specify 'column' as a string." + INVALID_COLUMN_VALUE = f"{error_prefix} Validation error. columnValues key has a value of type {{}}. Specify columnValues key as list." + INVALID_FIELDS_VALUE = f"{error_prefix} Validation error. fields key has a value of type{{}}. Specify fields key as list." + BOTH_OFFSET_AND_LIMIT_SPECIFIED = f"${error_prefix} Validation error. Both offset and limit cannot be present at the same time" + INVALID_OFF_SET_VALUE = f"{error_prefix} Validation error. offset key has a value of type {{}}. Specify offset key as integer." + INVALID_LIMIT_VALUE = f"{error_prefix} Validation error. limit key has a value of type {{}}. Specify limit key as integer." + INVALID_DOWNLOAD_URL_VALUE = f"{error_prefix} Validation error. download_url key has a value of type {{}}. Specify download_url key as boolean." + REDACTION_WITH_TOKENS_NOT_SUPPORTED = f"{error_prefix} Validation error. 'redaction' can't be used when tokens are specified. Remove 'redaction' from payload if tokens are specified." + TOKENS_GET_COLUMN_NOT_SUPPORTED = f"{error_prefix} Validation error. Column name and/or column values can't be used when tokens are specified. Remove unique column values or tokens from the payload." + BOTH_IDS_AND_COLUMN_DETAILS_SPECIFIED = f"{error_prefix} Validation error. Both Skyflow IDs and column details can't be specified. Either specify Skyflow IDs or unique column details." + INVALID_ORDER_BY_VALUE = f"{error_prefix} Validation error. order_by key has a value of type {{}}. Specify order_by key as Skyflow.OrderBy" + + UPDATE_FIELD_KEY_ERROR = f"{error_prefix} Validation error. Fields are empty in an update payload. Specify at least one field." + INVALID_FIELDS_TYPE = f"{error_prefix} Validation error. The 'data' key has a value of type {{}}. Specify 'data' as a dictionary." + IDS_KEY_ERROR = f"{error_prefix} Validation error. 'ids' key is missing from the payload. Specify an 'ids' key." + INVALID_TOKENS_LIST_VALUE = f"{error_prefix} Validation error. The 'tokens' key has a value of type {{}}. Specify 'tokens' as a list." + EMPTY_TOKENS_LIST_VALUE = f"{error_prefix} Validation error. Tokens are empty in detokenize payload. Specify at lease one token" + + INVALID_TOKENIZE_PARAMETERS = f"{error_prefix} Validation error. The 'tokenize_parameters' key has a value of type {{}}. Specify 'tokenize_parameters' as a list." + EMPTY_TOKENIZE_PARAMETERS = f"{error_prefix} Validation error. Tokenize parameters are empty in tokenize payload. Specify at least one parameter." + INVALID_TOKENIZE_PARAMETER = f"{error_prefix} Validation error. Tokenize parameter at index {{}} has a value of type {{}}. Specify as a dictionary." + EMPTY_TOKENIZE_PARAMETER_VALUE = f"{error_prefix} Validation error. Tokenize parameter value at index {{}} is empty. Specify a valid value." + EMPTY_TOKENIZE_PARAMETER_COLUMN_GROUP = f"{error_prefix} Validation error. Tokenize parameter column group at index {{}} is empty. Specify a valid column group." + INVALID_TOKENIZE_PARAMETER_KEY = f"{error_prefix} Validation error. Tokenize parameter key at index {{}} is invalid. Specify a valid key value." + + INVALID_REQUEST_BODY = f"{error_prefix} Validation error. Invalid request body. Specify the request body as an object." + INVALID_REQUEST_HEADERS = f"{error_prefix} Validation error. Invalid request headers. Specify the request as an object." + INVALID_URL = f"{error_prefix} Validation error. Connection url {{}} is invalid. Specify a valid connection url." + INVALID_PATH_PARAMS = f"{error_prefix} Validation error. Path parameters aren't valid. Specify valid path parameters." + INVALID_QUERY_PARAMS = f"{error_prefix} Validation error. Query parameters aren't valid. Specify valid query parameters." + + MISSING_PRIVATE_KEY = f"{error_prefix} Initialization failed. Unable to read private key in credentials. Verify your private key." + MISSING_CLIENT_ID = f"{error_prefix} Initialization failed. Unable to read client ID in credentials. Verify your client ID." + MISSING_KEY_ID = f"{error_prefix} Initialization failed. Unable to read key ID in credentials. Verify your key ID." + MISSING_TOKEN_URI = f"{error_prefix} Initialization failed. Unable to read token URI in credentials. Verify your token URI." + JWT_INVALID_FORMAT = f"{error_prefix} Initialization failed. Invalid private key format. Verify your credentials." + JWT_DECODE_ERROR = f"{error_prefix} Validation error. Invalid access token. Verify your credentials." + FILE_INVALID_JSON = f"{error_prefix} Initialization failed. File at {{}} is not in valid JSON format. Verify the file contents." class Info(Enum): - INFO_MESSAGE = "INFO MESSAGE" + INITIALIZE_CLIENT = "Initializing skyflow client" + CLIENT_INITIALIZED = "Initialized skyflow client successfully" + VALIDATE_INSERT_RECORDS = "Validating insert records" + VALIDATE_DELETE_RECORDS = "Validating delete records" + VALIDATE_DETOKENIZE_INPUT = "Validating detokenize input" + VALIDATE_GET_RECORDS = "Validating get records" + VALIDATE_CONNECTION_CONFIG = "Validating connection config" + INSERT_DATA_SUCCESS = "Data has been inserted successfully." + DETOKENIZE_SUCCESS = "Data has been detokenized successfully." + GET_SUCCESS = "Records fetched successfully." + QUERY_SUCCESS = "Query executed successfully." + BEARER_TOKEN_RECEIVED = "tokenProvider returned token successfully." + INSERT_TRIGGERED = "Insert method triggered." + DETOKENIZE_TRIGGERED = "Detokenize method triggered." + GET_TRIGGERED = "Get triggered." + INVOKE_CONNECTION_TRIGGERED = "Invoke connection triggered." + QUERY_TRIGGERED = "Query method triggered." + GENERATE_BEARER_TOKEN_TRIGGERED = "Generate bearer token triggered" + GENERATE_BEARER_TOKEN_SUCCESS = "Generate bearer token returned successfully" + IS_TOKEN_VALID_TRIGGERED = "isTokenValid() triggered" + IS_EXPIRED_TRIGGERED = "is_expired() triggered" + EMPTY_ACCESS_TOKEN = "Give access token is empty" + INVALID_TOKEN = "Given token is invalid" + UPDATE_TRIGGERED = "Update method triggered" + UPDATE_DATA_SUCCESS = "Data has been updated successfully" + DELETE_TRIGGERED = "Delete triggered." + DELETE_DATA_SUCCESS = "Data has been deleted successfully." class Warning(Enum): WARNING_MESSAGE = "WARNING MESSAGE" + + class InterfaceName(Enum): + CLIENT = "client" + INSERT = "client.insert" + DETOKENIZE = "client.detokenize" + TOKENIZE = "client.tokenize" + GET = "client.get" + UPDATE = "client.update" + INVOKE_CONNECTION = "client.invoke_connection" + QUERY = "client.query" + GENERATE_BEARER_TOKEN = "service_account.generate_bearer_token" + IS_TOKEN_VALID = "service_account.is_token_valid" + IS_EXPIRED = "service_account.is_expired" + DELETE = "client.delete" + + diff --git a/skyflow/utils/_utils.py b/skyflow/utils/_utils.py index 61282a1..94d6bf9 100644 --- a/skyflow/utils/_utils.py +++ b/skyflow/utils/_utils.py @@ -3,10 +3,20 @@ from urllib.parse import urlparse import urllib.parse import requests +import platform +import sys +from requests import PreparedRequest from skyflow.error import SkyflowError -from skyflow.generated.rest import RedactionEnumREDACTION, V1FieldRecords -from skyflow.utils.enums import Env, ContentType +from skyflow.generated.rest import RedactionEnumREDACTION, V1UpdateRecordResponse, V1BulkDeleteRecordResponse, \ + V1DetokenizeResponse, V1TokenizeResponse, V1GetQueryResponse, V1BulkGetRecordResponse +from . import SkyflowMessages, SDK_VERSION, log_error +from .enums import Env, ContentType, Redaction import skyflow.generated.rest as vault_client +from skyflow.vault.data import InsertResponse, UpdateResponse, DeleteResponse, QueryResponse, GetResponse +from .validations import validate_invoke_connection_params +from ..vault.tokens import DetokenizeResponse, TokenizeResponse + +invalid_input_error_code = SkyflowMessages.ErrorCodes.INVALID_INPUT.value def get_credentials(config_level_creds = None, common_skyflow_creds = None): env_skyflow_credentials = os.getenv("SKYFLOW_CREDENTIALS") @@ -49,10 +59,6 @@ def format_scope(scopes): return " ".join([f"role:{scope}" for scope in scopes]) -def get_redaction_type(redaction_type): - if redaction_type == "plain-text": - return RedactionEnumREDACTION.PLAIN_TEXT - def parse_path_params(url, path_params): result = url for param, value in path_params.items(): @@ -60,10 +66,30 @@ def parse_path_params(url, path_params): return result -def construct_invoke_connection_request(request, connection_url): - url = parse_path_params(connection_url.rstrip('/'), connection_url.pathParams) - header = dict() - header['content-type'] = ContentType.JSON +def to_lowercase_keys(dict): + ''' + convert keys of dictionary to lowercase + ''' + result = {} + for key, value in dict.items(): + result[key.lower()] = value + + return result + +def construct_invoke_connection_request(request, connection_url, logger) -> PreparedRequest: + url = parse_path_params(connection_url.rstrip('/'), request.path_params) + + try: + if isinstance(request.request_headers, dict): + header = to_lowercase_keys(json.loads( + json.dumps(request.request_headers))) + else: + raise SkyflowError(SkyflowMessages.Error.INVALID_REQUEST_BODY.value, invalid_input_error_code, logger = logger, logger_method=log_error) + except Exception: + raise SkyflowError(SkyflowMessages.Error.INVALID_REQUEST_HEADERS.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not 'Content-Type'.lower() in header: + header['content-type'] = ContentType.JSON try: if isinstance(request.body, dict): @@ -71,9 +97,11 @@ def construct_invoke_connection_request(request, connection_url): request.body, header["content-type"] ) else: - raise SyntaxError("Given response body is not valid") + raise SkyflowError(SkyflowMessages.Error.INVALID_REQUEST_HEADERS.value, invalid_input_error_code, logger = logger, logger_method=log_error) except Exception as e: - raise SyntaxError("Given request body is not valid") + raise SkyflowError( SkyflowMessages.Error.INVALID_REQUEST_HEADERS.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + validate_invoke_connection_params(logger, request.query_params, request.path_params) try: return requests.Request( @@ -85,7 +113,7 @@ def construct_invoke_connection_request(request, connection_url): files = files ).prepare() except requests.exceptions.InvalidURL: - raise SkyflowError("Invalid URL") + raise SkyflowError(SkyflowMessages.Error.INVALID_URL.value.format(connection_url), invalid_input_error_code, logger = logger, logger_method=log_error) def http_build_query(data): @@ -141,3 +169,193 @@ def get_data_from_content_type(data, content_type): converted_data = json.dumps(data) return converted_data, files + + +def get_metrics(): + ''' fetch metrics + ''' + sdk_name_version = "skyflow-python@" + SDK_VERSION + + try: + sdk_client_device_model = platform.node() + except Exception: + sdk_client_device_model = "" + + try: + sdk_client_os_details = sys.platform + except Exception: + sdk_client_os_details = "" + + try: + sdk_runtime_details = sys.version + except Exception: + sdk_runtime_details = "" + + details_dic = { + 'sdk_name_version': sdk_name_version, + 'sdk_client_device_model': sdk_client_device_model, + 'sdk_client_os_details': sdk_client_os_details, + 'sdk_runtime_details': "Python " + sdk_runtime_details, + } + return details_dic + + +def parse_insert_response(api_response, continue_on_error): + inserted_fields = [] + errors = [] + insert_response = InsertResponse() + if continue_on_error: + for idx, response in enumerate(api_response.responses): + if response['Status'] == 200: + body = response['Body'] + if 'records' in body: + for record in body['records']: + inserted_field = { + 'skyflow_id': record['skyflow_id'], + 'request_index': idx + } + + if 'tokens' in record: + inserted_field.update(record['tokens']) + inserted_fields.append(inserted_field) + elif response['Status'] == 400: + error = { + 'request_index': idx, + 'error': response['Body']['error'] + } + errors.append(error) + + insert_response.inserted_fields = inserted_fields + insert_response.error_data = errors + + else: + for record in api_response.records: + field_data = { + 'skyflow_id': record.skyflow_id + } + + if record.tokens: + field_data.update(record.tokens) + + inserted_fields.append(field_data) + insert_response.inserted_fields = inserted_fields + + return insert_response + +def parse_update_record_response(api_response: V1UpdateRecordResponse): + update_response = UpdateResponse() + updated_field = dict() + updated_field['skyflow_id'] = api_response.skyflow_id + if api_response.tokens is not None: + updated_field.update(api_response.tokens) + + update_response.updated_field = updated_field + + return update_response + +def parse_delete_response(api_response: V1BulkDeleteRecordResponse): + delete_response = DeleteResponse() + deleted_ids = api_response.record_id_response + delete_response.deleted_ids = deleted_ids + delete_response.error = [] + return delete_response + + +def parse_get_response(api_response: V1BulkGetRecordResponse): + get_response = GetResponse() + data = [] + error = [] + for record in api_response.records: + field_data = {field: value for field, value in record.fields.items()} + data.append(field_data) + + get_response.data = data + get_response.error = error + + return get_response + +def parse_detokenize_response(api_response: V1DetokenizeResponse): + detokenized_fields = [] + errors = [] + + for record in api_response.records: + if record.error: + errors.append({ + "token": record.token, + "error": record.error + }) + else: + value_type = record.value_type.value if record.value_type else None + detokenized_fields.append({ + "token": record.token, + "value": record.value, + "type": value_type + }) + + detokenized_fields = detokenized_fields + errors = errors + detokenize_response = DetokenizeResponse() + detokenize_response.detokenized_fields = detokenized_fields + detokenize_response.errors = errors + + return detokenize_response + +def parse_tokenize_response(api_response: V1TokenizeResponse): + tokenize_response = TokenizeResponse() + tokenized_fields = [{"token": record.token} for record in api_response.records] + + tokenize_response.tokenized_fields = tokenized_fields + + return tokenize_response + +def parse_query_response(api_response: V1GetQueryResponse): + query_response = QueryResponse() + fields = [] + for record in api_response.records: + field_object = { + **record.fields, + "tokenized_data": {} + } + fields.append(field_object) + query_response.fields = fields + return query_response + + +def log_and_reject_error(description, status_code, request_id, http_status=None, grpc_code=None, details=None, logger = None): + log_error(description, status_code, request_id, grpc_code, http_status, details, logger= logger) + +def handle_exception(error, logger): + request_id = error.headers.get('x-request-id', 'unknown-request-id') + content_type = error.headers.get('content-type') + data = error.body + + # Call relevant handler based on content type + if content_type: + if 'application/json' in content_type: + handle_json_error(error, data, request_id, logger) + elif 'text/plain' in content_type: + handle_text_error(error, data, request_id, logger) + else: + handle_generic_error(error, request_id, logger) + else: + handle_generic_error(error, request_id, logger) + +def handle_json_error(err, data, request_id, logger): + try: + description = json.loads(data) + status_code = description.get('error', {}).get('http_code', 500) # Default to 500 if not found + http_status = description.get('error', {}).get('http_status') + grpc_code = description.get('error', {}).get('grpc_code') + details = description.get('error', {}).get('details') + + description_message = description.get('error', {}).get('message', "An unknown error occurred.") + log_and_reject_error(description_message, status_code, request_id, http_status, grpc_code, details, logger = logger) + except json.JSONDecodeError: + log_and_reject_error("Invalid JSON response received.", err, request_id, logger = logger) + +def handle_text_error(err, data, request_id, logger): + log_and_reject_error(data, err, request_id, logger = logger) + +def handle_generic_error(err, request_id, logger): + description = "An error occurred." + log_and_reject_error(description, err, request_id, logger = logger) diff --git a/skyflow/utils/_version.py b/skyflow/utils/_version.py new file mode 100644 index 0000000..551e675 --- /dev/null +++ b/skyflow/utils/_version.py @@ -0,0 +1 @@ +SDK_VERSION = "v2" \ No newline at end of file diff --git a/skyflow/utils/enums/__init__.py b/skyflow/utils/enums/__init__.py index 15f6982..63e6e65 100644 --- a/skyflow/utils/enums/__init__.py +++ b/skyflow/utils/enums/__init__.py @@ -1,3 +1,8 @@ from .env import Env from .log_level import LogLevel -from .content_types import ContentType \ No newline at end of file +from .content_types import ContentType +from .interface_name import InterfaceName +from .token_strict import TokenStrict +from .batch_method import BatchMethod +from .redaction_type import Redaction +from .order_by import OrderBy \ No newline at end of file diff --git a/skyflow/utils/enums/batch_method.py b/skyflow/utils/enums/batch_method.py new file mode 100644 index 0000000..18300af --- /dev/null +++ b/skyflow/utils/enums/batch_method.py @@ -0,0 +1,8 @@ +from enum import Enum + +class BatchMethod(Enum): + GET = "GET" + POST = "POST" + PUT = "PUT" + DELETE = "DELETE" + NONE = "NONE" \ No newline at end of file diff --git a/skyflow/utils/enums/interface_name.py b/skyflow/utils/enums/interface_name.py new file mode 100644 index 0000000..cf49dc9 --- /dev/null +++ b/skyflow/utils/enums/interface_name.py @@ -0,0 +1,15 @@ +from enum import Enum + +class InterfaceName(Enum): + CLIENT = "client" + INSERT = "client.insert" + DETOKENIZE = "client.detokenize" + GET_BY_ID = "client.get_by_id" + GET = "client.get" + UPDATE = "client.update" + INVOKE_CONNECTION = "client.invoke_connection" + QUERY = "client.query" + GENERATE_BEARER_TOKEN = "service_account.generate_bearer_token" + IS_TOKEN_VALID = "service_account.isTokenValid" + IS_EXPIRED = "service_account.is_expired" + DELETE = "client.delete" \ No newline at end of file diff --git a/skyflow/utils/enums/log_level.py b/skyflow/utils/enums/log_level.py index b5b9dbb..c92e914 100644 --- a/skyflow/utils/enums/log_level.py +++ b/skyflow/utils/enums/log_level.py @@ -1,8 +1,8 @@ from enum import Enum class LogLevel(Enum): - WARN = 'WARN' - INFO = 'INFO' - DEBUG = 'DEBUG' - ERROR = 'ERROR' - OFF = 'OFF' \ No newline at end of file + DEBUG = 1 + INFO = 2 + WARN = 3 + ERROR = 4 + OFF = 5 diff --git a/skyflow/utils/enums/order_by.py b/skyflow/utils/enums/order_by.py new file mode 100644 index 0000000..4c93731 --- /dev/null +++ b/skyflow/utils/enums/order_by.py @@ -0,0 +1,6 @@ +from enum import Enum + +class OrderBy(Enum): + ASCENDING = "ASCENDING" + DESCENDING = "DESCENDING" + NONE = "NONE" \ No newline at end of file diff --git a/skyflow/utils/enums/redaction_type.py b/skyflow/utils/enums/redaction_type.py index afe3920..0b48efa 100644 --- a/skyflow/utils/enums/redaction_type.py +++ b/skyflow/utils/enums/redaction_type.py @@ -1,4 +1,10 @@ from enum import Enum -class RedactionType(Enum): - PLAIN_TEXT = "plain-text" \ No newline at end of file +from skyflow.generated.rest import RedactionEnumREDACTION + + +class Redaction(Enum): + PLAIN_TEXT = RedactionEnumREDACTION.PLAIN_TEXT + MASKED = RedactionEnumREDACTION.MASKED + DEFAULT = RedactionEnumREDACTION.DEFAULT + REDACTED = RedactionEnumREDACTION.REDACTED \ No newline at end of file diff --git a/skyflow/utils/enums/token_strict.py b/skyflow/utils/enums/token_strict.py new file mode 100644 index 0000000..92c0043 --- /dev/null +++ b/skyflow/utils/enums/token_strict.py @@ -0,0 +1,6 @@ +from enum import Enum + +class TokenStrict(Enum): + DISABLE = "DISABLE" + ENABLE = "ENABLE" + ENABLE_STRICT = "ENABLE_STRICT" \ No newline at end of file diff --git a/skyflow/utils/validations/__init__.py b/skyflow/utils/validations/__init__.py index 6f3e035..d78e625 100644 --- a/skyflow/utils/validations/__init__.py +++ b/skyflow/utils/validations/__init__.py @@ -2,5 +2,15 @@ validate_vault_config, validate_insert_request, validate_connection_config, - validate_credentials + validate_update_vault_config, + validate_update_connection_config, + validate_credentials, + validate_log_level, + validate_delete_request, + validate_query_request, + validate_get_request, + validate_update_request, + validate_detokenize_request, + validate_tokenize_request, + validate_invoke_connection_params ) \ No newline at end of file diff --git a/skyflow/utils/validations/_validations.py b/skyflow/utils/validations/_validations.py index df892c5..bb839ff 100644 --- a/skyflow/utils/validations/_validations.py +++ b/skyflow/utils/validations/_validations.py @@ -1,23 +1,399 @@ +import json + +from skyflow.service_account import is_expired +from skyflow.utils.enums import LogLevel, TokenStrict, Redaction, Env from skyflow.error import SkyflowError +from skyflow.utils import SkyflowMessages, log_error + +valid_vault_config_keys = ["vault_id", "cluster_id", "credentials", "env"] +valid_connection_config_keys = ["connection_id", "connection_url", "credentials"] +valid_credentials_keys = ["path", "roles", "context", "token", "credentials_string"] +invalid_input_error_code = SkyflowMessages.ErrorCodes.INVALID_INPUT.value + +def validate_required_field(logger, config, field_name, expected_type, empty_error, invalid_error): + field_value = config.get(field_name) + + if field_name not in config or not isinstance(field_value, expected_type): + raise SkyflowError(invalid_error, invalid_input_error_code, logger = logger, logger_method=log_error) + + if isinstance(field_value, str) and not field_value.strip(): + raise SkyflowError(empty_error, invalid_input_error_code, logger = logger, logger_method=log_error) + +def validate_credentials(logger, credentials): + key_present = [k for k in ["path", "token", "credentials_string", "api_key"] if credentials.get(k)] + if len(key_present) == 0: + raise SkyflowError(SkyflowMessages.Error.INVALID_CREDENTIALS.value, invalid_input_error_code, logger = logger, logger_method=log_error) + elif len(key_present) > 1: + raise SkyflowError(SkyflowMessages.Error.MULTIPLE_CREDENTIALS_PASSED.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if "roles" in credentials: + validate_required_field( + logger, credentials, "roles", list, + SkyflowMessages.Error.INVALID_ROLES_KEY_TYPE.value, + SkyflowMessages.Error.EMPTY_ROLES.value + ) + + if "context" in credentials: + validate_required_field( + logger, credentials, "context", str, + SkyflowMessages.Error.EMPTY_CONTEXT.value, + SkyflowMessages.Error.INVALID_CONTEXT.value + ) + + if "credentials_string" in credentials: + validate_required_field( + logger, credentials, "credentials_string", str, + SkyflowMessages.Error.EMPTY_CREDENTIALS_STRING.value, + SkyflowMessages.Error.INVALID_CREDENTIALS_STRING.value + ) + elif "path" in credentials: + validate_required_field( + logger, credentials, "path", str, + SkyflowMessages.Error.EMPTY_CREDENTIAL_FILE_PATH.value, + SkyflowMessages.Error.INVALID_CREDENTIAL_FILE_PATH.value + ) + elif "token" in credentials: + validate_required_field( + logger, credentials, "token", str, + SkyflowMessages.Error.EMPTY_CREDENTIALS_TOKEN.value, + SkyflowMessages.Error.INVALID_CREDENTIALS_TOKEN.value + ) + + if is_expired(credentials.get("token"), logger): + raise SkyflowError() + elif "api_key" in credentials: + validate_required_field( + logger, credentials, "api_key", str, + SkyflowMessages.Error.EMPTY_API_KEY.value, + SkyflowMessages.Error.INVALID_API_KEY.value + ) + +def validate_log_level(logger, log_level): + if not isinstance(log_level, LogLevel): + raise SkyflowError( SkyflowMessages.Error.INVALID_LOG_LEVEL.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if log_level is None: + raise SkyflowError(SkyflowMessages.Error.EMPTY_LOG_LEVEL.value, invalid_input_error_code, logger = logger, logger_method=log_error) + +def validate_keys(logger, config, config_keys): + for key in config.keys(): + if key not in config_keys: + raise SkyflowError(SkyflowMessages.Error.INVALID_KEY.value.format(key), invalid_input_error_code, logger = logger, logger_method=log_error) + +def validate_vault_config(logger, config): + + validate_keys(logger, config, valid_vault_config_keys) + + # Validate vault_id (string, not empty) + validate_required_field( + logger, config, "vault_id", str, + SkyflowMessages.Error.EMPTY_VAULT_ID.value, + SkyflowMessages.Error.INVALID_VAULT_ID.value + ) + + # Validate cluster_id (string, not empty) + validate_required_field( + logger, config, "cluster_id", str, + SkyflowMessages.Error.EMPTY_CLUSTER_ID.value, + SkyflowMessages.Error.INVALID_CLUSTER_ID.value + ) + + # Validate credentials (dict, not empty) + if "credentials" not in config: + raise SkyflowError(SkyflowMessages.Error.EMPTY_CREDENTIALS.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + validate_credentials(logger, config.get("credentials")) + + # Validate env (optional, should be one of LogLevel values) + if "env" in config and config.get("env") not in Env: + raise SkyflowError(SkyflowMessages.Error.INVALID_ENV.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + return True + +def validate_update_vault_config(logger, config): + + validate_keys(logger, config, valid_vault_config_keys) + + # Validate vault_id (string, not empty) + validate_required_field( + logger, config, "vault_id", str, + SkyflowMessages.Error.EMPTY_VAULT_ID.value, + SkyflowMessages.Error.INVALID_VAULT_ID.value + ) + + if "cluster_id" in config and not config.get("cluster_id"): + raise SkyflowError(SkyflowMessages.Error.INVALID_CLUSTER_ID.value, invalid_input_error_code, logger = logger, logger_method=log_error) + if "env" in config and config.get("env") not in LogLevel: + raise SkyflowError(SkyflowMessages.Error.INVALID_ENV.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if "credentials" not in config: + raise SkyflowError(SkyflowMessages.Error.EMPTY_CREDENTIALS.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + validate_credentials(logger, config.get("credentials")) -def validate_vault_config(config): - #validate vault configuration return True -def validate_connection_config(config): - #validate connection configuration +def validate_connection_config(logger, config): + validate_keys(logger, config, valid_connection_config_keys) + + validate_required_field( + logger, config, "connection_id" , str, + SkyflowMessages.Error.EMPTY_CONNECTION_ID.value, + SkyflowMessages.Error.INVALID_CONNECTION_ID.value + ) + + validate_required_field( + logger, config, "connection_url", str, + SkyflowMessages.Error.EMPTY_CONNECTION_URL.value, + SkyflowMessages.Error.INVALID_CONNECTION_URL.value + ) + + if "credentials" not in config: + raise SkyflowError(SkyflowMessages.Error.EMPTY_CREDENTIALS.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + validate_credentials(logger, config.get("credentials")) + + return True + +def validate_update_connection_config(logger, config): + + validate_keys(logger, config, valid_connection_config_keys) + + validate_required_field( + logger, config, "connection_id", str, + SkyflowMessages.Error.EMPTY_CONNECTION_ID.value, + SkyflowMessages.Error.INVALID_CONNECTION_ID.value + ) + + validate_required_field( + logger, config, "connection_url", str, + SkyflowMessages.Error.EMPTY_CONNECTION_URL.value, + SkyflowMessages.Error.INVALID_CONNECTION_URL.value + ) + + if "credentials" not in config: + raise SkyflowError(SkyflowMessages.Error.EMPTY_CREDENTIALS, invalid_input_error_code, logger = logger, logger_method=log_error) + validate_credentials(logger, config.get("credentials")) + return True -def validate_insert_request(request): - if not request.table_name: - raise SkyflowError("Table name is required.") - if not isinstance(request.values, list) or len(request.values) == 0: - raise SkyflowError("At least one record must be provided.") -def validate_credentials(credentials): - keys_to_check = ['path', 'token', 'credentials_string'] - present_keys = [key for key in keys_to_check if credentials.get(key)] - if len(present_keys) > 1: +def validate_insert_request(logger, request): + if not isinstance(request.table_name, str): + raise SkyflowError(SkyflowMessages.Error.INVALID_TABLE_NAME_IN_INSERT.value, invalid_input_error_code, logger = logger, logger_method=log_error) + if not request.table_name.strip(): + raise SkyflowError(SkyflowMessages.Error.MISSING_TABLE_NAME_IN_INSERT.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not isinstance(request.values, list) or not all(isinstance(v, dict) for v in request.values): + raise SkyflowError(SkyflowMessages.Error.INVALID_TYPE_OF_DATA_IN_INSERT.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not len(request.values): + raise SkyflowError(SkyflowMessages.Error.EMPTY_DATA_IN_INSERT.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if request.upsert is not None and (not isinstance(request.upsert, str) or not request.upsert.strip()): + raise SkyflowError(SkyflowMessages.Error.INVALID_UPSERT_OPTIONS_TYPE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not isinstance(request.homogeneous, bool): + raise SkyflowError(SkyflowMessages.Error.INVALID_HOMOGENEOUS_TYPE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if request.token_strict is not None: + if not isinstance(request.token_strict, TokenStrict): + raise SkyflowError(SkyflowMessages.Error.INVALID_TOKEN_STRICT_TYPE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not isinstance(request.return_tokens, bool): + raise SkyflowError(SkyflowMessages.Error.INVALID_RETURN_TOKENS_TYPE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not isinstance(request.continue_on_error, bool): + raise SkyflowError(SkyflowMessages.Error.INVALID_CONTINUE_ON_ERROR_TYPE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if request.tokens: + if not isinstance(request.tokens, list) or not request.tokens or not all( + isinstance(t, dict) for t in request.tokens): + raise SkyflowError(SkyflowMessages.Error.INVALID_TYPE_OF_DATA_IN_INSERT.value, invalid_input_error_code, + logger=logger, logger_method=log_error) + + if request.token_strict == TokenStrict.ENABLE and not request.tokens: + raise SkyflowError(SkyflowMessages.Error.NO_TOKENS_IN_INSERT.value.format(request.token_Strict), invalid_input_error_code, logger = logger, logger_method=log_error) + + if request.token_strict == TokenStrict.DISABLE and request.tokens: + raise SkyflowError(SkyflowMessages.Error.TOKENS_PASSED_FOR_TOKEN_STRICT_DISABLE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if request.token_strict == TokenStrict.ENABLE_STRICT: + if len(request.values) != len(request.tokens): + raise SkyflowError(SkyflowMessages.Error.INSUFFICIENT_TOKENS_PASSED_FOR_TOKEN_STRICT_ENABLE_STRICT.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + for v, t in zip(request.values, request.tokens): + if set(v.keys()) != set(t.keys()): + raise SkyflowError(SkyflowMessages.Error.INSUFFICIENT_TOKENS_PASSED_FOR_TOKEN_STRICT_ENABLE_STRICT.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + +def validate_delete_request(logger, request): + if not isinstance(request.table, str): + raise SkyflowError(SkyflowMessages.Error.INVALID_TABLE_VALUE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + if not request.table.strip(): + raise SkyflowError(SkyflowMessages.Error.EMPTY_TABLE_VALUE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not request.ids: + raise SkyflowError(SkyflowMessages.Error.EMPTY_RECORD_IDS_IN_DELETE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + +def validate_query_request(logger, request): + if not isinstance(request.query, str): + query_type = str(type(request.query)) + raise SkyflowError(SkyflowMessages.Error.INVALID_QUERY_TYPE.value.format(query_type), invalid_input_error_code, logger = logger, logger_method=log_error) + + if not request.query.strip(): + raise SkyflowError(SkyflowMessages.Error.EMPTY_QUERY.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not request.query.upper().startswith("SELECT"): + command = request.query + raise SkyflowError(SkyflowMessages.Error.INVALID_QUERY_COMMAND.value.format(command), invalid_input_error_code, logger = logger, logger_method=log_error) + +def validate_get_request(logger, request): + redaction_type = request.redaction_type + column_name = request.column_name + column_values = request.column_values + skyflow_ids = request.ids + fields = request.fields + offset = request.offset + limit = request.limit + download_url = request.download_url + + if skyflow_ids and (not isinstance(skyflow_ids, list) or not skyflow_ids): + raise SkyflowError(SkyflowMessages.Error.INVALID_IDS_TYPE.value.format(type(skyflow_ids)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if not isinstance(request.return_tokens, bool): + raise SkyflowError(SkyflowMessages.Error.INVALID_RETURN_TOKENS_TYPE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if redaction_type is not None and not isinstance(redaction_type, Redaction): + raise SkyflowError(SkyflowMessages.Error.INVALID_REDACTION_TYPE.value.format(type(redaction_type)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if fields is not None and (not isinstance(fields, list) or not fields): + raise SkyflowError(SkyflowMessages.Error.INVALID_FIELDS_VALUE.value.format(type(fields)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if offset is not None and limit is not None: raise SkyflowError( - f"Only one of 'path', 'token', or 'credentials_string' should be present. Found multiple: {present_keys}") \ No newline at end of file + SkyflowMessages.Error.BOTH_OFFSET_AND_LIMIT_SPECIFIED.value, + invalid_input_error_code, logger=logger, logger_method=log_error) + + if offset is not None and not isinstance(offset, str): + raise SkyflowError(SkyflowMessages.Error.INVALID_OFF_SET_VALUE.value(type(offset)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if limit is not None and not isinstance(limit, str): + raise SkyflowError(SkyflowMessages.Error.INVALID_LIMIT_VALUE.value(type(limit)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if download_url is not None and not isinstance(download_url, bool): + raise SkyflowError(SkyflowMessages.Error.INVALID_DOWNLOAD_URL_VALUE.value(type(download_url)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if column_name is not None and (not isinstance(column_name, str) or not column_name.strip()): + raise SkyflowError(SkyflowMessages.Error.INVALID_COLUMN_NAME.value.format(type(column_name)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if column_values is not None and ( + not isinstance(column_values, list) or not column_values or not all( + isinstance(val, str) for val in column_values)): + raise SkyflowError(SkyflowMessages.Error.INVALID_COLUMN_VALUE.value.format(type(column_values)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if request.return_tokens and redaction_type: + raise SkyflowError(SkyflowMessages.Error.REDACTION_WITH_TOKENS_NOT_SUPPORTED.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if (column_name or column_values) and request.return_tokens: + raise SkyflowError(SkyflowMessages.Error.TOKENS_GET_COLUMN_NOT_SUPPORTED.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if column_values and not column_name: + raise SkyflowError(SkyflowMessages.Error.INVALID_COLUMN_VALUE.value.format(type(column_values)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if column_name and not column_values: + SkyflowError(SkyflowMessages.Error.INVALID_COLUMN_NAME.value.format(type(column_name)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if (column_name or column_values) and skyflow_ids: + raise SkyflowError(SkyflowMessages.Error.BOTH_IDS_AND_COLUMN_DETAILS_SPECIFIED.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + + + + +def validate_update_request(logger, request): + if not isinstance(request.table, str): + raise SkyflowError(SkyflowMessages.Error.INVALID_TABLE_VALUE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + if not request.table.strip(): + raise SkyflowError(SkyflowMessages.Error.EMPTY_TABLE_VALUE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not isinstance(request.return_tokens, bool): + raise SkyflowError(SkyflowMessages.Error.INVALID_RETURN_TOKENS_TYPE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not isinstance(request.data, dict): + raise SkyflowError(SkyflowMessages.Error.INVALID_FIELDS_TYPE.value(type(request.data)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if not len(request.data.items()): + raise SkyflowError(SkyflowMessages.Error.UPDATE_FIELD_KEY_ERROR.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if request.token_strict is not None: + if not isinstance(request.token_strict, TokenStrict): + raise SkyflowError(SkyflowMessages.Error.INVALID_TOKEN_STRICT_TYPE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if request.tokens: + if not isinstance(request.tokens, list) or not request.tokens or not all( + isinstance(t, dict) for t in request.tokens): + raise SkyflowError(SkyflowMessages.Error.INVALID_TYPE_OF_DATA_IN_INSERT.value, invalid_input_error_code, + logger=logger, logger_method=log_error) + + if 'id' not in request.data: + raise SkyflowError(SkyflowMessages.Error.IDS_KEY_ERROR.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + +def validate_detokenize_request(logger, request): + if not isinstance(request.redaction_type, Redaction): + raise SkyflowError(SkyflowMessages.Error.INVALID_REDACTION_TYPE.value.format(type(request.redaction_type)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if not isinstance(request.continue_on_error, bool): + raise SkyflowError(SkyflowMessages.Error.INVALID_CONTINUE_ON_ERROR_TYPE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not len(request.tokens): + raise SkyflowError(SkyflowMessages.Error.EMPTY_TOKENS_LIST_VALUE.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not isinstance(request.tokens, list): + raise SkyflowError(SkyflowMessages.Error.INVALID_TOKENS_LIST_VALUE.value(type(request.tokens)), invalid_input_error_code, logger = logger, logger_method=log_error) + + +def validate_tokenize_request(logger, request): + parameters = request.tokenize_parameters + if not isinstance(parameters, list): + raise SkyflowError(SkyflowMessages.Error.INVALID_TOKENIZE_PARAMETERS.value.format(type(parameters)), invalid_input_error_code, logger = logger, logger_method=log_error) + + if not len(parameters): + raise SkyflowError(SkyflowMessages.Error.EMPTY_TOKENIZE_PARAMETERS.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + for i, param in enumerate(parameters): + if not isinstance(param, dict): + raise SkyflowError(SkyflowMessages.Error.INVALID_TOKENIZE_PARAMETER.value.format(i, type(param)), invalid_input_error_code, logger = logger, logger_method=log_error) + + allowed_keys = {"value", "column_group"} + + if set(param.keys()) != allowed_keys: + raise SkyflowError(SkyflowMessages.Error.INVALID_TOKENIZE_PARAMETER_KEY.value.format(i), invalid_input_error_code, logger = logger, logger_method=log_error) + + if not param.get("value"): + raise SkyflowError(SkyflowMessages.Error.EMPTY_TOKENIZE_PARAMETER_VALUE.value.format(i), invalid_input_error_code, logger = logger, logger_method=log_error) + if not param.get("column_group"): + raise SkyflowError(SkyflowMessages.Error.EMPTY_TOKENIZE_PARAMETER_COLUMN_GROUP.value.format(i), invalid_input_error_code, logger = logger, logger_method=log_error) + + +def validate_invoke_connection_params(logger, query_params, path_params): + if not isinstance(path_params, dict): + raise SkyflowError(SkyflowMessages.Error.INVALID_PATH_PARAMS.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + if not isinstance(query_params, dict): + raise SkyflowError(SkyflowMessages.Error.INVALID_QUERY_PARAMS.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + for param, value in path_params.items(): + if not(isinstance(param, str) and isinstance(value, str)): + raise SkyflowError(SkyflowMessages.Error.INVALID_PATH_PARAMS.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + for param, value in query_params.items(): + if not isinstance(param, str): + raise SkyflowError(SkyflowMessages.Error.INVALID_QUERY_PARAMS.value, invalid_input_error_code, logger = logger, logger_method=log_error) + + try: + json.dumps(query_params) + except TypeError: + raise SkyflowError(SkyflowMessages.Error.INVALID_QUERY_PARAMS.value, invalid_input_error_code, logger = logger, logger_method=log_error) diff --git a/skyflow/vault/client/client.py b/skyflow/vault/client/client.py index 3005226..23a503b 100644 --- a/skyflow/vault/client/client.py +++ b/skyflow/vault/client/client.py @@ -1,6 +1,6 @@ from skyflow.generated.rest import Configuration, RecordsApi, ApiClient, TokensApi, QueryApi from skyflow.service_account import generate_bearer_token, generate_bearer_token_from_creds -from skyflow.utils import get_vault_url, get_credentials +from skyflow.utils import get_vault_url, get_credentials, SkyflowMessages, log_info class VaultClient: @@ -10,12 +10,14 @@ def __init__(self, config): self.__log_level = None self.__client_configuration = None self.__api_client = None + self.__logger = None def set_common_skyflow_credentials(self, credentials): self.__common_skyflow_credentials = credentials - def set_log_level(self, log_level): + def set_logger(self, log_level, logger): self.__log_level = log_level + self.__logger = logger def initialize_client_configuration(self): credentials = get_credentials(self.__config.get("credentials"), self.__common_skyflow_credentials) @@ -41,16 +43,31 @@ def get_vault_id(self): return self.__config.get("vault_id") def get_bearer_token(self, credentials): - if 'token' in credentials: + interface = SkyflowMessages.InterfaceName.GENERATE_BEARER_TOKEN.value + if 'api_key' in credentials: + return credentials.get('api_key') + elif 'token' in credentials: return credentials.get("token") elif 'path' in credentials: credentials = self.__config.get("credentials") - roles = self.__config.get("roles") if "roles" in self.__config else None - return generate_bearer_token(credentials.get("path"), roles) + options = { + "role_ids": self.__config.get("roles"), + "ctx": self.__config.get("ctx") + } + log_info(self.__logger, SkyflowMessages.Info.GENERATE_BEARER_TOKEN_TRIGGERED, interface) + token, _ = generate_bearer_token(credentials.get("path"), options, self.__logger) + log_info(self.__logger, SkyflowMessages.Info.GENERATE_BEARER_TOKEN_SUCCESS, interface) + return token else: credentials = self.__config.get("credentials") - roles = self.__config.get("roles") if "roles" in self.__config else None - return generate_bearer_token_from_creds(credentials.get("credentials_string"), roles) + options = { + "role_ids": self.__config.get("roles"), + "ctx": self.__config.get("ctx") + } + log_info(self.__logger, SkyflowMessages.Info.GENERATE_BEARER_TOKEN_TRIGGERED, interface) + token, _ = generate_bearer_token_from_creds(credentials.get("credentials_string"), options, self.__logger) + log_info(self.__logger, SkyflowMessages.Info.GENERATE_BEARER_TOKEN_SUCCESS, interface) + return token def update_config(self, config): self.__config.update(config) @@ -62,4 +79,7 @@ def get_common_skyflow_credentials(self): return self.__common_skyflow_credentials def get_log_level(self): - return self.__log_level \ No newline at end of file + return self.__log_level + + def get_logger(self): + return self.__logger \ No newline at end of file diff --git a/skyflow/vault/connection/_invoke_connection_request.py b/skyflow/vault/connection/_invoke_connection_request.py index 6b98568..25d9ec0 100644 --- a/skyflow/vault/connection/_invoke_connection_request.py +++ b/skyflow/vault/connection/_invoke_connection_request.py @@ -1,5 +1,12 @@ class InvokeConnectionRequest: - def __init__(self, method, params, body): + def __init__(self, + method, + body = None, + path_params = None, + query_params = None, + request_headers = None): + self.body = body if body is not None else {} self.method = method - self.params = params - self.body = body \ No newline at end of file + self.path_params = path_params if path_params is not None else {} + self.query_params = query_params if query_params is not None else {} + self.request_headers = request_headers if request_headers is not None else {} \ No newline at end of file diff --git a/skyflow/vault/connection/_invoke_connection_response.py b/skyflow/vault/connection/_invoke_connection_response.py index 2c58b21..67b2882 100644 --- a/skyflow/vault/connection/_invoke_connection_response.py +++ b/skyflow/vault/connection/_invoke_connection_response.py @@ -1,14 +1,12 @@ class InvokeConnectionResponse: - def __init__(self, fields, error): - self.fields = fields - self.error = error + def __init__(self, response = None): + self.response = response def __repr__(self): - return f"ConnectionResponse({self.fields}, errors={self.error})" + return f"ConnectionResponse({self.response})" def __str__(self): return self.__repr__() - @classmethod - def parse_invoke_connection_response(cls, response): - return response \ No newline at end of file + def parse_invoke_connection_response(self, response): + self.response = response \ No newline at end of file diff --git a/skyflow/vault/controller/_connections.py b/skyflow/vault/controller/_connections.py index bb405da..508339a 100644 --- a/skyflow/vault/controller/_connections.py +++ b/skyflow/vault/controller/_connections.py @@ -1,21 +1,39 @@ +import json + import requests -from skyflow.utils import construct_invoke_connection_request +from skyflow.error import SkyflowError +from skyflow.utils import construct_invoke_connection_request, log_info, SkyflowMessages, get_metrics, log_error from skyflow.vault.connection import InvokeConnectionRequest, InvokeConnectionResponse class Connection: - def __init__(self, vault_client): self.__vault_client = vault_client + self.logger = self.__vault_client.get_logger() def invoke(self, request: InvokeConnectionRequest): + interface = SkyflowMessages.InterfaceName.INVOKE_CONNECTION.value + log_info(self.logger, SkyflowMessages.Info.INVOKE_CONNECTION_TRIGGERED, interface) + session = requests.Session() + config = self.__vault_client.get_config() bearer_token = self.__vault_client.get_bearer_token(config.get("credentials")) + connection_url = config.get("connection_url") - request = construct_invoke_connection_request(request, connection_url) - request.headers['x-skyflow-authorization'] = f"Bearer ${bearer_token}" + invoke_connection_request = construct_invoke_connection_request(request, connection_url, self.logger) + + if not 'X-Skyflow-Authorization'.lower() in invoke_connection_request.headers: + invoke_connection_request.headers['x-skyflow-authorization'] = f"Bearer ${bearer_token}" + + invoke_connection_request.headers['sky-metadata'] = json.dumps(get_metrics()) + + log_info(self.logger, SkyflowMessages.Info.INVOKE_CONNECTION_TRIGGERED, interface) - response = session.send(request) - session.close() - return InvokeConnectionResponse.parse_invoke_connection_response(response) \ No newline at end of file + try: + response = session.send(invoke_connection_request) + session.close() + invoke_connection_response = InvokeConnectionResponse() + return invoke_connection_response.parse_invoke_connection_response(response) + except: + raise SkyflowError(SkyflowMessages.Error.INVOKE_CONNECTION_FAILED.value, SkyflowMessages.ErrorCodes.SERVER_ERROR, logger = self.logger, logger_method=log_error) \ No newline at end of file diff --git a/skyflow/vault/controller/_vault.py b/skyflow/vault/controller/_vault.py index 95a3682..5c73804 100644 --- a/skyflow/vault/controller/_vault.py +++ b/skyflow/vault/controller/_vault.py @@ -1,12 +1,15 @@ from skyflow.generated.rest import V1FieldRecords, RecordServiceInsertRecordBody, V1DetokenizeRecordRequest, \ V1DetokenizePayload, V1TokenizeRecordRequest, V1TokenizePayload, QueryServiceExecuteQueryBody, \ - RecordServiceBulkDeleteRecordBody, RecordServiceUpdateRecordBody -from skyflow.utils import get_redaction_type -from skyflow.utils.validations import validate_insert_request -from skyflow.vault.data import InsertRequest, UpdateRequest, DeleteRequest, GetRequest, UploadFileRequest, QueryRequest, \ - InsertResponse, GetResponse, QueryResponse -from skyflow.vault.tokens import DetokenizeRequest, TokenizeRequest, DetokenizeResponse, TokenizeResponse -from skyflow.error import SkyflowError + RecordServiceBulkDeleteRecordBody, RecordServiceUpdateRecordBody, RecordServiceBatchOperationBody, V1BatchRecord, \ + BatchRecordMethod +from skyflow.generated.rest.exceptions import BadRequestException +from skyflow.utils import log_info, SkyflowMessages, parse_insert_response, \ + handle_exception, parse_update_record_response, parse_delete_response, parse_detokenize_response, \ + parse_tokenize_response, parse_query_response, parse_get_response +from skyflow.utils.validations import validate_insert_request, validate_delete_request, validate_query_request, \ + validate_get_request, validate_update_request, validate_detokenize_request, validate_tokenize_request +from skyflow.vault.data import InsertRequest, UpdateRequest, DeleteRequest, GetRequest, QueryRequest, GetResponse +from skyflow.vault.tokens import DetokenizeRequest, TokenizeRequest class Vault: def __init__(self, vault_client): @@ -15,133 +18,202 @@ def __init__(self, vault_client): def __initialize(self): self.__vault_client.initialize_client_configuration() - def __build_field_records(self, values): + def __build_bulk_field_records(self, values): return [V1FieldRecords(fields=record) for record in values] + def __build_batch_field_records(self, values, tokens, table_name, return_tokens, upsert): + batch_record_list = [] + for i, value in enumerate(values): + token = tokens[i] if tokens is not None else None + batch_record = V1BatchRecord( + fields=value, + table_name=table_name, + method=BatchRecordMethod.POST, + tokenization=return_tokens, + ) + if token is not None: + batch_record.tokens = token + batch_record_list.append(batch_record) + return batch_record_list + + def __build_insert_body(self, request: InsertRequest): + if request.continue_on_error: + records_list = self.__build_batch_field_records( + request.values, + request.tokens, + request.table_name, + request.return_tokens, + request.upsert + ) + body = RecordServiceBatchOperationBody( + records=records_list, + continue_on_error=request.continue_on_error, + byot=request.token_strict + ) + return body + else: + records_list = self.__build_bulk_field_records(request.values) + return RecordServiceInsertRecordBody( + records=records_list, + tokenization=request.return_tokens, + upsert=request.upsert, + homogeneous=request.homogeneous + ) + def insert(self, request: InsertRequest): - print(self.__vault_client.get_log_level()) - validate_insert_request(request) + interface = SkyflowMessages.InterfaceName.INSERT.value + log_info(SkyflowMessages.Info.VALIDATE_INSERT_RECORDS.value, interface, self.__vault_client.get_logger()) + validate_insert_request(self.__vault_client.get_logger(), request) self.__initialize() - records_list = self.__build_field_records(request.values) - body = RecordServiceInsertRecordBody( - records = records_list, - tokenization = request.return_tokens, - upsert=request.upsert, - homogeneous=request.homogeneous, - ) records_api = self.__vault_client.get_records_api() + + insert_body = self.__build_insert_body(request) + try: - api_response = records_api.record_service_insert_record(self.__vault_client.get_vault_id(), request.table_name, body) - return InsertResponse.parse_insert_response(api_response) - except Exception: - raise SkyflowError("Insert Failed") + log_info(SkyflowMessages.Info.INSERT_TRIGGERED.value, interface, self.__vault_client.get_logger()) + + if request.continue_on_error: + api_response = records_api.record_service_batch_operation(self.__vault_client.get_vault_id(), + insert_body) + + else: + api_response = records_api.record_service_insert_record(self.__vault_client.get_vault_id(), + request.table_name, insert_body) + + insert_response = parse_insert_response(api_response, request.continue_on_error) + log_info(SkyflowMessages.Info.INSERT_DATA_SUCCESS.value, interface, self.__vault_client.get_logger()) + return insert_response + + except BadRequestException as e: + handle_exception(e, self.__vault_client.get_logger()) def update(self, request: UpdateRequest): + interface = SkyflowMessages.InterfaceName.UPDATE + validate_update_request(self.__vault_client.get_logger(), request) self.__initialize() - fields = {key: value for key, value in request.data.items() if key != "id"} - record = V1FieldRecords(fields=fields) - payload = RecordServiceUpdateRecordBody(record=record, tokenization=True) + field = {key: value for key, value in request.data.items() if key != "id"} + record = V1FieldRecords(fields=field, tokens = request.tokens) + payload = RecordServiceUpdateRecordBody(record=record, tokenization=request.return_tokens, byot=request.token_strict) records_api = self.__vault_client.get_records_api() try: + log_info(SkyflowMessages.Info.UPDATE_TRIGGERED.value, interface, self.__vault_client.get_logger()) api_response = records_api.record_service_update_record( self.__vault_client.get_vault_id(), request.table, - request.data.id, + request.data.get("id"), payload ) - return api_response - except Exception: - raise SkyflowError("Update Failed") - - -def delete(self, request: DeleteRequest): - # validate_delete_request(request) - self._initialize() - + log_info(SkyflowMessages.Info.UPDATE_DATA_SUCCESS.value, interface, self.__vault_client.get_logger()) + update_response = parse_update_record_response(api_response) + return update_response + except Exception as e: + handle_exception(e, self.__vault_client.get_logger()) + + def delete(self, request: DeleteRequest): + interface = SkyflowMessages.InterfaceName.DELETE.value + log_info(SkyflowMessages.Info.VALIDATE_DELETE_RECORDS.value, interface, self.__vault_client.get_logger()) + validate_delete_request(self.__vault_client.get_logger(), request) + self.__initialize() payload = RecordServiceBulkDeleteRecordBody(skyflow_ids=request.ids) records_api = self.__vault_client.get_records_api() try: + log_info(SkyflowMessages.Info.DETOKENIZE_TRIGGERED.value, interface, self.__vault_client.get_logger()) api_response = records_api.record_service_bulk_delete_record( self.__vault_client.get_vault_id(), request.table, payload ) - return api_response - except Exception: - raise SkyflowError("Bulk Delete Failed") - -def get(self, request: GetRequest): - # validate_get_request(request) - self._initialize() - - records_api = self.__vault_client.get_records_api() - try: - api_response = records_api.record_service_bulk_get_record( - self.__vault_client.get_vault_id(), - table_name=request.table, - skyflow_ids=request.ids, - redaction=get_redaction_type(request.redaction_type), - tokenization=request.tokenization, - fields=request.fields, - offset=request.offset, - limit=request.limit, - download_url=request.download_url, - column_name=request.column_name, - column_values=request.column_values, - order_by=request.order_by - ) - return GetResponse.parsed_get_response(api_response) - except Exception: - raise SkyflowError("Get Failed") - -def query(self, request: QueryRequest): - self._initialize() - - payload = QueryServiceExecuteQueryBody(query=request.query) - query_api = self.__vault_client.get_query_api() - try: - api_response = query_api.query_service_execute_query( - self.__vault_client.get_vault_id(), - payload - ) - return QueryResponse.parse_query_response(api_response) - except Exception: - raise SkyflowError("Query Failed") - -def detokenize(self, request: DetokenizeRequest): - # validate_detokenize_request(request) - self._initialize() - - tokens_list = [ - V1DetokenizeRecordRequest(token=token, redaction=get_redaction_type(request.redaction_type)) - for token in request.tokens - ] - payload = V1DetokenizePayload(detokenization_parameters=tokens_list, continue_on_error=request.continue_on_error) - tokens_api = self.__vault_client.get_tokens_api() - try: - api_response = tokens_api.record_service_detokenize( - self.__vault_client.get_vault_id(), - detokenize_payload=payload - ) - return DetokenizeResponse.parse_detokenize_response(api_response) - except Exception: - raise SkyflowError("Detokenize Failed") - -def tokenize(self, request: TokenizeRequest): - self._initialize() - - records_list = [ - V1TokenizeRecordRequest(value=item["values"], column_group=item["cg"]) - for item in request.tokenize_parameters - ] - payload = V1TokenizePayload(tokenization_parameters=records_list) - tokens_api = self.__vault_client.get_tokens_api() - try: - api_response = tokens_api.record_service_tokenize( - self.__vault_client.get_vault_id(), - tokenize_payload=payload - ) - return TokenizeResponse.parse_tokenize_response(api_response) - except Exception: - raise SkyflowError("Tokenize Failed") + log_info(SkyflowMessages.Info.DETOKENIZE_SUCCESS.value, interface, self.__vault_client.get_logger()) + delete_response = parse_delete_response(api_response) + return delete_response + except Exception as e: + handle_exception(e, self.__vault_client.get_logger()) + + def get(self, request: GetRequest): + interface = SkyflowMessages.InterfaceName.GET.value + log_info(SkyflowMessages.Info.VALIDATE_GET_RECORDS.value, interface, self.__vault_client.get_logger()) + validate_get_request(self.__vault_client.get_logger(), request) + self.__initialize() + records_api = self.__vault_client.get_records_api() + try: + log_info(SkyflowMessages.Info.GET_TRIGGERED.value, interface, self.__vault_client.get_logger()) + api_response = records_api.record_service_bulk_get_record( + self.__vault_client.get_vault_id(), + object_name=request.table, + skyflow_ids=request.ids, + redaction=request.redaction_type, + tokenization=request.return_tokens, + fields=request.fields, + offset=request.offset, + limit=request.limit, + download_url=request.download_url, + column_name=request.column_name, + column_values=request.column_values, + ) + log_info(SkyflowMessages.Info.GET_SUCCESS.value, interface, self.__vault_client.get_logger()) + get_response = parse_get_response(api_response) + return get_response + except Exception as e: + handle_exception(e, self.__vault_client.get_logger()) + + def query(self, request: QueryRequest): + interface = SkyflowMessages.InterfaceName.QUERY.value + validate_query_request(self.__vault_client.get_logger(), request) + self.__initialize() + payload = QueryServiceExecuteQueryBody(query=request.query) + query_api = self.__vault_client.get_query_api() + try: + log_info(SkyflowMessages.Info.QUERY_TRIGGERED.value, interface, self.__vault_client.get_logger()) + api_response = query_api.query_service_execute_query( + self.__vault_client.get_vault_id(), + payload + ) + log_info(SkyflowMessages.Info.QUERY_SUCCESS.value, interface, self.__vault_client.get_logger()) + query_response = parse_query_response(api_response) + return query_response + except Exception as e: + handle_exception(e, self.__vault_client.get_logger()) + + def detokenize(self, request: DetokenizeRequest): + interface = SkyflowMessages.InterfaceName.DETOKENIZE.value + log_info(SkyflowMessages.Info.VALIDATE_DETOKENIZE_INPUT.value, interface, self.__vault_client.get_logger()) + validate_detokenize_request(self.__vault_client.get_logger(), request) + self.__initialize() + tokens_list = [ + V1DetokenizeRecordRequest(token=token, redaction=request.redaction_type) + for token in request.tokens + ] + payload = V1DetokenizePayload(detokenization_parameters=tokens_list, continue_on_error=request.continue_on_error) + tokens_api = self.__vault_client.get_tokens_api() + try: + log_info(SkyflowMessages.Info.DETOKENIZE_TRIGGERED.value, interface, self.__vault_client.get_logger()) + api_response = tokens_api.record_service_detokenize( + self.__vault_client.get_vault_id(), + detokenize_payload=payload + ) + log_info(SkyflowMessages.Info.DETOKENIZE_SUCCESS.value, interface, self.__vault_client.get_logger()) + detokenize_response = parse_detokenize_response(api_response) + return detokenize_response + except Exception as e: + handle_exception(e, self.__vault_client.get_logger()) + + def tokenize(self, request: TokenizeRequest): + validate_tokenize_request(self.__vault_client.get_logger(), request) + self.__initialize() + + records_list = [ + V1TokenizeRecordRequest(value=item["value"], column_group=item["column_group"]) + for item in request.tokenize_parameters + ] + payload = V1TokenizePayload(tokenization_parameters=records_list) + tokens_api = self.__vault_client.get_tokens_api() + try: + api_response = tokens_api.record_service_tokenize( + self.__vault_client.get_vault_id(), + tokenize_payload=payload + ) + tokenize_response = parse_tokenize_response(api_response) + return tokenize_response + except Exception as e: + handle_exception(e, self.__vault_client.get_logger()) diff --git a/skyflow/vault/data/_delete_response.py b/skyflow/vault/data/_delete_response.py index b373538..a76d76a 100644 --- a/skyflow/vault/data/_delete_response.py +++ b/skyflow/vault/data/_delete_response.py @@ -2,7 +2,7 @@ class DeleteResponse: - def __init__(self, deleted_ids, error): + def __init__(self, deleted_ids = None, error = None): self.deleted_ids = deleted_ids self.error = error @@ -12,8 +12,3 @@ def __repr__(self): def __str__(self): return self.__repr__() - @staticmethod - def parsed_delete_response(delete_response: V1BulkDeleteRecordResponse): - deleted_ids = delete_response.record_id_response - error = [] - return DeleteResponse(deleted_ids=deleted_ids, error=error) \ No newline at end of file diff --git a/skyflow/vault/data/_get_request.py b/skyflow/vault/data/_get_request.py index 5be60e3..84a617c 100644 --- a/skyflow/vault/data/_get_request.py +++ b/skyflow/vault/data/_get_request.py @@ -1,23 +1,25 @@ +from skyflow.utils.enums import Redaction, OrderBy + + class GetRequest: - def __init__(self, table, + def __init__(self, + table, ids, - redaction_type, - tokenization = None, + redaction_type = None, + return_tokens = False, fields = None, offset = None, limit = None, download_url = None, column_name = None, - column_values = None, - order_by = None): + column_values = None): self.table = table self.ids = ids self.redaction_type = redaction_type - self.tokenization = tokenization + self.return_tokens = return_tokens self.fields = fields self.offset = offset self.limit = limit self.download_url = download_url self.column_name = column_name self.column_values = column_values - self.order_by = order_by diff --git a/skyflow/vault/data/_get_response.py b/skyflow/vault/data/_get_response.py index df963b9..428d248 100644 --- a/skyflow/vault/data/_get_response.py +++ b/skyflow/vault/data/_get_response.py @@ -2,7 +2,7 @@ class GetResponse: - def __init__(self, data=None, error = []): + def __init__(self, data=None, error = None): self.data = data if data else [] self.error = error @@ -10,14 +10,4 @@ def __repr__(self): return f"GetResponse(data={self.data}, error={self.error})" def __str__(self): - return self.__repr__() - - @staticmethod - def parsed_get_response(get_response: V1BulkGetRecordResponse): - data = [] - error=[] - for record in get_response.records: - field_data = {field: value for field, value in record.fields.items()} - data.append(field_data) - - return GetResponse(data, error=error) \ No newline at end of file + return self.__repr__() \ No newline at end of file diff --git a/skyflow/vault/data/_insert_request.py b/skyflow/vault/data/_insert_request.py index 7f28672..169866a 100644 --- a/skyflow/vault/data/_insert_request.py +++ b/skyflow/vault/data/_insert_request.py @@ -1,17 +1,23 @@ +from skyflow.generated.rest import V1BYOT +from skyflow.utils.enums import TokenStrict + + class InsertRequest: def __init__(self, table_name, values, + tokens = None, upsert = None, homogeneous = False, - token_mode = None, - token_strict = None, - return_tokens= True,): + token_strict = TokenStrict.DISABLE, + return_tokens = True, + continue_on_error = False): self.table_name = table_name self.values = values + self.tokens = tokens self.upsert = upsert self.homogeneous = homogeneous - self.token_mode = token_mode self.token_strict = token_strict self.return_tokens = return_tokens + self.continue_on_error = continue_on_error diff --git a/skyflow/vault/data/_insert_response.py b/skyflow/vault/data/_insert_response.py index b86f95c..2f1dd06 100644 --- a/skyflow/vault/data/_insert_response.py +++ b/skyflow/vault/data/_insert_response.py @@ -1,8 +1,8 @@ -from skyflow.generated.rest import V1InsertRecordResponse +from skyflow.generated.rest import V1InsertRecordResponse, V1BatchOperationResponse class InsertResponse: - def __init__(self, inserted_fields, error_data = None): + def __init__(self, inserted_fields = None, error_data = None): self.inserted_fields = inserted_fields self.error_data = error_data @@ -11,26 +11,3 @@ def __repr__(self): def __str__(self): return self.__repr__() - - @staticmethod - def parse_insert_response(response: V1InsertRecordResponse): - inserted_fields = [] - - if response.records: - for record in response.records: - - record_data = dict() - record_data["skyflow_id"] = record.skyflow_id - - if record.tokens: - for field_name, token in record.tokens.items(): - record_data[field_name] = token - - inserted_fields.append(record_data) - - insert_response = InsertResponse( - inserted_fields = inserted_fields, - error_data=[] - ) - - return insert_response diff --git a/skyflow/vault/data/_query_response.py b/skyflow/vault/data/_query_response.py index 5b83e03..998eee2 100644 --- a/skyflow/vault/data/_query_response.py +++ b/skyflow/vault/data/_query_response.py @@ -1,23 +1,12 @@ from skyflow.generated.rest import V1GetQueryResponse class QueryResponse: - def __init__(self, records): + def __init__(self): self.fields = [] self.error = [] - for record in records: - field_object = { - **record.fields, - "tokenizedData": {} - } - self.fields.append(field_object) - def __repr__(self): return f"QueryResponse(fields={self.fields}, error={self.error})" def __str__(self): return self.__repr__() - - @staticmethod - def parse_query_response(api_response: V1GetQueryResponse): - return QueryResponse(api_response.records) \ No newline at end of file diff --git a/skyflow/vault/data/_update_request.py b/skyflow/vault/data/_update_request.py index f9f933d..511ff90 100644 --- a/skyflow/vault/data/_update_request.py +++ b/skyflow/vault/data/_update_request.py @@ -1,5 +1,10 @@ +from skyflow.utils.enums import TokenStrict + + class UpdateRequest: - def __init__(self, table, data, return_tokens): + def __init__(self, table, data, tokens = None, return_tokens = False, token_strict = TokenStrict.DISABLE): self.table = table self.data = data - self.return_tokens = return_tokens \ No newline at end of file + self.tokens = tokens + self.return_tokens = return_tokens + self.token_strict = token_strict diff --git a/skyflow/vault/data/_update_response.py b/skyflow/vault/data/_update_response.py index 79496d8..1249238 100644 --- a/skyflow/vault/data/_update_response.py +++ b/skyflow/vault/data/_update_response.py @@ -1,7 +1,7 @@ from skyflow.generated.rest import V1UpdateRecordResponse class UpdateResponse: - def __init__(self, updated_field, error=None): + def __init__(self, updated_field = None, error=None): self.updated_field = updated_field self.error = error if error is not None else [] @@ -10,12 +10,3 @@ def __repr__(self): def __str__(self): return self.__repr__() - - @staticmethod - def parse_update_field_response(update_response: V1UpdateRecordResponse): - updated_field = {} - updated_field['skyflow_id'] = update_response.skyflow_id - if update_response.tokens is not None: - updated_field.update(update_response.tokens) - - return UpdateResponse(updated_field) diff --git a/skyflow/vault/tokens/_detokenize_request.py b/skyflow/vault/tokens/_detokenize_request.py index 6864180..e4775b4 100644 --- a/skyflow/vault/tokens/_detokenize_request.py +++ b/skyflow/vault/tokens/_detokenize_request.py @@ -1,5 +1,8 @@ +from skyflow.utils.enums import Redaction + + class DetokenizeRequest: - def __init__(self, tokens,redaction_type): + def __init__(self, tokens,redaction_type = Redaction.PLAIN_TEXT, continue_on_error = False): self.tokens = tokens self.redaction_type = redaction_type - self.continue_on_error = bool \ No newline at end of file + self.continue_on_error = continue_on_error \ No newline at end of file diff --git a/skyflow/vault/tokens/_detokenize_response.py b/skyflow/vault/tokens/_detokenize_response.py index 5af907f..8ec072f 100644 --- a/skyflow/vault/tokens/_detokenize_response.py +++ b/skyflow/vault/tokens/_detokenize_response.py @@ -1,7 +1,7 @@ from skyflow.generated.rest import V1DetokenizeResponse class DetokenizeResponse: - def __init__(self, detokenized_fields, errors): + def __init__(self, detokenized_fields = None, errors = None): self.detokenized_fields = detokenized_fields self.errors = errors @@ -11,27 +11,4 @@ def __repr__(self): def __str__(self): return self.__repr__() - @staticmethod - def parse_detokenize_response(v1_detokenize_response: V1DetokenizeResponse): - detokenized_fields = [] - errors = [] - - for record in v1_detokenize_response.records: - if record.error: - errors.append({ - "token": record.token, - "error": record.error - }) - else: - value_type = record.value_type.value if record.value_type else None - detokenized_fields.append({ - "token": record.token, - "value": record.value, - "type": value_type - }) - - return DetokenizeResponse( - detokenized_fields=detokenized_fields, - errors=errors - ) diff --git a/skyflow/vault/tokens/_tokenize_response.py b/skyflow/vault/tokens/_tokenize_response.py index 2a08ed8..dd1e855 100644 --- a/skyflow/vault/tokens/_tokenize_response.py +++ b/skyflow/vault/tokens/_tokenize_response.py @@ -1,8 +1,5 @@ -from skyflow.generated.rest import V1TokenizeResponse - - class TokenizeResponse: - def __init__(self, tokenized_fields): + def __init__(self, tokenized_fields = None): self.tokenized_fields = tokenized_fields @@ -12,10 +9,3 @@ def __repr__(self): def __str__(self): return self.__repr__() - @staticmethod - def parse_tokenize_response(tokenize_response: V1TokenizeResponse): - tokenized_fields = [{"token": record.token} for record in tokenize_response.records] - - return TokenizeResponse( - tokenized_fields = tokenized_fields - ) \ No newline at end of file