diff --git a/samples/delete_by_id_sample.py b/samples/delete_by_id_sample.py new file mode 100644 index 0000000..8846c50 --- /dev/null +++ b/samples/delete_by_id_sample.py @@ -0,0 +1,40 @@ +''' + Copyright (c) 2022 Skyflow, Inc. +''' +from skyflow.errors import SkyflowError +from skyflow.service_account import generate_bearer_token, is_expired +from skyflow.vault import Client, Configuration,DeleteOptions + + +# cache token for reuse +bearerToken = '' + + +def token_provider(): + global bearerToken + if is_expired(bearerToken): + bearerToken, _ = generate_bearer_token('') + return bearerToken + + +try: + config = Configuration( + '', '', token_provider) + client = Client(config) + options = DeleteOptions(False) + + data = {"records": [ + { + "id": "", + "table": "", + }, + { + "id": "", + "table": "", + } + ]} + + response = client.delete_by_id(data,options=options) + print('Response:', response) +except SkyflowError as e: + print('Error Occurred:', e) diff --git a/skyflow/_utils.py b/skyflow/_utils.py index 1c56830..c53d1f9 100644 --- a/skyflow/_utils.py +++ b/skyflow/_utils.py @@ -75,6 +75,8 @@ class InfoMessages(Enum): UPDATE_DATA_SUCCESS = "Data has been updated successfully" GET_TRIGGERED = "Get triggered." GET_SUCCESS = "Data fetched successfully." + DELETE_BY_ID_TRIGGERED = "Delete by ID triggered." + DELETE_DATA_SUCCESS = "Data has been deleted successfully." class InterfaceName(Enum): @@ -89,6 +91,7 @@ class InterfaceName(Enum): IS_TOKEN_VALID = "service_account.isTokenValid" IS_EXPIRED = "service_account.is_expired" + DELETE_BY_ID = "client.delete_by_id" def http_build_query(data): diff --git a/skyflow/errors/_skyflow_errors.py b/skyflow/errors/_skyflow_errors.py index a2dbbf0..414d91f 100644 --- a/skyflow/errors/_skyflow_errors.py +++ b/skyflow/errors/_skyflow_errors.py @@ -7,6 +7,7 @@ class SkyflowErrorCodes(Enum): INVALID_INPUT = 400 + INVALID_INDEX = 404 SERVER_ERROR = 500 PARTIAL_SUCCESS = 500 @@ -43,11 +44,18 @@ class SkyflowErrorMessages(Enum): INVALID_RECORDS_TYPE = "Records key has value of type %s, expected list" INVALID_FIELDS_TYPE = "Fields key has value of type %s, expected dict" INVALID_TABLE_TYPE = "Table key has value of type %s, expected string" + INVALID_TABLE_TYPE_DELETE = "Table of type string is required at index %s in records array" INVALID_IDS_TYPE = "Ids key has value of type %s, expected list" INVALID_ID_TYPE = "Id key has value of type %s, expected string" + INVALID_ID_TYPE_DELETE = "Id of type string is required at index %s in records array" INVALID_REDACTION_TYPE = "Redaction key has value of type %s, expected Skyflow.Redaction" INVALID_COLUMN_NAME = "Column name has value of type %s, expected string" INVALID_COLUMN_VALUE = "Column values has value of type %s, expected list" + INVALID_RECORDS_IN_DELETE = "Invalid records. records object should be an array" + EMPTY_RECORDS_IN_DELETE = "records array cannot be empty" + EMPTY_ID_IN_DELETE = "Id cannot be empty in records array at index %s" + EMPTY_TABLE_IN_DELETE = "Table cannot be empty in records array at index %s" + RECORDS_KEY_NOT_FOUND_DELETE = "records object is required" INVALID_REQUEST_BODY = "Given request body is not valid" INVALID_RESPONSE_BODY = "Given response body is not valid" diff --git a/skyflow/vault/_client.py b/skyflow/vault/_client.py index a422b10..303a4cf 100644 --- a/skyflow/vault/_client.py +++ b/skyflow/vault/_client.py @@ -4,9 +4,11 @@ import json import types import requests + +from ._delete_by_id import deleteProcessResponse from ._insert import getInsertRequestBody, processResponse, convertResponse from ._update import sendUpdateRequests, createUpdateResponseBody -from ._config import Configuration +from ._config import Configuration, DeleteOptions from ._config import InsertOptions, ConnectionConfig, UpdateOptions from ._connection import createRequest from ._detokenize import sendDetokenizeRequests, createDetokenizeResponseBody @@ -172,4 +174,85 @@ def update(self, updateInput, options: UpdateOptions = UpdateOptions()): SkyflowErrorMessages.PARTIAL_SUCCESS, result, interface=interface) else: log_info(InfoMessages.UPDATE_DATA_SUCCESS.value, interface) - return result \ No newline at end of file + return result + + def delete_by_id(self, records: dict,options: DeleteOptions = DeleteOptions()): + interface = InterfaceName.DELETE_BY_ID.value + log_info(InfoMessages.DELETE_BY_ID_TRIGGERED.value, interface=interface) + + self._checkConfig(interface) + + self.storedToken = tokenProviderWrapper( + self.storedToken, self.tokenProvider, interface) + headers = { + "Authorization": "Bearer " + self.storedToken, + "sky-metadata": json.dumps(getMetrics()) + } + error_list = [] + result_list = [] + errors = {} + result = {} + error = {} + try: + if not isinstance(records, dict) or "records" not in records: + error = {"error": {"code": SkyflowErrorCodes.INVALID_INPUT.value, + "description": SkyflowErrorMessages.RECORDS_KEY_NOT_FOUND_DELETE.value}} + return error + records_list = records["records"] + if not isinstance(records_list, list): + error.update({"error": {"code": SkyflowErrorCodes.INVALID_INPUT.value, + "description": SkyflowErrorMessages.INVALID_RECORDS_IN_DELETE.value}}) + return error + elif len(records_list) == 0: + error = {"error": {"code": SkyflowErrorCodes.INVALID_INPUT.value, + "description": SkyflowErrorMessages.EMPTY_RECORDS_IN_DELETE.value}} + return error + except KeyError: + raise SkyflowError(SkyflowErrorCodes.INVALID_INPUT, + SkyflowErrorMessages.RECORDS_KEY_ERROR, interface=interface) + try: + for index,record in enumerate(records["records"]): + record_list = record["id"] + if not isinstance(record_list, str): + error.update({"error": {"code": SkyflowErrorCodes.INVALID_INDEX.value, + "description": SkyflowErrorMessages.INVALID_ID_TYPE_DELETE.value % (index)}}) + return error + elif record_list == "": + error.update({"error": {"code": SkyflowErrorCodes.INVALID_INPUT.value, + "description": SkyflowErrorMessages.EMPTY_ID_IN_DELETE.value % (index)}}) + return error + except KeyError: + error.update({"error": {"code": SkyflowErrorCodes.INVALID_INDEX.value, + "description": SkyflowErrorMessages.IDS_KEY_ERROR.value}}) + return error + try: + for index,record in enumerate(records["records"]): + record_table = record["table"] + if not isinstance(record_table, str): + error.update({"error": {"code": SkyflowErrorCodes.INVALID_INPUT.value, + "description": SkyflowErrorMessages.INVALID_TABLE_TYPE_DELETE.value % (index)}}) + return error + elif record_table == "": + error.update({"error": {"code": SkyflowErrorCodes.INVALID_INPUT.value, + "description": SkyflowErrorMessages.EMPTY_TABLE_IN_DELETE.value % (index)}}) + return error + except KeyError: + error.update({"error": {"code": SkyflowErrorCodes.INVALID_INDEX.value, + "description": SkyflowErrorMessages.TABLE_KEY_ERROR.value}}) + return error + for record in records["records"]: + request_url = self._get_complete_vault_url() + "/" + record["table"] + "/" + record["id"] + response = requests.delete(request_url, headers=headers) + processed_response = deleteProcessResponse(response, records) + if processed_response is not None and processed_response.get('code') == 404: + errors.update({'id': record["id"], 'error': processed_response}) + error_list.append(errors) + else: + result_list.append(processed_response) + if result_list: + result.update({'records': result_list}) + if errors: + result.update({'errors': error_list}) + + log_info(InfoMessages.DELETE_DATA_SUCCESS.value, interface) + return result diff --git a/skyflow/vault/_config.py b/skyflow/vault/_config.py index 25fe3cd..77abd39 100644 --- a/skyflow/vault/_config.py +++ b/skyflow/vault/_config.py @@ -38,6 +38,10 @@ class UpdateOptions: def __init__(self, tokens: bool=True): self.tokens = tokens +class DeleteOptions: + def __init__(self, tokens: bool=False): + self.tokens = tokens + class RequestMethod(Enum): GET = 'GET' POST = 'POST' diff --git a/skyflow/vault/_delete_by_id.py b/skyflow/vault/_delete_by_id.py new file mode 100644 index 0000000..f275c93 --- /dev/null +++ b/skyflow/vault/_delete_by_id.py @@ -0,0 +1,41 @@ +''' + Copyright (c) 2022 Skyflow, Inc. +''' +import json + +import requests +from requests.models import HTTPError +from skyflow.errors._skyflow_errors import SkyflowError, SkyflowErrorCodes, SkyflowErrorMessages +from skyflow._utils import InterfaceName + +interface = InterfaceName.DELETE_BY_ID.value + + +def deleteProcessResponse(response: requests.Response, interface=interface): + statusCode = response.status_code + content = response.content + try: + response.raise_for_status() + if statusCode == 204: + return None + try: + return json.loads(content) + except: + raise SkyflowError( + statusCode, SkyflowErrorMessages.RESPONSE_NOT_JSON.value % content, interface=interface) + except HTTPError: + message = SkyflowErrorMessages.API_ERROR.value % statusCode + if response is not None and response.content is not None: + try: + errorResponse = json.loads(content) + if 'error' in errorResponse and type(errorResponse['error']) == dict and 'message' in errorResponse[ + 'error']: + message = errorResponse['error']['message'] + except: + message = SkyflowErrorMessages.RESPONSE_NOT_JSON.value % content + error = {} + if 'x-request-id' in response.headers: + message += ' - request id: ' + response.headers['x-request-id'] + error.update({"code": statusCode, "description": message}) + return error + diff --git a/tests/vault/test_delete_by_id.py b/tests/vault/test_delete_by_id.py new file mode 100644 index 0000000..9617ac4 --- /dev/null +++ b/tests/vault/test_delete_by_id.py @@ -0,0 +1,217 @@ +import json +import unittest +import os + +import asyncio +import warnings +from unittest import mock +from unittest.mock import patch, MagicMock + +import requests +from requests import HTTPError +from requests.models import Response +from dotenv import dotenv_values + +from skyflow.errors import SkyflowError, SkyflowErrorCodes +from skyflow.errors._skyflow_errors import SkyflowErrorMessages +from skyflow.service_account import generate_bearer_token +from skyflow.vault._client import Client +from skyflow.vault._config import Configuration,DeleteOptions +from skyflow.vault._delete_by_id import deleteProcessResponse + + +class TestDelete(unittest.TestCase): + + def setUp(self) -> None: + self.envValues = dotenv_values(".env") + self.dataPath = os.path.join(os.getcwd(), 'tests/vault/data/') + self.event_loop = asyncio.new_event_loop() + self.mocked_futures = [] + + def tokenProvider(): + token, type = generate_bearer_token( + self.envValues["CREDENTIALS_FILE_PATH"]) + return token + + config = Configuration( + "12345", "demo", tokenProvider) + self.client = Client(config) + warnings.filterwarnings( + action="ignore", message="unclosed", category=ResourceWarning) + + self.record_id = "123" + + self.mockResponse = { + "responses": [ + { + "records": [ + { + "skyflow_id": self.record_id, + "deleted": True + } + ] + } + ] + } + self.DeleteOptions = DeleteOptions(tokens=False) + + return super().setUp() + + def getDataPath(self, file): + return self.dataPath + file + '.json' + + def testDeleteByIdInvalidRecordsType(self): + invalidData = "invalid_data" + result = self.client.delete_by_id(invalidData) + self.assertEqual(result, { + "error": { + "code": SkyflowErrorCodes.INVALID_INPUT.value, + "description": SkyflowErrorMessages.RECORDS_KEY_NOT_FOUND_DELETE.value + } + }) + + def testDeleteByIdMissingRecordsKey(self): + invalidData = {"some_other_key": "value"} + result = self.client.delete_by_id(invalidData) + self.assertEqual(result, { + "error": { + "code": SkyflowErrorCodes.INVALID_INPUT.value, + "description": SkyflowErrorMessages.RECORDS_KEY_NOT_FOUND_DELETE.value + } + }) + + def testDeleteByIdInvalidRecordsListType(self): + invalidData = {"records": "invalid_data"} + result = self.client.delete_by_id(invalidData) + self.assertEqual(result, { + "error": { + "code": SkyflowErrorCodes.INVALID_INPUT.value, + "description": SkyflowErrorMessages.INVALID_RECORDS_IN_DELETE.value + } + }) + + def testDeleteByIdEmptyRecordsListType(self): + invalidData = {"records": []} + result = self.client.delete_by_id(invalidData) + # Assert the error response for an empty "records_list". + self.assertEqual(result, { + "error": { + "code": SkyflowErrorCodes.INVALID_INPUT.value, + "description": SkyflowErrorMessages.EMPTY_RECORDS_IN_DELETE.value + } + }) + + def testDeleteByIdEmptyTable(self): + invalidData = {"records": [{"id": "id1", "table": ""}]} + response = self.client.delete_by_id(invalidData) + self.assertIn("error", response) + error = response["error"] + self.assertEqual(error["code"], SkyflowErrorCodes.INVALID_INPUT.value) + self.assertEqual(error["description"], SkyflowErrorMessages.EMPTY_TABLE_IN_DELETE.value % (0)) + + def testDeleteByIdEmptyId(self): + invalidData = {"records": [{"id": "", "table": "stripe"}]} + response = self.client.delete_by_id(invalidData) + self.assertIn("error", response) + error = response["error"] + self.assertEqual(error["code"], SkyflowErrorCodes.INVALID_INPUT.value) + self.assertEqual(error["description"], SkyflowErrorMessages.EMPTY_ID_IN_DELETE.value % (0)) + + def testDeleteByIdInvalidIdType(self): + invalidData = {"records": [ + {"id": ["invalid"], "table": "stripe"}]} + response = self.client.delete_by_id(invalidData) + self.assertIn("error", response) + error = response["error"] + self.assertEqual(error["code"], SkyflowErrorCodes.INVALID_INDEX.value) + self.assertEqual(error["description"], SkyflowErrorMessages.INVALID_ID_TYPE_DELETE.value % (0)) + + def testDeleteByIdNoId(self): + invalidData = {"records": [{"invalid": "invalid", "table": "stripe"}]} + response = self.client.delete_by_id(invalidData) + self.assertIn("error", response) + error = response["error"] + self.assertEqual(error["code"], SkyflowErrorCodes.INVALID_INDEX.value) + self.assertEqual(error["description"], SkyflowErrorMessages.IDS_KEY_ERROR.value) + + def testDeleteByIdNoTable(self): + invalidData = {"records": [{"id": "id1", "invalid": "invalid"}]} + response = self.client.delete_by_id(invalidData) + self.assertIn("error", response) + error = response["error"] + self.assertEqual(error["code"], SkyflowErrorCodes.INVALID_INDEX.value) + self.assertEqual(error["description"], SkyflowErrorMessages.TABLE_KEY_ERROR.value) + + def testDeleteByIdInvalidTableType(self): + invalidData = {"records": [ + {"id": "id1", "table": ["invalid"]}]} + result = self.client.delete_by_id(invalidData) + self.assertIn("error", result) + error = result["error"] + self.assertEqual(error["code"], SkyflowErrorCodes.INVALID_INPUT.value) + self.assertEqual(error["description"], SkyflowErrorMessages.INVALID_TABLE_TYPE_DELETE.value % (0)) + + def testDeleteProcessResponseWithSuccessfulResponse(self): + mock_response = requests.Response() + mock_response.status_code = 200 + mock_response._content = b'{"key": "value"}' + result = deleteProcessResponse(mock_response) + self.assertIsInstance(result, dict) + self.assertEqual(result, {"key": "value"}) + + def testDeleteProcessResponseWithNoContentResponse(self): + mock_response = requests.Response() + mock_response.status_code = 204 + result = deleteProcessResponse(mock_response) + self.assertIsNone(result) + + def test_http_error_with_error_message(self): + error_response = { + 'code': 400, + 'description': 'Error occurred' + } + response = mock.Mock(spec=requests.Response) + response.status_code = 400 + response.content = json.dumps(error_response).encode() + error = deleteProcessResponse(response) + self.assertEqual(error, { + "code": 400, + "description": "Error occurred", + }) + + def test_delete_data_success(self): + records = {"records": [ + {"id": "id1", "table": "stripe"}]} + self.mock_response = mock.Mock(spec=requests.Response) + self.mock_response.status_code = 204 + self.mock_response.content = b'' + with mock.patch('requests.delete', return_value=self.mock_response): + result = self.client.delete_by_id(records) + self.assertIn('records', result) + self.assertEqual(result['records'], [None]) + + def test_delete_data_with_errors(self): + response = mock.Mock(spec=requests.Response) + response.status_code = 404 + response.content = b'{"code": 404, "description": "Not found"}' + with mock.patch('requests.delete', return_value=response): + records = {"records": [ + {"id": "id1", "table": "stripe"}, + ]} + result = self.client.delete_by_id(records) + + self.assertIn('errors', result) + error = result['errors'][0] + self.assertEqual(error['id'], "id1") + self.assertEqual(error['error'], {'code': 404, 'description': 'Not found'}) + + def testDeleteProcessInvalidResponse(self): + response = Response() + response.status_code = 500 + response._content = b"Invalid Request" + try: + deleteProcessResponse(response) + except SkyflowError as e: + self.assertEqual(e.code, 500) + self.assertEqual(e.message, SkyflowErrorMessages.RESPONSE_NOT_JSON.value % + response.content.decode('utf-8')) \ No newline at end of file