From 6add72a5d3346096e8828ec7c217f8dbf284c3bd Mon Sep 17 00:00:00 2001 From: skyflow-bharti Date: Wed, 17 May 2023 17:59:59 +0530 Subject: [PATCH 1/3] SK-678 add metrices in python-sdk --- .github/workflows/release.yml | 1 + ci-scripts/bump_version.sh | 3 +++ skyflow/_utils.py | 14 ++++++++++++++ skyflow/vault/_client.py | 6 ++++-- skyflow/vault/_detokenize.py | 6 ++++-- skyflow/vault/_get.py | 6 ++++-- skyflow/vault/_get_by_id.py | 5 +++-- skyflow/vault/_update.py | 5 +++-- tests/vault/test_url_encoder.py | 16 ++++++++++++++-- version.py | 1 + 10 files changed, 51 insertions(+), 12 deletions(-) create mode 100644 version.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1c47f8..9888770 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,6 +39,7 @@ jobs: git config user.name ${{ github.actor }} git config user.email ${{ github.actor }}@users.noreply.github.com git add setup.py + git add version.py git commit -m "[AUTOMATED] Public Release - ${{ steps.previoustag.outputs.tag }}" git push origin diff --git a/ci-scripts/bump_version.sh b/ci-scripts/bump_version.sh index 59f00df..5f02750 100755 --- a/ci-scripts/bump_version.sh +++ b/ci-scripts/bump_version.sh @@ -6,6 +6,8 @@ then echo "Bumping package version to $1" sed -E "s/current_version = .+/current_version = \'$SEMVER\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile + sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER\'/g" version.py > tempfile && cat tempfile > version.py && rm -f tempfile + echo -------------------------- echo "Done, Package now at $1" @@ -13,6 +15,7 @@ else echo "Bumping package version to $1-dev.$2" sed -E "s/current_version = .+/current_version = \'$SEMVER-dev.$2\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile + sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER-dev.$2\'/g" version.py > tempfile && cat tempfile > version.py && rm -f tempfile echo -------------------------- echo "Done, Package now at $1-dev.$2" diff --git a/skyflow/_utils.py b/skyflow/_utils.py index 5186dbe..1e165f4 100644 --- a/skyflow/_utils.py +++ b/skyflow/_utils.py @@ -4,6 +4,9 @@ import urllib.parse import logging from enum import Enum +import platform +import sys +from version import SDK_VERSION skyflowLog = logging.getLogger('skyflow') skyflowLog.setLevel(logging.ERROR) @@ -127,3 +130,14 @@ def render_key(parents): outStr += s % str(x) depth += 1 return outStr + +def getMetrics(): + ''' fetch metrics + ''' + details_dic = { + 'sdk_name_version': "skyflow-python@" + SDK_VERSION, + 'sdk_client_device_model': platform.node(), + 'sdk_client_os_details': sys.platform, + 'sdk_runtime_details': sys.version, + } + return details_dic \ No newline at end of file diff --git a/skyflow/vault/_client.py b/skyflow/vault/_client.py index ff7aaa4..d8febed 100644 --- a/skyflow/vault/_client.py +++ b/skyflow/vault/_client.py @@ -1,6 +1,7 @@ ''' Copyright (c) 2022 Skyflow, Inc. ''' +import json import types import requests from ._insert import getInsertRequestBody, processResponse, convertResponse @@ -13,7 +14,7 @@ from ._get import sendGetRequests import asyncio from skyflow.errors._skyflow_errors import SkyflowError, SkyflowErrorCodes, SkyflowErrorMessages -from skyflow._utils import log_info, InfoMessages, InterfaceName +from skyflow._utils import log_info, InfoMessages, InterfaceName, getMetrics from ._token import tokenProviderWrapper @@ -52,7 +53,8 @@ def insert(self, records: dict, options: InsertOptions = InsertOptions()): self.storedToken = tokenProviderWrapper( self.storedToken, self.tokenProvider, interface) headers = { - "Authorization": "Bearer " + self.storedToken + "Authorization": "Bearer " + self.storedToken, + "sky-metadata": json.dumps(getMetrics()) } response = requests.post(requestURL, data=jsonBody, headers=headers) diff --git a/skyflow/vault/_detokenize.py b/skyflow/vault/_detokenize.py index de5d09f..8370de0 100644 --- a/skyflow/vault/_detokenize.py +++ b/skyflow/vault/_detokenize.py @@ -5,7 +5,7 @@ import asyncio from aiohttp import ClientSession, request import json -from skyflow._utils import InterfaceName +from skyflow._utils import InterfaceName, getMetrics interface = InterfaceName.DETOKENIZE.value @@ -48,7 +48,9 @@ async def sendDetokenizeRequests(data, url, token): async with ClientSession() as session: for record in validatedRecords: headers = { - "Authorization": "Bearer " + token + "Authorization": "Bearer " + token, + "sky-metadata": json.dumps(getMetrics()) + } task = asyncio.ensure_future(post(url, record, headers, session)) tasks.append(task) diff --git a/skyflow/vault/_get.py b/skyflow/vault/_get.py index 0670206..8099366 100644 --- a/skyflow/vault/_get.py +++ b/skyflow/vault/_get.py @@ -1,11 +1,12 @@ ''' Copyright (c) 2022 Skyflow, Inc. ''' +import json from skyflow.errors._skyflow_errors import SkyflowError, SkyflowErrorCodes, SkyflowErrorMessages import asyncio from aiohttp import ClientSession from ._config import RedactionType -from skyflow._utils import InterfaceName +from skyflow._utils import InterfaceName, getMetrics from ._get_by_id import get interface = InterfaceName.GET.value @@ -83,7 +84,8 @@ async def sendGetRequests(data, url, token): async with ClientSession() as session: for record in validatedRecords: headers = { - "Authorization": "Bearer " + token + "Authorization": "Bearer " + token, + "sky-metadata": json.dumps(getMetrics()) } params = {"redaction": redaction} if ids is not None: diff --git a/skyflow/vault/_get_by_id.py b/skyflow/vault/_get_by_id.py index 7cd3adc..1ac7a6f 100644 --- a/skyflow/vault/_get_by_id.py +++ b/skyflow/vault/_get_by_id.py @@ -6,7 +6,7 @@ from aiohttp import ClientSession import json from ._config import RedactionType -from skyflow._utils import InterfaceName +from skyflow._utils import InterfaceName, getMetrics interface = InterfaceName.GET_BY_ID.value @@ -66,7 +66,8 @@ async def sendGetByIdRequests(data, url, token): async with ClientSession() as session: for record in validatedRecords: headers = { - "Authorization": "Bearer " + token + "Authorization": "Bearer " + token, + "sky-metadata": json.dumps(getMetrics()) } params = {"skyflow_ids": record[0], "redaction": record[2]} task = asyncio.ensure_future( diff --git a/skyflow/vault/_update.py b/skyflow/vault/_update.py index ade08b2..c27a091 100644 --- a/skyflow/vault/_update.py +++ b/skyflow/vault/_update.py @@ -6,7 +6,7 @@ import asyncio from skyflow.errors._skyflow_errors import SkyflowError, SkyflowErrorCodes, SkyflowErrorMessages from ._insert import getTableAndFields -from skyflow._utils import InterfaceName +from skyflow._utils import InterfaceName, getMetrics from aiohttp import ClientSession from ._config import UpdateOptions @@ -40,7 +40,8 @@ async def sendUpdateRequests(data,options: UpdateOptions,url,token): } reqBody = json.dumps(reqBody) headers = { - "Authorization": "Bearer " + token + "Authorization": "Bearer " + token, + "sky-metadata": json.dumps(getMetrics()) } task = asyncio.ensure_future(put(recordUrl, reqBody, headers, session)) tasks.append(task) diff --git a/tests/vault/test_url_encoder.py b/tests/vault/test_url_encoder.py index c36a468..80d99db 100644 --- a/tests/vault/test_url_encoder.py +++ b/tests/vault/test_url_encoder.py @@ -1,9 +1,11 @@ ''' Copyright (c) 2022 Skyflow, Inc. ''' +import platform +import sys import unittest -from skyflow._utils import http_build_query - +from skyflow._utils import http_build_query, getMetrics +from version import SDK_VERSION class TestUrlEncoder(unittest.TestCase): def setUp(self) -> None: @@ -50,3 +52,13 @@ def test_encoder_array(self): self.assertEqual( http_data, "key=value&nested%5Barray%5D%5B0%5D=one&nested%5Barray%5D%5B1%5D=two&nested%5Bkey%5D=value") + + def test_get_metrics(self): + expected = { + 'sdk_name_version': "skyflow-python@" + SDK_VERSION, + 'sdk_client_device_model': platform.node(), + 'sdk_client_os_details': sys.platform, + 'sdk_runtime_details': sys.version, + } + actual = getMetrics() + self.assertEqual(actual, expected) \ No newline at end of file diff --git a/version.py b/version.py new file mode 100644 index 0000000..ccff82c --- /dev/null +++ b/version.py @@ -0,0 +1 @@ +SDK_VERSION = '1.8.1' \ No newline at end of file From a501618da01204909fba6d17be9ff1386d19b35a Mon Sep 17 00:00:00 2001 From: skyflow-bharti Date: Mon, 22 May 2023 09:51:56 +0530 Subject: [PATCH 2/3] SK-678 Added metrics in invoke connection header --- .github/workflows/release.yml | 1 + skyflow/_utils.py | 25 +++++++++++--- skyflow/service_account/_token.py | 5 +-- skyflow/vault/_client.py | 2 ++ tests/vault/test_url_encoder.py | 55 +++++++++++++++++++++++++++++-- 5 files changed, 80 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9888770..e87b00f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,6 +7,7 @@ on: - "setup.py" - "*.yml" - "*.md" + - "version.py" jobs: build-and-deploy: diff --git a/skyflow/_utils.py b/skyflow/_utils.py index 1e165f4..39d7e6d 100644 --- a/skyflow/_utils.py +++ b/skyflow/_utils.py @@ -134,10 +134,27 @@ def render_key(parents): def getMetrics(): ''' fetch metrics ''' + sdk_name_version = "skyflow-python@" + SDK_VERSION + + try: + sdk_client_device_model = platform.node() + except Exception: + sdk_client_device_model = "" + + try: + sdk_client_os_details = sys.platform + except Exception: + sdk_client_os_details = "" + + try: + sdk_runtime_details = sys.version + except Exception: + sdk_runtime_details = "" + details_dic = { - 'sdk_name_version': "skyflow-python@" + SDK_VERSION, - 'sdk_client_device_model': platform.node(), - 'sdk_client_os_details': sys.platform, - 'sdk_runtime_details': sys.version, + 'sdk_name_version': sdk_name_version, + 'sdk_client_device_model': sdk_client_device_model, + 'sdk_client_os_details': sdk_client_os_details, + 'sdk_runtime_details': "Python " + sdk_runtime_details, } return details_dic \ No newline at end of file diff --git a/skyflow/service_account/_token.py b/skyflow/service_account/_token.py index e459944..f73191b 100644 --- a/skyflow/service_account/_token.py +++ b/skyflow/service_account/_token.py @@ -7,7 +7,7 @@ import requests from warnings import warn from collections import namedtuple -from skyflow._utils import log_info, InterfaceName, InfoMessages +from skyflow._utils import log_info, InterfaceName, InfoMessages, getMetrics from skyflow.errors._skyflow_errors import * @@ -126,7 +126,8 @@ def getSignedJWT(clientID, keyID, tokenURI, privateKey): def sendRequestWithToken(url, token): headers = { - "content-type": "application/json" + "content-type": "application/json", + "sky-metadata": json.dumps(getMetrics()) } payload = { "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", diff --git a/skyflow/vault/_client.py b/skyflow/vault/_client.py index d8febed..a422b10 100644 --- a/skyflow/vault/_client.py +++ b/skyflow/vault/_client.py @@ -133,6 +133,8 @@ def invoke_connection(self, config: ConnectionConfig): if not 'X-Skyflow-Authorization'.lower() in request.headers: request.headers['x-skyflow-authorization'] = self.storedToken + request.headers['sky-metadata'] = json.dumps(getMetrics()) + response = session.send(request) session.close() return processResponse(response, interface=interface) diff --git a/tests/vault/test_url_encoder.py b/tests/vault/test_url_encoder.py index 80d99db..9afdf51 100644 --- a/tests/vault/test_url_encoder.py +++ b/tests/vault/test_url_encoder.py @@ -4,6 +4,7 @@ import platform import sys import unittest +from unittest import mock from skyflow._utils import http_build_query, getMetrics from version import SDK_VERSION @@ -53,12 +54,62 @@ def test_encoder_array(self): self.assertEqual( http_data, "key=value&nested%5Barray%5D%5B0%5D=one&nested%5Barray%5D%5B1%5D=two&nested%5Bkey%5D=value") + # Test Case 1: Success case def test_get_metrics(self): expected = { 'sdk_name_version': "skyflow-python@" + SDK_VERSION, 'sdk_client_device_model': platform.node(), 'sdk_client_os_details': sys.platform, - 'sdk_runtime_details': sys.version, + 'sdk_runtime_details': "Python " + sys.version, } actual = getMetrics() - self.assertEqual(actual, expected) \ No newline at end of file + self.assertEqual(actual, expected) + + @mock.patch('platform.node', return_value='') + def test_getMetrics_no_device_model(self, mock_node): + expected_output = { + 'sdk_name_version': 'skyflow-python@' + SDK_VERSION, + 'sdk_client_device_model': '', + 'sdk_client_os_details': sys.platform, + 'sdk_runtime_details': "Python " + sys.version + } + + actual_output = getMetrics() + expected_output['sdk_client_device_model'] = '' + self.assertEqual(actual_output, expected_output) + + @mock.patch('platform.node', return_value='Mocked Device Model') + def test_getMetrics_with_device_model(self, mock_node): + expected_output = { + 'sdk_name_version': 'skyflow-python@' + SDK_VERSION, + 'sdk_client_device_model': 'Mocked Device Model', + 'sdk_client_os_details': sys.platform, + 'sdk_runtime_details': "Python " + sys.version + } + + actual_output = getMetrics() + self.assertEqual(actual_output, expected_output) + + @mock.patch('sys.platform', return_value='mocked_os') + def test_getMetrics_with_os_details(self, mock_platform): + expected_output = { + 'sdk_name_version': 'skyflow-python@' + SDK_VERSION, + 'sdk_client_device_model': platform.node(), + 'sdk_client_os_details': sys.platform, + 'sdk_runtime_details': "Python " + sys.version + } + actual_output = getMetrics() + self.assertEqual(actual_output, expected_output) + + def test_getMetrics_with_runtime_details(self): + expected_output = { + 'sdk_name_version': 'skyflow-python@' + SDK_VERSION, + 'sdk_client_device_model': platform.node(), + 'sdk_client_os_details': sys.platform, + 'sdk_runtime_details': 'Python ' + 'mocked_version' + } + + with mock.patch('sys.version', 'mocked_version'), \ + mock.patch('sys.version_info', new=(3, 11, 2)): + actual_output = getMetrics() + self.assertEqual(actual_output, expected_output) From cafeb293529f372d7066951f2509fd0163c5223f Mon Sep 17 00:00:00 2001 From: skyflow-bharti Date: Mon, 29 May 2023 19:06:09 +0530 Subject: [PATCH 3/3] SK-734-add-redaction-type-in-reveal-and-detokenize-in-python-sdk --- skyflow/vault/_detokenize.py | 19 ++++++++++--- tests/vault/test_detokenize.py | 49 +++++++++++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/skyflow/vault/_detokenize.py b/skyflow/vault/_detokenize.py index de5d09f..7e2c4cc 100644 --- a/skyflow/vault/_detokenize.py +++ b/skyflow/vault/_detokenize.py @@ -6,6 +6,7 @@ from aiohttp import ClientSession, request import json from skyflow._utils import InterfaceName +from ._config import RedactionType interface = InterfaceName.DETOKENIZE.value @@ -15,14 +16,27 @@ def getDetokenizeRequestBody(data): token = data["token"] except KeyError: raise SkyflowError(SkyflowErrorCodes.INVALID_INPUT, - SkyflowErrorMessages.TOKEN_KEY_ERROR, interface=interface) + SkyflowErrorMessages.TOKEN_KEY_ERROR, interface=interface) if not isinstance(token, str): tokenType = str(type(token)) raise SkyflowError(SkyflowErrorCodes.INVALID_INPUT, SkyflowErrorMessages.INVALID_TOKEN_TYPE.value % ( tokenType), interface=interface) + + if "redaction" in data: + if not isinstance(data["redaction"], RedactionType): + redactionType = str(type(data["redaction"])) + raise SkyflowError(SkyflowErrorCodes.INVALID_INPUT, SkyflowErrorMessages.INVALID_REDACTION_TYPE.value % ( + redactionType), interface=interface) + else: + redactionType = data["redaction"] + else: + redactionType = RedactionType.PLAIN_TEXT + requestBody = {"detokenizationParameters": []} requestBody["detokenizationParameters"].append({ - "token": token}) + "token": token, + "redaction": redactionType.value + }) return requestBody @@ -39,7 +53,6 @@ async def sendDetokenizeRequests(data, url, token): recordsType = str(type(records)) raise SkyflowError(SkyflowErrorCodes.INVALID_INPUT, SkyflowErrorMessages.INVALID_RECORDS_TYPE.value % ( recordsType), interface=interface) - validatedRecords = [] for record in records: requestBody = getDetokenizeRequestBody(record) diff --git a/tests/vault/test_detokenize.py b/tests/vault/test_detokenize.py index 609d954..8edbfb3 100644 --- a/tests/vault/test_detokenize.py +++ b/tests/vault/test_detokenize.py @@ -7,6 +7,7 @@ from skyflow.errors._skyflow_errors import SkyflowError, SkyflowErrorCodes, SkyflowErrorMessages from skyflow.vault._client import Client, Configuration from skyflow.service_account import generate_bearer_token +from skyflow.vault._config import RedactionType from dotenv import dotenv_values import warnings @@ -55,7 +56,8 @@ def testGetDetokenizeRequestBodyWithValidBody(self): body = getDetokenizeRequestBody(self.tokenField) expectedOutput = { "detokenizationParameters": [{ - "token": self.testToken + "token": self.testToken, + "redaction": "PLAIN_TEXT" }] } @@ -101,6 +103,15 @@ def testDetokenizeTokenInvalidType(self): self.assertEqual( e.message, SkyflowErrorMessages.INVALID_TOKEN_TYPE.value % (list)) + def testDetokenizeRedactionInvalidType(self): + invalidData = {"records": [{"token": "valid", "redaction": 'demo'}]} + try: + self.client.detokenize(invalidData) + except SkyflowError as error: + self.assertTrue(error) + self.assertEqual(error.code, SkyflowErrorCodes.INVALID_INPUT.value) + self.assertEqual(error.message, SkyflowErrorMessages.INVALID_REDACTION_TYPE.value % str(type("demo"))) + def testResponseBodySuccess(self): response = {"records": [{"token": "abc", "value": "secret"}]} self.add_mock_response(response, 200) @@ -135,3 +146,39 @@ def testResponseNotJson(self): self.assertEqual(error.code, 200) self.assertEqual(error.message, expectedError.value % response.decode('utf-8')) + + def testRequestBodyNoRedactionKey(self): + expectedOutput = { + "detokenizationParameters": [{ + "token": self.testToken, + "redaction": "PLAIN_TEXT" + }] + } + requestBody = getDetokenizeRequestBody(self.tokenField) + self.assertEqual(requestBody, expectedOutput) + + def testRequestBodyWithValidRedaction(self): + expectedOutput = { + "detokenizationParameters": [{ + "token": self.testToken, + "redaction": "REDACTED" + }] + } + data = { + "token": self.testToken, + "redaction": RedactionType.REDACTED + } + requestBody = getDetokenizeRequestBody(data) + self.assertEqual(expectedOutput, requestBody) + + def testRequestBodyWithInValidRedaction(self): + data = { + "token": self.testToken, + "redaction": "123" + } + try: + getDetokenizeRequestBody(data) + except SkyflowError as error: + self.assertTrue(error) + self.assertEqual(error.code, SkyflowErrorCodes.INVALID_INPUT.value) + self.assertEqual(error.message, SkyflowErrorMessages.INVALID_REDACTION_TYPE.value % str(type(data["redaction"]))) \ No newline at end of file