Merge pull request #80 from skyflowapi/release/23.6.1

Release/23.6.1

.github/workflows/release.yml

@@ -7,6 +7,7 @@ on:
       - "setup.py"
       - "*.yml"
       - "*.md"
+      - "version.py"
 
 jobs:
   build-and-deploy:
@@ -39,6 +40,7 @@ jobs:
           git config user.name ${{ github.actor }}
           git config user.email ${{ github.actor }}@users.noreply.github.com
           git add setup.py
+          git add version.py
           git commit -m "[AUTOMATED] Public Release - ${{ steps.previoustag.outputs.tag }}"
           git push origin
           

ci-scripts/bump_version.sh

@@ -6,13 +6,16 @@ then
 	echo "Bumping package version to $1"
 
 	sed -E "s/current_version = .+/current_version = \'$SEMVER\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile
+	sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER\'/g" version.py > tempfile && cat tempfile > version.py && rm -f tempfile
+
 	echo --------------------------
 	echo "Done, Package now at $1"
 
 else
 	echo "Bumping package version to $1-dev.$2"
 
 	sed -E "s/current_version = .+/current_version = \'$SEMVER-dev.$2\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile
+	sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER-dev.$2\'/g" version.py > tempfile && cat tempfile > version.py && rm -f tempfile
 
 	echo --------------------------
 	echo "Done, Package now at $1-dev.$2"

skyflow/_utils.py

@@ -4,6 +4,9 @@
 import urllib.parse
 import logging
 from enum import Enum
+import platform
+import sys
+from version import SDK_VERSION
 
 skyflowLog = logging.getLogger('skyflow')
 skyflowLog.setLevel(logging.ERROR)
@@ -127,3 +130,31 @@ def render_key(parents):
         outStr += s % str(x)
         depth += 1
     return outStr
+
+def getMetrics():
+    ''' fetch metrics
+    '''
+    sdk_name_version = "skyflow-python@" + SDK_VERSION
+
+    try:
+        sdk_client_device_model = platform.node()
+    except Exception:
+        sdk_client_device_model = ""
+
+    try:
+        sdk_client_os_details = sys.platform
+    except Exception:
+        sdk_client_os_details = ""
+
+    try:
+        sdk_runtime_details = sys.version
+    except Exception:
+        sdk_runtime_details = ""
+
+    details_dic = {
+        'sdk_name_version': sdk_name_version,
+        'sdk_client_device_model': sdk_client_device_model,
+        'sdk_client_os_details': sdk_client_os_details,
+        'sdk_runtime_details': "Python " + sdk_runtime_details,
+    }
+    return details_dic

skyflow/service_account/_token.py

@@ -7,7 +7,7 @@
 import requests
 from warnings import warn
 from collections import namedtuple
-from skyflow._utils import log_info, InterfaceName, InfoMessages
+from skyflow._utils import log_info, InterfaceName, InfoMessages, getMetrics
 
 
 from skyflow.errors._skyflow_errors import *
@@ -126,7 +126,8 @@ def getSignedJWT(clientID, keyID, tokenURI, privateKey):
 
 def sendRequestWithToken(url, token):
     headers = {
-        "content-type": "application/json"
+        "content-type": "application/json",
+        "sky-metadata": json.dumps(getMetrics())
     }
     payload = {
         "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",

skyflow/vault/_client.py

@@ -1,6 +1,7 @@
 '''
 	Copyright (c) 2022 Skyflow, Inc.
 '''
+import json
 import types
 import requests
 from ._insert import getInsertRequestBody, processResponse, convertResponse
@@ -13,7 +14,7 @@
 from ._get import sendGetRequests
 import asyncio
 from skyflow.errors._skyflow_errors import SkyflowError, SkyflowErrorCodes, SkyflowErrorMessages
-from skyflow._utils import log_info, InfoMessages, InterfaceName
+from skyflow._utils import log_info, InfoMessages, InterfaceName, getMetrics
 from ._token import tokenProviderWrapper
 
 
@@ -52,7 +53,8 @@ def insert(self, records: dict, options: InsertOptions = InsertOptions()):
         self.storedToken = tokenProviderWrapper(
             self.storedToken, self.tokenProvider, interface)
         headers = {
-            "Authorization": "Bearer " + self.storedToken
+            "Authorization": "Bearer " + self.storedToken,
+            "sky-metadata": json.dumps(getMetrics())
         }
 
         response = requests.post(requestURL, data=jsonBody, headers=headers)
@@ -131,6 +133,8 @@ def invoke_connection(self, config: ConnectionConfig):
         if not 'X-Skyflow-Authorization'.lower() in request.headers:
             request.headers['x-skyflow-authorization'] = self.storedToken
 
+        request.headers['sky-metadata'] = json.dumps(getMetrics())
+
         response = session.send(request)
         session.close()
         return processResponse(response, interface=interface)

skyflow/vault/_detokenize.py

@@ -5,7 +5,8 @@
 import asyncio
 from aiohttp import ClientSession, request
 import json
-from skyflow._utils import InterfaceName
+from ._config import RedactionType
+from skyflow._utils import InterfaceName, getMetrics
 
 interface = InterfaceName.DETOKENIZE.value
 
@@ -15,14 +16,27 @@ def getDetokenizeRequestBody(data):
         token = data["token"]
     except KeyError:
         raise SkyflowError(SkyflowErrorCodes.INVALID_INPUT,
-                           SkyflowErrorMessages.TOKEN_KEY_ERROR, interface=interface)
+                            SkyflowErrorMessages.TOKEN_KEY_ERROR, interface=interface)
     if not isinstance(token, str):
         tokenType = str(type(token))
         raise SkyflowError(SkyflowErrorCodes.INVALID_INPUT, SkyflowErrorMessages.INVALID_TOKEN_TYPE.value % (
             tokenType), interface=interface)
+
+    if "redaction" in data:
+        if not isinstance(data["redaction"], RedactionType):
+            redactionType = str(type(data["redaction"]))
+            raise SkyflowError(SkyflowErrorCodes.INVALID_INPUT, SkyflowErrorMessages.INVALID_REDACTION_TYPE.value % (
+            redactionType), interface=interface)
+        else:
+            redactionType =  data["redaction"]
+    else:
+        redactionType = RedactionType.PLAIN_TEXT
+
     requestBody = {"detokenizationParameters": []}
     requestBody["detokenizationParameters"].append({
-        "token": token})
+        "token": token,
+        "redaction": redactionType.value
+        })
     return requestBody
 
 
@@ -39,7 +53,6 @@ async def sendDetokenizeRequests(data, url, token):
         recordsType = str(type(records))
         raise SkyflowError(SkyflowErrorCodes.INVALID_INPUT, SkyflowErrorMessages.INVALID_RECORDS_TYPE.value % (
             recordsType), interface=interface)
-
     validatedRecords = []
     for record in records:
         requestBody = getDetokenizeRequestBody(record)
@@ -48,7 +61,9 @@ async def sendDetokenizeRequests(data, url, token):
     async with ClientSession() as session:
         for record in validatedRecords:
             headers = {
-                "Authorization": "Bearer " + token
+                "Authorization": "Bearer " + token,
+                "sky-metadata": json.dumps(getMetrics())
+
             }
             task = asyncio.ensure_future(post(url, record, headers, session))
             tasks.append(task)

skyflow/vault/_get.py

@@ -1,11 +1,12 @@
 '''
 	Copyright (c) 2022 Skyflow, Inc.
 '''
+import json
 from skyflow.errors._skyflow_errors import SkyflowError, SkyflowErrorCodes, SkyflowErrorMessages
 import asyncio
 from aiohttp import ClientSession
 from ._config import RedactionType
-from skyflow._utils import InterfaceName
+from skyflow._utils import InterfaceName, getMetrics
 from ._get_by_id import get
 
 interface = InterfaceName.GET.value
@@ -83,7 +84,8 @@ async def sendGetRequests(data, url, token):
     async with ClientSession() as session:
         for record in validatedRecords:
             headers = {
-                "Authorization": "Bearer " + token
+                "Authorization": "Bearer " + token,
+                "sky-metadata": json.dumps(getMetrics())
             }
             params = {"redaction": redaction}
             if ids is not None:

skyflow/vault/_get_by_id.py

@@ -6,7 +6,7 @@
 from aiohttp import ClientSession
 import json
 from ._config import RedactionType
-from skyflow._utils import InterfaceName
+from skyflow._utils import InterfaceName, getMetrics
 
 interface = InterfaceName.GET_BY_ID.value
 
@@ -66,7 +66,8 @@ async def sendGetByIdRequests(data, url, token):
     async with ClientSession() as session:
         for record in validatedRecords:
             headers = {
-                "Authorization": "Bearer " + token
+                "Authorization": "Bearer " + token,
+                "sky-metadata": json.dumps(getMetrics())
             }
             params = {"skyflow_ids": record[0], "redaction": record[2]}
             task = asyncio.ensure_future(

skyflow/vault/_update.py

@@ -6,7 +6,7 @@
 import asyncio
 from skyflow.errors._skyflow_errors import SkyflowError, SkyflowErrorCodes, SkyflowErrorMessages
 from ._insert import getTableAndFields
-from skyflow._utils import InterfaceName
+from skyflow._utils import InterfaceName, getMetrics
 from aiohttp import ClientSession
 from ._config import UpdateOptions
 
@@ -40,7 +40,8 @@ async def sendUpdateRequests(data,options: UpdateOptions,url,token):
             }
             reqBody = json.dumps(reqBody)
             headers = {
-                "Authorization": "Bearer " + token
+                "Authorization": "Bearer " + token,
+                "sky-metadata": json.dumps(getMetrics())
             }
             task = asyncio.ensure_future(put(recordUrl, reqBody, headers, session))
             tasks.append(task)

tests/vault/test_detokenize.py

@@ -7,6 +7,7 @@
 from skyflow.errors._skyflow_errors import SkyflowError, SkyflowErrorCodes, SkyflowErrorMessages
 from skyflow.vault._client import Client, Configuration
 from skyflow.service_account import generate_bearer_token
+from skyflow.vault._config import RedactionType
 from dotenv import dotenv_values
 import warnings
 
@@ -55,7 +56,8 @@ def testGetDetokenizeRequestBodyWithValidBody(self):
         body = getDetokenizeRequestBody(self.tokenField)
         expectedOutput = {
             "detokenizationParameters": [{
-                "token": self.testToken
+                "token": self.testToken,
+                "redaction": "PLAIN_TEXT"
             }]
         }
 
@@ -101,6 +103,15 @@ def testDetokenizeTokenInvalidType(self):
             self.assertEqual(
                 e.message, SkyflowErrorMessages.INVALID_TOKEN_TYPE.value % (list))
 
+    def testDetokenizeRedactionInvalidType(self):
+        invalidData = {"records": [{"token": "valid", "redaction": 'demo'}]}
+        try:
+            self.client.detokenize(invalidData)
+        except SkyflowError as error:
+            self.assertTrue(error)
+            self.assertEqual(error.code, SkyflowErrorCodes.INVALID_INPUT.value)
+            self.assertEqual(error.message, SkyflowErrorMessages.INVALID_REDACTION_TYPE.value % str(type("demo")))
+
     def testResponseBodySuccess(self):
         response = {"records": [{"token": "abc", "value": "secret"}]}
         self.add_mock_response(response, 200)
@@ -135,3 +146,39 @@ def testResponseNotJson(self):
             self.assertEqual(error.code, 200)
             self.assertEqual(error.message, expectedError.value %
                              response.decode('utf-8'))
+
+    def testRequestBodyNoRedactionKey(self):
+        expectedOutput = {
+            "detokenizationParameters": [{
+                "token": self.testToken,
+                "redaction": "PLAIN_TEXT"
+            }]
+        }    
+        requestBody =  getDetokenizeRequestBody(self.tokenField)
+        self.assertEqual(requestBody, expectedOutput)
+
+    def testRequestBodyWithValidRedaction(self):
+        expectedOutput = {
+            "detokenizationParameters": [{
+                "token": self.testToken,
+                "redaction": "REDACTED"
+            }]
+        } 
+        data = {
+            "token": self.testToken,
+            "redaction": RedactionType.REDACTED
+        }
+        requestBody = getDetokenizeRequestBody(data)
+        self.assertEqual(expectedOutput, requestBody)
+
+    def testRequestBodyWithInValidRedaction(self):
+        data = {
+            "token": self.testToken,
+            "redaction": "123"
+        } 
+        try: 
+            getDetokenizeRequestBody(data)
+        except SkyflowError as error:
+            self.assertTrue(error)
+            self.assertEqual(error.code, SkyflowErrorCodes.INVALID_INPUT.value)
+            self.assertEqual(error.message, SkyflowErrorMessages.INVALID_REDACTION_TYPE.value % str(type(data["redaction"])))