SK-1731: Updated workflows for internal release

.github/workflows/beta-release.yml

@@ -0,0 +1,19 @@
+# .github/workflows/beta-release.yml
+name: Public Beta Release
+
+on:
+  push:
+    tags: '*.*.*b*'
+    paths-ignore:
+      - "setup.py"
+      - "*.yml"
+      - "*.md"
+      - "skyflow/utils/_version.py"
+
+jobs:
+  build-and-deploy:
+    uses: ./shared-build-and-deploy.yml
+    with:
+      ref: main
+      pypi-token: ${{ secrets.PYPI_PUBLISH_TOKEN }}
+      is-internal: false

.github/workflows/ci.yml

@@ -16,46 +16,8 @@ jobs:
           checkAllCommitMessages: 'true'
           accessToken: ${{ secrets.PAT_ACTIONS }}
           error: 'One of your your commit messages is not matching the format with JIRA ID Ex: ( SDK-123 commit message )'
-  Test:
-    runs-on: ubuntu-latest
-    steps:   
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: '3.7'
-
-    - name: create-json
-      id: create-json
-      uses: jsdaniell/[email protected]
-      with:
-        name: "credentials.json"
-        json: ${{ secrets.CREDENTIALS_FILE_STR }}
-
-    - name: 'Setup .env'
-      run: |
-        touch .env
-        echo VAULT_ID=${{secrets.VAULT_ID}} >> .env
-        echo VAULT_URL=${{secrets.VAULT_URL}} >> .env
-        echo DETOKENIZE_TEST_TOKEN=${{secrets.DETOKENIZE_TEST_TOKEN}} >> .env
-        echo DETOKENIZE_TEST_VALUE=${{secrets.DETOKENIZE_TEST_VALUE}} >> .env
-        echo CREDENTIALS_FILE_PATH=./credentials.json >> .env
-        echo CVV_GEN_CONNECTION_URL=${{secrets.CVV_GEN_CONNECTION_URL}} >> .env
-        echo VISA_CONNECTION_BASIC_AUTH=${{secrets.VISA_CONNECTION_BASIC_AUTH}} >> .env
-        echo SKYFLOW_ID1=${{secrets.SKYFLOW_ID1}} >> .env
-        echo SKYFLOW_ID2=${{secrets.SKYFLOW_ID2}} >> .env
-        echo SKYFLOW_ID3=${{secrets.SKYFLOW_ID3}} >> .env
 
-    - name: 'Run Tests'
-      run: |
-        python -m pip install --upgrade pip
-        pip install requests pyjwt datetime aiohttp cryptography python-dotenv coverage
-        coverage run --source skyflow -m unittest discover
-    - name: coverage
-      run: coverage xml -o test-coverage.xml
-    - name: Codecov
-      uses: codecov/[email protected]
-      with:
-        token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
-        files: test-coverage.xml
-        name: codecov-skyflow-python
-        verbose: true
+  test:
+    uses: ./shared-tests.yml
+    with:
+      python-version: '3.7'

.github/workflows/internal-release.yml

@@ -0,0 +1,22 @@
+name: Internal Release
+
+on:
+  push:
+    tags-ignore:
+      - '*.*'
+    paths-ignore:
+      - "setup.py"
+      - "*.yml"
+      - "*.md"
+      - "skyflow/utils/_version.py"
+    branches:
+      - release/*
+
+jobs:
+  build-and-deploy:
+    uses: ./shared-build-and-deploy.yml
+    with:
+      ref: ${{ github.ref_name }}
+      artifactory-username: ${{ secrets.JFROG_USERNAME }}
+      artifactory-password: ${{ secrets.JFROG_PASSWORD }}
+      is-internal: true

.github/workflows/main.yml

@@ -6,46 +6,7 @@ on:
       - main
 
 jobs:
-  Test:
-    runs-on: ubuntu-latest
-    steps:   
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-      with:
-        python-version: '3.7'
-
-    - name: create-json
-      id: create-json
-      uses: jsdaniell/[email protected]
-      with:
-        name: "credentials.json"
-        json: ${{ secrets.CREDENTIALS_FILE_STR }}
-
-    - name: 'Setup .env'
-      run: |
-        touch .env
-        echo VAULT_ID=${{secrets.VAULT_ID}} >> .env
-        echo VAULT_URL=${{secrets.VAULT_URL}} >> .env
-        echo DETOKENIZE_TEST_TOKEN=${{secrets.DETOKENIZE_TEST_TOKEN}} >> .env
-        echo DETOKENIZE_TEST_VALUE=${{secrets.DETOKENIZE_TEST_VALUE}} >> .env
-        echo CREDENTIALS_FILE_PATH=./credentials.json >> .env
-        echo CVV_GEN_CONNECTION_URL=${{secrets.CVV_GEN_CONNECTION_URL}} >> .env
-        echo VISA_CONNECTION_BASIC_AUTH=${{secrets.VISA_CONNECTION_BASIC_AUTH}} >> .env
-        echo SKYFLOW_ID1=${{secrets.SKYFLOW_ID1}} >> .env
-        echo SKYFLOW_ID2=${{secrets.SKYFLOW_ID2}} >> .env
-        echo SKYFLOW_ID3=${{secrets.SKYFLOW_ID3}} >> .env
-
-    - name: 'Run Tests'
-      run: |
-        python -m pip install --upgrade pip
-        pip install requests pyjwt datetime aiohttp cryptography python-dotenv coverage
-        coverage run --source skyflow -m unittest discover
-    - name: coverage
-      run: coverage xml -o test-coverage.xml
-    - name: Codecov
-      uses: codecov/[email protected]
-      with:
-        token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
-        files: test-coverage.xml
-        name: codecov-skyflow-python
-        verbose: true
+  test:
+    uses: ./shared-tests.yml
+    with:
+      python-version: '3.7'

.github/workflows/release.yml

@@ -7,47 +7,12 @@ on:
       - "setup.py"
       - "*.yml"
       - "*.md"
-      - "skyflow/version.py"
+      - "skyflow/utils/_version.py"
 
 jobs:
   build-and-deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          token: ${{ secrets.PAT_ACTIONS }}
-          ref: main
-          fetch-depth: 0
-      - uses: actions/setup-python@v2
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-
-      - name: Get Previous tag
-        id: previoustag
-        uses: WyriHaximus/github-action-get-previous-tag@v1
-        with:
-          fallback: 1.0.0
-
-      - name: Bump Version
-        run: |
-          chmod +x ./ci-scripts/bump_version.sh
-          ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}"
-
-      - name: Commit changes
-        run: |
-          git config user.name ${{ github.actor }}
-          git config user.email ${{ github.actor }}@users.noreply.github.com
-          git add setup.py
-          git add skyflow/version.py
-          git commit -m "[AUTOMATED] Public Release - ${{ steps.previoustag.outputs.tag }}"
-          git push origin
-          
-      - name: Build and publish
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_TOKEN }}
-        run: |
-          python setup.py sdist bdist_wheel
-          twine upload dist/*
+    uses: ./shared-build-and-deploy.yml
+    with:
+      ref: main
+      pypi-token: ${{ secrets.PYPI_PUBLISH_TOKEN }}
+      is-internal: false

.github/workflows/shared-build-and-deploy.yml

@@ -0,0 +1,90 @@
+name: Shared Build and Deploy
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git reference to use (e.g., main or branch name)'
+        required: true
+        type: string
+      pypi-token:
+        description: 'PyPI token for publishing'
+        required: false
+        type: string
+      artifactory-username:
+        description: 'Artifactory username for internal release'
+        required: false
+        type: string
+      artifactory-password:
+        description: 'Artifactory password for internal release'
+        required: false
+        type: string
+      is-internal:
+        description: 'Flag for internal release'
+        required: true
+        type: boolean
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.PAT_ACTIONS }}
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+
+      - name: Get Previous tag
+        id: previoustag
+        uses: WyriHaximus/github-action-get-previous-tag@v1
+        with:
+          fallback: 1.0.0
+
+      - name: Bump Version
+        run: |
+          chmod +x ./ci-scripts/bump_version.sh
+          if ${{ inputs.is-internal }}; then
+            ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}" "$(git rev-parse --short "$GITHUB_SHA")"
+          else
+            ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}"
+          fi
+
+      - name: Commit changes
+        run: |
+          git config user.name ${{ github.actor }}
+          git config user.email ${{ github.actor }}@users.noreply.github.com
+          git add setup.py
+          git add skyflow/version.py
+          if ${{ inputs.is-internal }}; then
+            git commit -m "[AUTOMATED] Private Release ${{ steps.previoustag.outputs.tag }}-dev+$(git rev-parse --short $GITHUB_SHA)"
+            git push origin ${{ github.ref_name }} -f
+          else
+            git commit -m "[AUTOMATED] Public Release - ${{ steps.previoustag.outputs.tag }}"
+            git push origin
+
+      - name: Build the package
+        run: |
+          python setup.py sdist bdist_wheel
+
+      - name: Publish Package
+        if: ${{ !inputs.is-internal }}
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ inputs.pypi-token }}
+        run: |
+          twine upload dist/*
+
+      - name: Publish to JFrog Artifactory
+        if: ${{ inputs.is-internal }}
+        env:
+          TWINE_USERNAME: ${{ inputs.artifactory-username }}
+          TWINE_PASSWORD: ${{ inputs.artifactory-password }}
+        run: |
+          twine upload --repository-url https://prekarilabs.jfrog.io/artifactory/api/pypi/skyflow-python/ dist/*

.github/workflows/shared-tests.yml

@@ -0,0 +1,34 @@
+name: Shared Test Steps
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        description: 'Python version to use'
+        required: true
+        type: string
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: 'Run Tests'
+        run: |
+          pip install -r requirements.txt
+          python -m coverage run --source=skyflow --omit=skyflow/generated/*,skyflow/utils/validations/*,skyflow/vault/data/*,skyflow/vault/tokens/*,skyflow/vault/connection/*,skyflow/error/*,skyflow/utils/enums/*,skyflow/vault/controller/_audit.py,skyflow/vault/controller/_bin_look_up.py -m unittest discover
+
+      - name: coverage
+        run: coverage xml -o test-coverage.xml
+
+      - name: Codecov
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }}
+          files: test-coverage.xml
+          name: codecov-skyflow-python
+          verbose: true

ci-scripts/bump_version.sh

@@ -6,7 +6,7 @@ then
 	echo "Bumping package version to $1"
 
 	sed -E "s/current_version = .+/current_version = \'$SEMVER\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile
-	sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER\'/g" skyflow/version.py > tempfile && cat tempfile > skyflow/version.py && rm -f tempfile
+	sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER\'/g" skyflow/utils/_version.py > tempfile && cat tempfile > skyflow/utils/_version.py && rm -f tempfile
 
 	echo --------------------------
 	echo "Done, Package now at $1"
@@ -15,7 +15,7 @@ else
 	echo "Bumping package version to $1-dev.$2"
 
 	sed -E "s/current_version = .+/current_version = \'$SEMVER-dev.$2\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile
-	sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER-dev.$2\'/g" skyflow/version.py > tempfile && cat tempfile > skyflow/version.py && rm -f tempfile
+	sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER-dev.$2\'/g" skyflow/utils/_version.py > tempfile && cat tempfile > skyflow/utils/_version.py && rm -f tempfile
 
 	echo --------------------------
 	echo "Done, Package now at $1-dev.$2"

requirements.txt

@@ -0,0 +1,8 @@
+python_dateutil >= 2.5.3
+setuptools >= 21.0.0
+urllib3 >= 1.25.3, < 2.1.0
+pydantic >= 2
+typing-extensions >= 4.7.1
+DateTime~=5.5
+PyJWT~=2.9.0
+requests~=2.32.3

setup.py

@@ -19,12 +19,14 @@
     description='Skyflow SDK for the Python programming language',
     long_description=open('README.rst').read(),
     install_requires=[
-        'PyJWT',
-        'datetime',
-        'requests',
-        'aiohttp',
-        'asyncio',
-        'cryptography>=3.3.1'
+        'python_dateutil >= 2.5.3',
+        'setuptools >= 21.0.0',
+        'urllib3 >= 1.25.3, < 2.1.0',
+        'pydantic >= 2',
+        'typing-extensions >= 4.7.1',
+        'DateTime~=5.5',
+        'PyJWT~=2.9.0',
+        'requests~=2.32.3'
     ],
     python_requires=">=3.7"
 )

skyflow/generated/rest/api/records_api.py

@@ -1045,6 +1045,7 @@ def record_service_batch_operation(
 
         _response_types_map: Dict[str, Optional[str]] = {
             '200': "V1BatchOperationResponse",
+            '207': "V1BatchOperationResponse",
             '404': "object",
         }
         response_data = self.api_client.call_api(