diff --git a/.github/workflows/beta-release.yml b/.github/workflows/beta-release.yml new file mode 100644 index 0000000..f6c8791 --- /dev/null +++ b/.github/workflows/beta-release.yml @@ -0,0 +1,19 @@ +# .github/workflows/beta-release.yml +name: Public Beta Release + +on: + push: + tags: '*.*.*b*' + paths-ignore: + - "setup.py" + - "*.yml" + - "*.md" + - "skyflow/utils/_version.py" + +jobs: + build-and-deploy: + uses: ./shared-build-and-deploy.yml + with: + ref: main + pypi-token: ${{ secrets.PYPI_PUBLISH_TOKEN }} + is-internal: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47a7281..68d83f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,46 +16,8 @@ jobs: checkAllCommitMessages: 'true' accessToken: ${{ secrets.PAT_ACTIONS }} error: 'One of your your commit messages is not matching the format with JIRA ID Ex: ( SDK-123 commit message )' - Test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: create-json - id: create-json - uses: jsdaniell/create-json@1.1.2 - with: - name: "credentials.json" - json: ${{ secrets.CREDENTIALS_FILE_STR }} - - - name: 'Setup .env' - run: | - touch .env - echo VAULT_ID=${{secrets.VAULT_ID}} >> .env - echo VAULT_URL=${{secrets.VAULT_URL}} >> .env - echo DETOKENIZE_TEST_TOKEN=${{secrets.DETOKENIZE_TEST_TOKEN}} >> .env - echo DETOKENIZE_TEST_VALUE=${{secrets.DETOKENIZE_TEST_VALUE}} >> .env - echo CREDENTIALS_FILE_PATH=./credentials.json >> .env - echo CVV_GEN_CONNECTION_URL=${{secrets.CVV_GEN_CONNECTION_URL}} >> .env - echo VISA_CONNECTION_BASIC_AUTH=${{secrets.VISA_CONNECTION_BASIC_AUTH}} >> .env - echo SKYFLOW_ID1=${{secrets.SKYFLOW_ID1}} >> .env - echo SKYFLOW_ID2=${{secrets.SKYFLOW_ID2}} >> .env - echo SKYFLOW_ID3=${{secrets.SKYFLOW_ID3}} >> .env - - name: 'Run Tests' - run: | - python -m pip install --upgrade pip - pip install requests pyjwt datetime aiohttp cryptography python-dotenv coverage - coverage run --source skyflow -m unittest discover - - name: coverage - run: coverage xml -o test-coverage.xml - - name: Codecov - uses: codecov/codecov-action@v2.1.0 - with: - token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }} - files: test-coverage.xml - name: codecov-skyflow-python - verbose: true + test: + uses: ./shared-tests.yml + with: + python-version: '3.7' diff --git a/.github/workflows/internal-release.yml b/.github/workflows/internal-release.yml new file mode 100644 index 0000000..29ba2e0 --- /dev/null +++ b/.github/workflows/internal-release.yml @@ -0,0 +1,22 @@ +name: Internal Release + +on: + push: + tags-ignore: + - '*.*' + paths-ignore: + - "setup.py" + - "*.yml" + - "*.md" + - "skyflow/utils/_version.py" + branches: + - release/* + +jobs: + build-and-deploy: + uses: ./shared-build-and-deploy.yml + with: + ref: ${{ github.ref_name }} + artifactory-username: ${{ secrets.JFROG_USERNAME }} + artifactory-password: ${{ secrets.JFROG_PASSWORD }} + is-internal: true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68d9b0a..08f22c6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,46 +6,7 @@ on: - main jobs: - Test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: create-json - id: create-json - uses: jsdaniell/create-json@1.1.2 - with: - name: "credentials.json" - json: ${{ secrets.CREDENTIALS_FILE_STR }} - - - name: 'Setup .env' - run: | - touch .env - echo VAULT_ID=${{secrets.VAULT_ID}} >> .env - echo VAULT_URL=${{secrets.VAULT_URL}} >> .env - echo DETOKENIZE_TEST_TOKEN=${{secrets.DETOKENIZE_TEST_TOKEN}} >> .env - echo DETOKENIZE_TEST_VALUE=${{secrets.DETOKENIZE_TEST_VALUE}} >> .env - echo CREDENTIALS_FILE_PATH=./credentials.json >> .env - echo CVV_GEN_CONNECTION_URL=${{secrets.CVV_GEN_CONNECTION_URL}} >> .env - echo VISA_CONNECTION_BASIC_AUTH=${{secrets.VISA_CONNECTION_BASIC_AUTH}} >> .env - echo SKYFLOW_ID1=${{secrets.SKYFLOW_ID1}} >> .env - echo SKYFLOW_ID2=${{secrets.SKYFLOW_ID2}} >> .env - echo SKYFLOW_ID3=${{secrets.SKYFLOW_ID3}} >> .env - - - name: 'Run Tests' - run: | - python -m pip install --upgrade pip - pip install requests pyjwt datetime aiohttp cryptography python-dotenv coverage - coverage run --source skyflow -m unittest discover - - name: coverage - run: coverage xml -o test-coverage.xml - - name: Codecov - uses: codecov/codecov-action@v2.1.0 - with: - token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }} - files: test-coverage.xml - name: codecov-skyflow-python - verbose: true \ No newline at end of file + test: + uses: ./shared-tests.yml + with: + python-version: '3.7' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5fcd5db..ca24a23 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,47 +7,12 @@ on: - "setup.py" - "*.yml" - "*.md" - - "skyflow/version.py" + - "skyflow/utils/_version.py" jobs: build-and-deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - token: ${{ secrets.PAT_ACTIONS }} - ref: main - fetch-depth: 0 - - uses: actions/setup-python@v2 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - - name: Get Previous tag - id: previoustag - uses: WyriHaximus/github-action-get-previous-tag@v1 - with: - fallback: 1.0.0 - - - name: Bump Version - run: | - chmod +x ./ci-scripts/bump_version.sh - ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}" - - - name: Commit changes - run: | - git config user.name ${{ github.actor }} - git config user.email ${{ github.actor }}@users.noreply.github.com - git add setup.py - git add skyflow/version.py - git commit -m "[AUTOMATED] Public Release - ${{ steps.previoustag.outputs.tag }}" - git push origin - - - name: Build and publish - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PUBLISH_TOKEN }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* + uses: ./shared-build-and-deploy.yml + with: + ref: main + pypi-token: ${{ secrets.PYPI_PUBLISH_TOKEN }} + is-internal: false diff --git a/.github/workflows/shared-build-and-deploy.yml b/.github/workflows/shared-build-and-deploy.yml new file mode 100644 index 0000000..20d67e4 --- /dev/null +++ b/.github/workflows/shared-build-and-deploy.yml @@ -0,0 +1,90 @@ +name: Shared Build and Deploy + +on: + workflow_call: + inputs: + ref: + description: 'Git reference to use (e.g., main or branch name)' + required: true + type: string + pypi-token: + description: 'PyPI token for publishing' + required: false + type: string + artifactory-username: + description: 'Artifactory username for internal release' + required: false + type: string + artifactory-password: + description: 'Artifactory password for internal release' + required: false + type: string + is-internal: + description: 'Flag for internal release' + required: true + type: boolean + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + token: ${{ secrets.PAT_ACTIONS }} + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - uses: actions/setup-python@v2 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + + - name: Get Previous tag + id: previoustag + uses: WyriHaximus/github-action-get-previous-tag@v1 + with: + fallback: 1.0.0 + + - name: Bump Version + run: | + chmod +x ./ci-scripts/bump_version.sh + if ${{ inputs.is-internal }}; then + ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}" "$(git rev-parse --short "$GITHUB_SHA")" + else + ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}" + fi + + - name: Commit changes + run: | + git config user.name ${{ github.actor }} + git config user.email ${{ github.actor }}@users.noreply.github.com + git add setup.py + git add skyflow/version.py + if ${{ inputs.is-internal }}; then + git commit -m "[AUTOMATED] Private Release ${{ steps.previoustag.outputs.tag }}-dev+$(git rev-parse --short $GITHUB_SHA)" + git push origin ${{ github.ref_name }} -f + else + git commit -m "[AUTOMATED] Public Release - ${{ steps.previoustag.outputs.tag }}" + git push origin + + - name: Build the package + run: | + python setup.py sdist bdist_wheel + + - name: Publish Package + if: ${{ !inputs.is-internal }} + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ inputs.pypi-token }} + run: | + twine upload dist/* + + - name: Publish to JFrog Artifactory + if: ${{ inputs.is-internal }} + env: + TWINE_USERNAME: ${{ inputs.artifactory-username }} + TWINE_PASSWORD: ${{ inputs.artifactory-password }} + run: | + twine upload --repository-url https://prekarilabs.jfrog.io/artifactory/api/pypi/skyflow-python/ dist/* diff --git a/.github/workflows/shared-tests.yml b/.github/workflows/shared-tests.yml new file mode 100644 index 0000000..7e7730d --- /dev/null +++ b/.github/workflows/shared-tests.yml @@ -0,0 +1,34 @@ +name: Shared Test Steps + +on: + workflow_call: + inputs: + python-version: + description: 'Python version to use' + required: true + type: string + +jobs: + run-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.python-version }} + + - name: 'Run Tests' + run: | + pip install -r requirements.txt + python -m coverage run --source=skyflow --omit=skyflow/generated/*,skyflow/utils/validations/*,skyflow/vault/data/*,skyflow/vault/tokens/*,skyflow/vault/connection/*,skyflow/error/*,skyflow/utils/enums/*,skyflow/vault/controller/_audit.py,skyflow/vault/controller/_bin_look_up.py -m unittest discover + + - name: coverage + run: coverage xml -o test-coverage.xml + + - name: Codecov + uses: codecov/codecov-action@v2.1.0 + with: + token: ${{ secrets.CODECOV_REPO_UPLOAD_TOKEN }} + files: test-coverage.xml + name: codecov-skyflow-python + verbose: true diff --git a/ci-scripts/bump_version.sh b/ci-scripts/bump_version.sh index b0a57a9..fc8975c 100755 --- a/ci-scripts/bump_version.sh +++ b/ci-scripts/bump_version.sh @@ -6,7 +6,7 @@ then echo "Bumping package version to $1" sed -E "s/current_version = .+/current_version = \'$SEMVER\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile - sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER\'/g" skyflow/version.py > tempfile && cat tempfile > skyflow/version.py && rm -f tempfile + sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER\'/g" skyflow/utils/_version.py > tempfile && cat tempfile > skyflow/utils/_version.py && rm -f tempfile echo -------------------------- echo "Done, Package now at $1" @@ -15,7 +15,7 @@ else echo "Bumping package version to $1-dev.$2" sed -E "s/current_version = .+/current_version = \'$SEMVER-dev.$2\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile - sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER-dev.$2\'/g" skyflow/version.py > tempfile && cat tempfile > skyflow/version.py && rm -f tempfile + sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER-dev.$2\'/g" skyflow/utils/_version.py > tempfile && cat tempfile > skyflow/utils/_version.py && rm -f tempfile echo -------------------------- echo "Done, Package now at $1-dev.$2" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0b443bd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,8 @@ +python_dateutil >= 2.5.3 +setuptools >= 21.0.0 +urllib3 >= 1.25.3, < 2.1.0 +pydantic >= 2 +typing-extensions >= 4.7.1 +DateTime~=5.5 +PyJWT~=2.9.0 +requests~=2.32.3 \ No newline at end of file diff --git a/setup.py b/setup.py index c775672..7f0e9e1 100644 --- a/setup.py +++ b/setup.py @@ -19,12 +19,14 @@ description='Skyflow SDK for the Python programming language', long_description=open('README.rst').read(), install_requires=[ - 'PyJWT', - 'datetime', - 'requests', - 'aiohttp', - 'asyncio', - 'cryptography>=3.3.1' + 'python_dateutil >= 2.5.3', + 'setuptools >= 21.0.0', + 'urllib3 >= 1.25.3, < 2.1.0', + 'pydantic >= 2', + 'typing-extensions >= 4.7.1', + 'DateTime~=5.5', + 'PyJWT~=2.9.0', + 'requests~=2.32.3' ], python_requires=">=3.7" ) diff --git a/skyflow/generated/rest/api/records_api.py b/skyflow/generated/rest/api/records_api.py index 80325db..ae9a2c2 100644 --- a/skyflow/generated/rest/api/records_api.py +++ b/skyflow/generated/rest/api/records_api.py @@ -1045,6 +1045,7 @@ def record_service_batch_operation( _response_types_map: Dict[str, Optional[str]] = { '200': "V1BatchOperationResponse", + '207': "V1BatchOperationResponse", '404': "object", } response_data = self.api_client.call_api(