diff --git a/.github/workflows/core-test.yml b/.github/workflows/core-test.yml index afc67499ee..ef523187c2 100644 --- a/.github/workflows/core-test.yml +++ b/.github/workflows/core-test.yml @@ -56,7 +56,7 @@ jobs: PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }} SMOKE_TEST_SUFFIX: ${{ github.run_id }} run: | - make test/smoke + make smoke/test - name: Cleanup Smoke Test if: always() @@ -67,7 +67,7 @@ jobs: PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }} SMOKE_TEST_SUFFIX: ${{ github.run_id }} run: | - make clean/smoke + make smoke/clean - name: Install current core for all integrations run: | diff --git a/.github/workflows/perf-test.yml b/.github/workflows/perf-test.yml index 3f80c8710f..dbc92b685d 100644 --- a/.github/workflows/perf-test.yml +++ b/.github/workflows/perf-test.yml @@ -51,6 +51,18 @@ on: - "20000" - "25000" - "35000" + ocean_log_level: + type: choice + default: 'INFO' + options: + - 'DEBUG' + - 'INFO' + description: Log level to use (defaults to INFO) + mock_port_api: + type: boolean + default: false + description: Mock the Port API instead of using the real one + jobs: test: name: 🌊 Ocean Performance Tests @@ -82,6 +94,8 @@ jobs: THIRD_PARTY_LATENCY_MS: ${{ inputs.third_party_latency_ms }} ENTITY_AMOUNT: ${{ inputs.entities_amount }} ENTITY_KB_SIZE: ${{ inputs.entity_kb_size }} + OCEAN_LOG_LEVEL: ${{ inputs.ocean_log_level }} + MOCK_PORT_API: ${{ inputs.mock_port_api && '1' || '0' }} run: | ./scripts/run-local-perf-test.sh @@ -92,8 +106,15 @@ jobs: PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }} PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }} SMOKE_TEST_SUFFIX: ${{ github.run_id }} + MOCK_PORT_API: ${{ inputs.mock_port_api && '1' || '0' }} run: | - make clean/smoke + if [[ "${MOCK_PORT_API}" = "1" ]]; then + make smoke/start-mock-api + make smoke/clean + make smoke/stop-mock-api + else + make smoke/clean + fi - name: Publish Performance Test Summary run: | diff --git a/Makefile b/Makefile index 9b9d01d705..7e89beca8b 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ define deactivate_virtualenv fi endef -.SILENT: install install/all test/all test/smoke clean/smoke lint lint/fix build run new test test/watch clean bump/integrations bump/single-integration execute/all +.SILENT: install install/all test/all smoke/test smoke/clean lint lint/fix build run new test test/watch clean bump/integrations bump/single-integration execute/all smoke/start-mock-api smoke/stop-mock-api # Install dependencies @@ -122,10 +122,10 @@ new: test: $(ACTIVATE) && pytest -m 'not smoke' -test/smoke: +smoke/test: $(ACTIVATE) && SMOKE_TEST_SUFFIX=$${SMOKE_TEST_SUFFIX:-default_value} pytest -m smoke -clean/smoke: +smoke/clean: $(ACTIVATE) && SMOKE_TEST_SUFFIX=$${SMOKE_TEST_SUFFIX:-default_value} python ./scripts/clean-smoke-test.py test/watch: @@ -156,3 +156,10 @@ bump/integrations: # make bump/single-integration INTEGRATION=aws bump/single-integration: ./scripts/bump-single-integration.sh -i $(INTEGRATION) + +# run a mock port api server for perf / smoke tests +smoke/start-mock-api: + $(ACTIVATE) && SMOKE_TEST_SUFFIX=$${SMOKE_TEST_SUFFIX:-default_value} python ./port_ocean/tests/helpers/fake_port_api.py & + +smoke/stop-mock-api: + ps aux | grep fake_port_api | egrep -v grep | awk '{print $$2};' | xargs kill -9 diff --git a/integrations/jira/.port/resources/port-app-config.yaml b/integrations/jira/.port/resources/port-app-config.yaml index f83db206be..97139b55b9 100644 --- a/integrations/jira/.port/resources/port-app-config.yaml +++ b/integrations/jira/.port/resources/port-app-config.yaml @@ -58,4 +58,3 @@ resources: subtasks: .fields.subtasks | map(.key) assignee: .fields.assignee.accountId reporter: .fields.reporter.accountId - diff --git a/port_ocean/tests/helpers/fake_port_api.py b/port_ocean/tests/helpers/fake_port_api.py new file mode 100644 index 0000000000..d3112914d7 --- /dev/null +++ b/port_ocean/tests/helpers/fake_port_api.py @@ -0,0 +1,191 @@ +import uvicorn +import os +from typing import Dict, Any +from fastapi import FastAPI, Request + +SMOKE_TEST_SUFFIX = os.environ.get("SMOKE_TEST_SUFFIX", "smoke") + +app = FastAPI() + +FAKE_DEPARTMENT_BLUEPRINT = { + "identifier": f"fake-department-{SMOKE_TEST_SUFFIX}", + "title": "Fake Department", + "icon": "Blueprint", + "schema": {"properties": {"name": {"type": "string"}, "id": {"type": "string"}}}, + "relations": {}, +} +FAKE_PERSON_BLUEPRINT = { + "identifier": f"fake-person-{SMOKE_TEST_SUFFIX}", + "title": "Fake Person", + "icon": "Blueprint", + "schema": { + "properties": { + "status": { + "type": "string", + "enum": ["WORKING", "NOPE"], + "enumColors": {"WORKING": "green", "NOPE": "red"}, + "title": "Status", + }, + "email": {"type": "string", "format": "email", "title": "Email"}, + "age": {"type": "number", "title": "Age"}, + "bio": {"type": "string", "title": "Bio"}, + } + }, + "relations": { + "department": { + "title": "Department", + "description": "Fake Department", + "target": f"fake-department-{SMOKE_TEST_SUFFIX}", + "required": False, + "many": False, + } + }, +} + + +@app.router.get("/v1/blueprints/{blueprint_id}") +@app.router.patch("/v1/blueprints/{blueprint_id}") +async def get_blueprint(blueprint_id: str) -> Dict[str, Any]: + return { + "blueprint": ( + FAKE_PERSON_BLUEPRINT + if blueprint_id.startswith("fake-person") + else FAKE_DEPARTMENT_BLUEPRINT + ) + } + + +@app.router.post("/v1/entities/search") +async def search_entities() -> Dict[str, Any]: + return {"ok": True, "entities": []} + + +@app.router.get("/v1/integration/{integration_id}") +@app.router.patch("/v1/integration/{integration_id}") +@app.router.patch("/v1/integration/{integration_id}/resync-state") +async def get_integration(integration_id: str) -> Dict[str, Any]: + return { + "integration": { + "identifer": integration_id, + "resyncState": { + "status": "completed", + "lastResyncEnd": "2024-11-20T12:01:54.225362+00:00", + "lastResyncStart": "2024-11-20T12:01:45.483844+00:00", + "nextResync": None, + "intervalInMinuets": None, + "updatedAt": "2024-11-20T12:01:54.355Z", + }, + "config": { + "deleteDependentEntities": True, + "createMissingRelatedEntities": True, + "enableMergeEntity": True, + "resources": [ + { + "kind": "fake-department", + "selector": {"query": "true"}, + "port": { + "entity": { + "mappings": { + "identifier": ".id", + "title": ".name", + "blueprint": f'"fake-department-{SMOKE_TEST_SUFFIX}"', + "properties": {"name": ".name", "id": ".id"}, + } + } + }, + }, + { + "kind": "fake-person", + "selector": {"query": "true"}, + "port": { + "entity": { + "mappings": { + "identifier": ".id", + "title": ".name", + "blueprint": f'"fake-person-{SMOKE_TEST_SUFFIX}"', + "properties": { + "name": ".name", + "email": ".email", + "status": ".status", + "age": ".age", + "department": ".department.name", + }, + "relations": {"department": ".department.id"}, + } + } + }, + }, + ], + }, + "installationType": "OnPrem", + "_orgId": "org_ZOMGMYUNIQUEID", + "_id": "integration_0dOOhnlJQDjMPnfe", + "identifier": f"smoke-test-integration-{SMOKE_TEST_SUFFIX}", + "integrationType": "smoke-test", + "createdBy": "APSQAYsYoIwPXqjn6XpwCAgnPakkNO67", + "updatedBy": "APSQAYsYoIwPXqjn6XpwCAgnPakkNO67", + "createdAt": "2024-11-20T12:01:42.651Z", + "updatedAt": "2024-11-20T12:01:54.355Z", + "clientId": "", + "logAttributes": { + "ingestId": "DOHSAIDHOMER", + "ingestUrl": "http://localhost:5555/logs/integration/DOHSAIDHOMER", + }, + }, + } + + +@app.router.post("/v1/blueprints/{blueprint_id}/entities") +async def upsert_entities(blueprint_id: str, request: Request) -> Dict[str, Any]: + json = await request.json() + + return { + "ok": True, + "entity": json, + } + + +@app.router.post("/v1/auth/access_token") +async def auth_token() -> Dict[str, Any]: + return { + "accessToken": "ZOMG", + "expiresIn": 1232131231, + "tokenType": "adadad", + } + + +@app.router.delete("/v1/blueprints/{blueprint_id}/all-entities") +async def delete_blueprint(blueprint_id: str, request: Request) -> Dict[str, Any]: + return {"migrationId": "ZOMG"} + + +@app.router.get("/v1/migrations/{migration_id}") +async def migration(migration_id: str, request: Request) -> Dict[str, Any]: + return { + "migration": { + "id": migration_id, + "status": "COMPLETE", + "actor": "Dwayne Scissors Johnson", + "sourceBlueprint": "leBlue", + "mapping": {}, + } + } + + +CATCH_ALL = "/{full_path:path}" + + +@app.router.get(CATCH_ALL) +@app.router.post(CATCH_ALL) +@app.router.patch(CATCH_ALL) +@app.router.delete(CATCH_ALL) +async def catch_all(full_path: str, request: Request) -> str: + return f"Hello there from fake Port API - {full_path}, thanks for accessing me with {request.method}" + + +def start() -> None: + uvicorn.run(app, host="0.0.0.0", port=5555) + + +if __name__ == "__main__": + start() diff --git a/scripts/run-local-perf-test.sh b/scripts/run-local-perf-test.sh index d2b0e08fc7..e5a0436f3d 100755 --- a/scripts/run-local-perf-test.sh +++ b/scripts/run-local-perf-test.sh @@ -23,6 +23,12 @@ export OCEAN__INTEGRATION__CONFIG__ENTITY_KB_SIZE=${ENTITY_KB_SIZE:--1} export OCEAN__INTEGRATION__CONFIG__THIRD_PARTY_BATCH_SIZE=${THIRD_PARTY_BATCH_SIZE:--1} export OCEAN__INTEGRATION__CONFIG__THIRD_PARTY_LATENCY_MS=${THIRD_PARTY_LATENCY_MS:--1} export OCEAN__INTEGRATION__CONFIG__SINGLE_DEPARTMENT_RUN=1 +export APPLICATION__LOG_LEVEL=${OCEAN_LOG_LEVEL:-'INFO'} + +if [[ "${MOCK_PORT_API:-0}" = "1" ]]; then + export PORT_BASE_URL=http://localhost:5555 + make smoke/start-mock-api +fi LOG_FILE_MD="${SCRIPT_BASE}/../perf-test-results-${SMOKE_TEST_SUFFIX}.log.md" @@ -58,6 +64,8 @@ RUN_LOG_FILE="./perf-sync.log" END_NS=$(date +%s%N) ELAPSED_MS=$(((END_NS - START_NS) / 1000000)) _log "Duration $((ELAPSED_MS / 1000)) seconds" + + UPSERTED=$(ruby -ne 'puts "#{$1}" if /Upserting (\d*) entities/' <"${RUN_LOG_FILE}" | xargs) if [[ -n "${UPSERTED}" ]]; then TOTAL_UPSERTED=0 @@ -75,4 +83,9 @@ if [[ -n "${DELETED}" ]]; then _log "Deleted: ${TOTAL_DELETED} entities" fi + +if [[ "${MOCK_PORT_API:-0}" = "1" ]]; then + make smoke/stop-mock-api +fi + _log "Perf test complete"