resolve merge conflicts

Signed-off-by: Arun Lakshmanan <[email protected]>

.flake8

@@ -8,7 +8,7 @@
 # W504 - Line break occurred after a binary operator
 ignore = E265,E402,E999,W293,W504
 max-line-length = 120
-exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules,.venv,*/cdk.out/*
+exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules,.venv,*/cdk.out/*,*/build/*
 
 # F401 - Unused imports -- this is the only way to have a file-wide rule exception
 per-file-ignores =

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @AndreKurait @chelma @gregschohn @lewijacn @mikaylathompson @okhasawn @peternied @sumobrian
+*   @AndreKurait @chelma @gregschohn @lewijacn @mikaylathompson @peternied @sumobrian @jugal-chauhan

.github/ISSUE_TEMPLATE/BUG_TEMPLATE.md

@@ -0,0 +1,21 @@
+---
+name: 🐛 Bug Report
+about: Create a report to help us improve
+title: '[BUG]'
+labels: 'bug, untriaged'
+assignees: ''
+---
+### What is the bug?
+_A clear and concise description of the bug._
+
+### What are your migration environments?
+_What versions of Elasticsearch / Opensearch, what version of the Migration Assistant._
+
+### How can one reproduce the bug?
+_Steps to reproduce the behavior._
+
+### What is the expected behavior?
+_A clear and concise description of what you expected to happen._
+
+### Do you have any additional context?
+_Add any other context about the problem, such as log files our console output._

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,22 +1,15 @@
 ### Description
-[Describe what this change achieves]
-* Category (Enhancement, New feature, Bug fix, Test fix, Refactoring, Maintenance, Documentation)
-* Why these changes are required?
-* What is the old behavior before changes and new behavior after changes?
+<!-- Describe what this change achieves -->
 
 ### Issues Resolved
-[List any issues this PR will resolve]
-
-Is this a backport? If so, please add backport PR # and/or commits #
+<!-- List any GitHub or Jira issues this PR will resolve -->
 
 ### Testing
-[Please provide details of testing done: unit testing, integration testing and manual testing]
+<!-- Please provide details of testing done: unit testing, integration testing and manual testing -->
 
 ### Check List
 - [ ] New functionality includes testing
-  - [ ] All tests pass, including unit test, integration test and doctest
-- [ ] New functionality has been documented
-- [ ] Commits are signed per the DCO using --signoff
+- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose), if applicable.
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
 For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).

.github/workflows/CI.yml

@@ -6,11 +6,32 @@ on:
 
 env:
   python-version: '3.11'
-  java-version: '11'
+  java-version: '17'
   gradle-version: '8.0.2'
   node-version: '18.x'
 
 jobs:
+  workflow-info:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create JSON Artifact
+        run: |
+          echo '{' > workflow-info.json
+          echo '  "pr_number": "${{ github.event.pull_request.number }}",' >> workflow-info.json
+          echo '  "commit": "${{ github.event.pull_request.head.sha || github.event.after || github.sha }}",' >> workflow-info.json
+          echo '  "branch": "${{ github.ref }}",' >> workflow-info.json
+          echo '  "commit_parent": "${{ github.event.pull_request.base.sha || github.event.before || github.base.sha }}",' >> workflow-info.json
+          echo '  "build_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",' >> workflow-info.json
+          echo '  "build": "${{ github.run_id }}"' >> workflow-info.json
+          echo '}' >> workflow-info.json
+          echo "workflow-info.json created"
+      - name: Upload JSON Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: workflow-info
+          path: workflow-info.json
+
+
   style-check:
     runs-on: ubuntu-latest
     steps:
@@ -19,11 +40,13 @@ jobs:
         with:
           java-version: ${{ env.java-version }}
           distribution: "corretto"
-      - uses: gradle/actions/setup-gradle@v3
+      - uses: gradle/actions/setup-gradle@v4
         with:
           gradle-version: ${{ env.gradle-version }}
       - name: Run Gradle Build
         run: ./gradlew spotlessCheck
+        env:
+          OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
 
   python-lint:
     runs-on: ubuntu-latest
@@ -40,37 +63,20 @@ jobs:
        run: |
         flake8 $(git ls-files '*.py')
 
-  python-tests:
+  console-python-tests:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         py-project:
-          - ./FetchMigration/python
-          - ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link
+          - lib/console_link
+          - cluster_tools
+          - console_api
+    env:
+      WORKING_DIR: ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/${{ matrix.py-project }}
     defaults:
       run:
-        working-directory: ${{ matrix.py-project }}
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-         python-version: ${{ env.python-version }}
-      - run: |
-          python3 -m pip install --upgrade pipenv
-          pipenv install --deploy --dev
-          pipenv run coverage run -m pytest
-          pipenv run coverage xml
-      - uses: codecov/codecov-action@v4
-        with:
-          files: ./coverage.xml
-          flags: python-test
-
-  console-api-tests:
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api
+        working-directory: ${{ env.WORKING_DIR }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -79,11 +85,19 @@ jobs:
       - run: |
           python3 -m pip install --upgrade pipenv
           pipenv install --deploy --dev
-          pipenv run coverage run --source='.' manage.py test console_api
+          pipenv run test
           pipenv run coverage xml
-      - uses: codecov/codecov-action@v4
+      - name: Get Sanitized Name
+        env:
+          PY_PROJECT: ${{ matrix.py-project }}
+        run: echo "SANITIZED_PY_PROJECT=${PY_PROJECT/\//-}" >> $GITHUB_ENV
+      - name: Upload Coverage Reports
+        uses: actions/upload-artifact@v4
         with:
-          files: ./coverage.xml
+          if-no-files-found: error
+          name: coverage-reports-python-tests-${{ env.SANITIZED_PY_PROJECT }}
+          path: ${{ env.WORKING_DIR }}/coverage.xml
+
 
   gradle-tests:
     runs-on: ubuntu-latest
@@ -93,33 +107,52 @@ jobs:
         with:
           java-version: ${{ env.java-version }}
           distribution: 'corretto'
-      - uses: gradle/actions/setup-gradle@v3
+      - uses: gradle/actions/setup-gradle@v4
         with:
           gradle-version: ${{ env.gradle-version }}
           gradle-home-cache-cleanup: true
       - name: Run Gradle Build
-        run: ./gradlew build -x test -x TrafficCapture:dockerSolution:build -x spotlessCheck --scan --stacktrace
+        run: ./gradlew build :TrafficCapture:dockerSolution:buildDockerImage_elasticsearch_client_test_console -x test -x spotlessCheck --stacktrace
         env:
           OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
 
       - name: Run Tests with Coverage
-        run: ./gradlew test -x TrafficCapture:dockerSolution:build -x spotlessCheck jacocoTestReport --scan --stacktrace
+        run: ./gradlew mergeJacocoReports -x spotlessCheck --stacktrace
         env:
           OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
 
+      - name: Detect Memory Dumps
+        if: failure()
+        run: |
+          if find . -type f -name "*.hprof" | grep -q '.'; then
+            echo "::group::Memory Dumps Detected"
+            echo "::warning::Memory dumps were found and uploaded as artifacts. Review these files to diagnose OOM issues."
+            echo "To download and inspect these files, navigate to 'Actions' -> 'Artifacts'."
+            echo "::endgroup::"
+          fi
+
+      - name: Upload memory dump
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          if-no-files-found: ignore
+          name: memory-dumps
+          path: ./**/*.hprof
+
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: test-reports
+          name: test-reports-gradle-tests
           path: |
             **/build/reports/tests/
-
-      - name: Upload to Codecov
-        uses: codecov/codecov-action@v4
+            **/reports/jacoco/mergedReport/
+      - name: Upload Coverage Reports
+        uses: actions/upload-artifact@v4
         with:
-          files: "**/jacocoTestReport.xml"
-          flags: gradle-test
-          fail_ci_if_error: false
+          if-no-files-found: error
+          name: coverage-reports-gradle-tests
+          path: ./**/jacocoMergedReport.xml
+
 
   python-e2e-tests:
     runs-on: ubuntu-latest
@@ -129,7 +162,7 @@ jobs:
         with:
           java-version: ${{ env.java-version }}
           distribution: 'corretto'
-      - uses: gradle/actions/setup-gradle@v3
+      - uses: gradle/actions/setup-gradle@v4
         with:
           gradle-version: ${{ env.gradle-version }}
       - uses: actions/setup-python@v5
@@ -146,33 +179,58 @@ jobs:
         uses: ScribeMD/[email protected]
         with:
           key: docker-${{ runner.os }}-${{ steps.generate_cache_key.outputs.key }}
+          read-only: true
       - name: Start Docker Solution
-        run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --scan --info --stacktrace
+        run: ./gradlew -p TrafficCapture dockerSolution:ComposeUp -x test -x spotlessCheck --info --stacktrace
         env:
           OS_MIGRATIONS_GRADLE_SCAN_TOS_AGREE_AND_ENABLED: ''
       - name: Run E2E test script
-        working-directory: TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/integ_test
         run: |
           docker exec $(docker ps --filter "name=migration-console" -q) pipenv run pytest /root/lib/integ_test/integ_test/replayer_tests.py --unique_id="testindex" -s
+      - name: Collect Docker, OpenSearch Benchmark, and Shared Logs
+        if: always()
+        run: |
+          mkdir -p logs/docker logs/opensearch_benchmark_logs logs/shared_logs_output
+          for container in $(docker ps -aq); do
+            container_name=$(docker inspect --format '{{.Name}}' $container | sed 's/\///')
+            docker logs $container > logs/docker/${container_name}_logs.txt 2>&1
+          done
+          docker cp $(docker ps --filter "name=migration-console" -q):/root/.benchmark/logs logs/opensearch_benchmark_logs
+          docker cp $(docker ps --filter "name=migration-console" -q):/shared-logs-output logs/shared_logs_output
+      - name: Upload Logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-test-logs
+          path: |
+            logs/docker
+            logs/opensearch_benchmark_logs
+            logs/shared_logs_output
       - name: Clean up migrations docker images before caching
         run: |
           docker stop $(docker ps -q) && docker system prune --volumes -f
           docker image ls --format '{{.Repository}}:{{.Tag}}' | grep '^migrations/' | xargs -I {} docker image rm {}
 
-  cdk-tests:
+  node-tests:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        npm-project:
+          - ./deployment/cdk/opensearch-service-migration
+          - ./deployment/migration-assistant-solution
     defaults:
       run:
-        working-directory: ./deployment/cdk/opensearch-service-migration
+        working-directory: ${{ matrix.npm-project }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: ${{ env.node-version }}
       - name: Install NPM dependencies
         run: npm ci
-      - name: Run CDK Jest Tests
-        run: npm test
+      - name: Run CDK Jest Tests (using mocked images)
+        run: npm run test
 
   link-checker:
     runs-on: ubuntu-latest
@@ -195,26 +253,16 @@ jobs:
         env:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
 
-  fetch-migration-docker-build:
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: ./FetchMigration
-    steps:
-      - uses: actions/checkout@v4
-      - name: Build Docker Image
-        run: docker build -t migrations/fetch-migration -f Dockerfile .
 
   all-ci-checks-pass:
     needs:
-      - style-check
-      - cdk-tests
+      - console-python-tests
       - gradle-tests
       - link-checker
+      - node-tests
       - python-e2e-tests
       - python-lint
-      - python-tests
-      - fetch-migration-docker-build
+      - style-check
     if: always()
     runs-on: ubuntu-latest
     steps: