Skip to content

Commit

Permalink
Change override flow (demisto#32832)
Browse files Browse the repository at this point in the history
* test changes in the override flow

* fix

* test

* fixed path to script

* test

* test

* fix logging

* fix logging

* test

* test

* fix

* test

* test

* test

* Test

* Test

* test

* test

* fixed path to artifcats

* fixed path2

* tests

* format

* removed file version

* removed file version

* last upload

* bucket

* di

* rn

* pytest

* revert

* removed unused inport

* fixed unit test

* override reverted to master

* CR

* cr

* override new template

* override only when content is shown

* fix
  • Loading branch information
eyalpalo authored Mar 10, 2024
1 parent f471e91 commit a8393af
Show file tree
Hide file tree
Showing 12 changed files with 331 additions and 341 deletions.
2 changes: 1 addition & 1 deletion .gitlab/ci/.gitlab-ci.global.yml
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@
.get_last_upload_commit: &get_last_upload_commit
- section_start "Getting last bucket upload commit" --collapsed
- gcloud auth activate-service-account --key-file="$GCS_MARKET_KEY" >> "${ARTIFACTS_FOLDER}/logs/gcloud_auth.log" 2>&1
- gsutil cp "gs://$GCS_MARKET_BUCKET/content/packs/index.json" "${ARTIFACTS_FOLDER_SERVER_TYPE}/previous_index.json"
- gsutil cp "gs://$GCS_PRODUCTION_BUCKET/content/packs/index.json" "${ARTIFACTS_FOLDER_SERVER_TYPE}/previous_index.json"
- export LAST_UPLOAD_COMMIT=$(cat "${ARTIFACTS_FOLDER_SERVER_TYPE}/previous_index.json" | jq -r ".\"commit\"")
- section_end "Getting last bucket upload commit"

Expand Down
4 changes: 4 additions & 0 deletions .gitlab/ci/.gitlab-ci.on-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,10 @@ validate-content-conf:
- ./Tests/scripts/prepare_content_packs_for_testing.sh "$MARKETPLACE_BUCKET" "$STORAGE_BASE_PATH" "$MARKETPLACE_VERSION"
- section_end "Prepare Content Packs for Testing"

- section_start "Override and upload core packs versions"
- ./Tests/Marketplace/upload_versions_core_files.sh "$MARKETPLACE_BUCKET" "$STORAGE_BASE_PATH" "$MARKETPLACE_VERSION" "$LAST_UPLOAD_COMMIT"
- section_end "Override and upload core packs versions"

- section_start "Create Instances for XSOAR"
- |
if [[ ${MARKETPLACE_VERSION} = "xsoar" ]]; then
Expand Down
2 changes: 1 addition & 1 deletion Tests/Marketplace/Tests/marketplace_services_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -3018,7 +3018,7 @@ class TestVersionsMetadataFile:

def test_version_map(self):
version_map_content = GCPConfig.versions_metadata_contents.get('version_map')
valid_keys = {'core_packs_file', 'core_packs_file_is_locked', 'file_version', 'marketplaces'}
valid_keys = {'core_packs_file', 'core_packs_file_is_locked', 'marketplaces'}
for version, core_packs_info in version_map_content.items():
missing_keys = set(valid_keys).difference(core_packs_info.keys()).difference({'marketplaces'})
unexpected_keys = set(core_packs_info.keys()).difference(valid_keys)
Expand Down
135 changes: 135 additions & 0 deletions Tests/Marketplace/Tests/override_core_packs_versions_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
import json
import os
import pytest
from unittest.mock import MagicMock


@pytest.fixture
def mocked_content_repo():
mocked_content_repo = MagicMock()
mocked_commit = MagicMock()
mocked_blob = MagicMock()
mocked_blob.data_stream.read.return_value = b'{\
"server_version": "8.3.0",\
"xsoar":\
{\
"updated_corepacks_content":\
{\
"corePacks": [],\
"upgradeCorePacks": [],\
"buildNumber": "123"\
}\
}\
}'
mocked_commit.tree.__truediv__.return_value = mocked_blob
mocked_content_repo.commit.return_value = mocked_commit
yield mocked_content_repo


def test_should_override_locked_corepacks_file(mocker, mocked_content_repo):
"""
When:
- running the should_override_locked_corepacks_file command
Given:
1. corepacks_versions.json file that wasn't changed since last upload.
2. corepacks_versions.json file that was changed since last upload, and the marketplace is as the upload.
3. corepacks_versions.json file that was changed since last upload, but the marketplace is not as the upload.
Then:
1. Assert that the result returned from the function is False
2. Assert that the result returned from the function is True
3. Assert that the result returned from the function is False
"""

from Tests.Marketplace.override_core_packs_versions import should_override_locked_corepacks_file
from Tests.Marketplace.marketplace_constants import GCPConfig
marketplace = "xsoar"
last_upload_commit = "abcd1234"

mocker.patch('Tests.Marketplace.override_core_packs_versions.get_content_git_client', return_value=mocked_content_repo)

# Case 1
corepacks_override = {
"server_version": "8.3.0",
'xsoar':
{
"updated_corepacks_content":
{
"corePacks": [],
"upgradeCorePacks": [],
"buildNumber": "123"
}

}
}
mocker.patch.object(GCPConfig, "corepacks_override_contents", corepacks_override)
assert not should_override_locked_corepacks_file(marketplace, last_upload_commit)

# Case 2
corepacks_override = {
"server_version": "8.2.0",
'xsoar':
{
"updated_corepacks_content":
{
"corePacks": [],
"upgradeCorePacks": [],
"buildNumber": "123"
}
}
}
mocker.patch.object(GCPConfig, "corepacks_override_contents", corepacks_override)
assert should_override_locked_corepacks_file(marketplace, last_upload_commit)

# Case 3
corepacks_override = {
"server_version": "8.2.0",
'marketplacev2':
{
"updated_corepacks_content":
{
"corePacks": [],
"upgradeCorePacks": [],
"buildNumber": "123"
}
}
}
mocker.patch.object(GCPConfig, "corepacks_override_contents", corepacks_override)
assert not should_override_locked_corepacks_file(marketplace, last_upload_commit)


def test_override_locked_corepacks_file(mocker):
"""
Test the override_locked_corepacks_file function.
"""
from Tests.Marketplace.override_core_packs_versions import override_locked_corepacks_file
from Tests.Marketplace.marketplace_constants import GCPConfig
import shutil

# Create a temp artifacts dir for the corepacks files:
artifacts_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tmp')
os.makedirs(artifacts_dir, exist_ok=True)

corepacks_override = {
"server_version": "8.2.0",
"xsoar": {
"updated_corepacks_content":
{
"corePacks": ['pack1', 'pack2'],
"upgradeCorePacks": ['pack3'],
"buildNumber": "123"
}

}
}
mocker.patch.object(GCPConfig, "corepacks_override_contents", corepacks_override)

override_locked_corepacks_file(build_number='456', artifacts_dir=artifacts_dir, marketplace='xsoar')

# Assert that the file was created in the artifacts folder with the build number as expected:
with open(os.path.join(artifacts_dir, 'corepacks-8.2.0.json')) as corepacks_file:
corepacks_file_contents = json.load(corepacks_file)
assert corepacks_file_contents.get('buildNumber') == '456'
assert corepacks_file_contents.get('corePacks') == ['pack1', 'pack2']

# Remove the temp artifacts dir that was created for testing:
shutil.rmtree(artifacts_dir)
137 changes: 0 additions & 137 deletions Tests/Marketplace/Tests/upload_packs_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -452,143 +452,6 @@ def test_corepacks_files_upload(self, mocker):
# Remove the temp artifacts dir that was created for testing:
shutil.rmtree(artifacts_dir)

def test_should_override_locked_corepacks_file(self, mocker):
"""
When:
- running the should_override_locked_corepacks_file command
Given:
1. A server version in the corepacks_override file that does not exist in the versions-metadata file
2. A valid server version in the corepacks_override file, but a file version that is not greater than the
file version in the versions-metadata file.
3. The marketplace to override the corepacks file to, doesn't match the current marketplace.
4. A valid server version and a file version greater than the file version in the versions-metadata file.
Then:
1. Assert that the result returned from the function is False
2. Assert that the result returned from the function is False
3. Assert that the result returned from the function is False
4. Assert that the result returned from the function is True
"""

from Tests.Marketplace.upload_packs import should_override_locked_corepacks_file
from Tests.Marketplace.marketplace_constants import GCPConfig

versions_metadata = {
"8.2.0": {
"core_packs_file": "corepacks-8.2.0.json",
"core_packs_file_is_locked": True,
"file_version": "1"
}
}
mocker.patch.object(GCPConfig, "core_packs_file_versions", versions_metadata)

# Case 1
corepacks_override = {
"server_version": "8.3.0",
"file_version": "1",
"updated_corepacks_content":
{
"corePacks": [],
"upgradeCorePacks": [],
"buildNumber": "123"
}
}
mocker.patch.object(GCPConfig, "corepacks_override_contents", corepacks_override)
assert not should_override_locked_corepacks_file()

# Case 2
corepacks_override = {
"server_version": "8.2.0",
"file_version": "1",
"updated_corepacks_content":
{
"corePacks": [],
"upgradeCorePacks": [],
"buildNumber": "123"
}
}
mocker.patch.object(GCPConfig, "corepacks_override_contents", corepacks_override)
assert not should_override_locked_corepacks_file()

# Case 3
corepacks_override = {
"server_version": "8.2.0",
"file_version": "1",
"marketplaces": [
"xsoar"
],
"updated_corepacks_content":
{
"corePacks": [],
"upgradeCorePacks": [],
"buildNumber": "123"
}
}
mocker.patch.object(GCPConfig, "corepacks_override_contents", corepacks_override)
assert not should_override_locked_corepacks_file(marketplace='marketplacev2')

# Case 4
corepacks_override = {
"server_version": "8.2.0",
"file_version": "2",
"updated_corepacks_content":
{
"corePacks": [],
"upgradeCorePacks": [],
"buildNumber": "123"
}
}
mocker.patch.object(GCPConfig, "corepacks_override_contents", corepacks_override)
assert should_override_locked_corepacks_file()

def test_override_locked_corepacks_file(self, mocker):
"""
Test the override_locked_corepacks_file function.
"""
from Tests.Marketplace.upload_packs import override_locked_corepacks_file
from Tests.Marketplace.marketplace_constants import GCPConfig
import shutil

# Create a temp artifacts dir for the corepacks files:
artifacts_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tmp')
os.makedirs(artifacts_dir, exist_ok=True)

corepacks_override = {
"server_version": "8.2.0",
"file_version": "2",
"updated_corepacks_content":
{
"corePacks": ['pack1', 'pack2'],
"upgradeCorePacks": ['pack3'],
"buildNumber": "123"
}
}
mocker.patch.object(GCPConfig, "corepacks_override_contents", corepacks_override)

versions_metadata_content = {
"version_map": {
"8.2.0": {
"core_packs_file": "corepacks-8.2.0.json",
"core_packs_file_is_locked": True,
"file_version": "1"
}
}
}
mocker.patch.object(GCPConfig, "versions_metadata_contents", versions_metadata_content)

override_locked_corepacks_file(build_number='456', artifacts_dir=artifacts_dir)

# Assert that the file was created in the artifacts folder with the build number as expected:
with open(os.path.join(artifacts_dir, 'corepacks-8.2.0.json')) as corepacks_file:
corepacks_file_contents = json.load(corepacks_file)
assert corepacks_file_contents.get('buildNumber') == '456'
assert corepacks_file_contents.get('corePacks') == ['pack1', 'pack2']

# Assert that the versions-metadata file was updated with the required file version:
assert GCPConfig.versions_metadata_contents.get('version_map').get('8.2.0').get('file_version') == '2'

# Remove the temp artifacts dir that was created for testing:
shutil.rmtree(artifacts_dir)


class TestUpdatedPrivatePacks:

Expand Down
73 changes: 7 additions & 66 deletions Tests/Marketplace/corepacks_override.json
Original file line number Diff line number Diff line change
@@ -1,67 +1,8 @@
{
"server_version": "8.4.0",
"file_version": "3",
"marketplaces": [
"xsoar_saas"
],
"updated_corepacks_content": {
"corePacks": [
"Whois/1.5.1/Whois.zip",
"Base/1.33.6/Base.zip",
"TIM_SIEM/1.1.1/TIM_SIEM.zip",
"CommonTypes/3.3.85/CommonTypes.zip",
"FeedUnit42v2/1.0.35/FeedUnit42v2.zip",
"EDL/3.2.8/EDL.zip",
"ImageOCR/1.1.18/ImageOCR.zip",
"CommonReports/1.0.3/CommonReports.zip",
"DemistoLocking/1.0.6/DemistoLocking.zip",
"CommonDashboards/1.3.6/CommonDashboards.zip",
"Unit42Intel/1.0.13/Unit42Intel.zip",
"VirusTotal/2.6.10/VirusTotal.zip",
"AutoFocus/2.1.14/AutoFocus.zip",
"DefaultPlaybook/2.0.4/DefaultPlaybook.zip",
"TIM_Processing/1.1.21/TIM_Processing.zip",
"DemistoRESTAPI/1.3.35/DemistoRESTAPI.zip",
"FiltersAndTransformers/1.2.29/FiltersAndTransformers.zip",
"CommonWidgets/1.2.30/CommonWidgets.zip",
"ThreatIntelligenceManagement/1.1.5/ThreatIntelligenceManagement.zip",
"PAN-OS/2.1.4/PAN-OS.zip",
"rasterize/1.2.25/rasterize.zip",
"HelloWorld/1.2.22/HelloWorld.zip",
"Palo_Alto_Networks_WildFire/2.1.33/Palo_Alto_Networks_WildFire.zip",
"CommonScripts/1.12.20/CommonScripts.zip",
"CommonPlaybooks/2.4.2/CommonPlaybooks.zip",
"ThreatIntelReports/1.0.10/ThreatIntelReports.zip",
"FeedMitreAttackv2/1.1.22/FeedMitreAttackv2.zip"
],
"upgradeCorePacks": [
"AutoFocus",
"Base",
"CommonDashboards",
"CommonPlaybooks",
"CommonReports",
"CommonScripts",
"CommonTypes",
"CommonWidgets",
"DefaultPlaybook",
"DemistoLocking",
"DemistoRESTAPI",
"EDL",
"FeedMitreAttackv2",
"FeedUnit42v2",
"FiltersAndTransformers",
"HelloWorld",
"ImageOCR",
"Palo_Alto_Networks_WildFire",
"PAN-OS",
"TIM_Processing",
"TIM_SIEM",
"ThreatIntelReports",
"ThreatIntelligenceManagement",
"Unit42Intel",
"VirusTotal",
"Whois",
"rasterize"
]
}
}
"server_version": "8.4.0",
"xsoar_saas": {
"updated_corepacks_content": {
}
}
}

1 change: 1 addition & 0 deletions Tests/Marketplace/marketplace_constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
XSOAR_SAAS_END_TAG = "</~XSOAR_SAAS>"
XSOAR_ON_PREM_TAG = "<~XSOAR_ON_PREM>"
XSOAR_ON_PREM_END_TAG = "</~XSOAR_ON_PREM>"
COREPACKS_OVERRIDE_FILE_PATH = "Tests/Marketplace/corepacks_override.json"

TAGS_BY_MP = {
XSIAM_MP: (XSIAM_START_TAG, XSIAM_END_TAG),
Expand Down
Loading

0 comments on commit a8393af

Please sign in to comment.