From 3964883f06e9eaa81bc0099809444185d4cce07a Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Tue, 2 Apr 2024 01:12:08 +0300 Subject: [PATCH] PMM-7 pass files list using action. --- .github/workflows/dashboard.yml | 24 ++++---- misc/cleanup-dash.py | 101 +++++++++++++++++++------------- 2 files changed, 72 insertions(+), 53 deletions(-) diff --git a/.github/workflows/dashboard.yml b/.github/workflows/dashboard.yml index 5bf3bb4aa2..440eb89da2 100644 --- a/.github/workflows/dashboard.yml +++ b/.github/workflows/dashboard.yml @@ -3,7 +3,7 @@ name: Test pipeline on: pull_request: # paths: -# - 'dashboard/**' +# - 'dashboards/**' jobs: cleanup_and_check: @@ -22,21 +22,23 @@ jobs: # python -m pip install --upgrade pip # pip install -r requirements.txt - - name: Get list of changed files - id: getfile - run: | - echo "::set-output name=files::$(git diff --name-only -- 'dashboard/*' | xargs)" + - name: Get all test, doc and src files that have changed + id: changed-files-yaml + uses: tj-actions/changed-files@v44 + with: + files_yaml: | + dashboards: + - dashboards/** - name: Run cleanup-dash.py on changed files + if: steps.changed-files-yaml.outputs.test_any_changed == 'true' + env: + TEST_ALL_CHANGED_FILES: ${{ steps.changed-files-yaml.outputs.test_all_changed_files }} run: | - IFS=" " read -ra FILES <<< "${{ steps.getfile.outputs.files }}" - for file in "${FILES[@]}"; do - python cleanup-dash.py "$file" + for file in ${TEST_ALL_CHANGED_FILES}; do + python cleanup-dash.py "$file" done - - name: Run cleanup-dash.py - run: python misc/cleanup-dash.py - - name: Check for changes run: | git diff --exit-code diff --git a/misc/cleanup-dash.py b/misc/cleanup-dash.py index 8857ad3eb3..8df4e7266f 100755 --- a/misc/cleanup-dash.py +++ b/misc/cleanup-dash.py @@ -11,9 +11,10 @@ import re __version__ = '1.0.0' -refresh_intervals = ['1s','5s','1m','5m','1h','6h','1d'] +refresh_intervals = ['1s', '5s', '1m', '5m', '1h', '6h', '1d'] year = str(datetime.date.today())[:4] + def set_title(dashboard): """Set Dashboard Title.""" prompt = 'Title [%s]: ' % (dashboard['title'],) @@ -91,9 +92,9 @@ def add_links(dashboard): if user_input: if user_input == 'Yes': compare_pattern = re.compile(r'^.*_Compare$') - compare_tags = [s for s in dashboard['tags'] if compare_pattern.match(s)] + compare_tags = [s for s in dashboard['tags'] if compare_pattern.match(s)] ha_pattern = re.compile(r'^.*_HA$') - ha_tags = [s for s in dashboard['tags'] if ha_pattern.match(s)] + ha_tags = [s for s in dashboard['tags'] if ha_pattern.match(s)] service_tag = "None" if len(compare_tags) > 0: match_compare = re.match("(MySQL|PostgreSQL|MongoDB|OS)", compare_tags[0]) @@ -105,18 +106,19 @@ def add_links(dashboard): if match_ha: print("HA dashboard is detected for the service %s" % match_ha.group(0)) service_tag = match_ha.group(0) - setOfLinks = ['Home', 'Query Analytics', 'Compare', 'OS', 'MySQL', 'MongoDB', 'PostgreSQL', 'MySQL_HA', 'MongoDB_HA', 'Services', 'PMM'] + set_of_links = ['Home', 'Query Analytics', 'Compare', 'OS', 'MySQL', 'MongoDB', 'PostgreSQL', 'MySQL_HA', + 'MongoDB_HA', 'Services', 'PMM'] for link in copy.deepcopy(dashboard['links']): dashboard['links'].remove(link) - for tag in setOfLinks: + for tag in set_of_links: if tag == 'Query Analytics': add_item = { 'icon': 'dashboard', 'includeVars': True, 'keepTime': True, - 'tags': [ tag ], + 'tags': [tag], 'targetBlank': False, 'title': 'Query Analytics', 'type': 'link', @@ -128,7 +130,7 @@ def add_links(dashboard): 'icon': 'doc', 'includeVars': True, 'keepTime': True, - 'tags': [ tag ], + 'tags': [tag], 'targetBlank': False, 'title': 'Home', 'type': 'link', @@ -141,7 +143,7 @@ def add_links(dashboard): 'icon': 'bolt', 'includeVars': True, 'keepTime': True, - 'tags': [ tag ], + 'tags': [tag], 'targetBlank': False, 'title': 'Compare', 'type': 'link', @@ -154,7 +156,7 @@ def add_links(dashboard): 'icon': 'bolt', 'includeVars': True, 'keepTime': True, - 'tags': [ tag ], + 'tags': [tag], 'targetBlank': False, 'title': 'Compare', 'type': 'link', @@ -167,7 +169,7 @@ def add_links(dashboard): 'icon': 'bolt', 'includeVars': True, 'keepTime': True, - 'tags': [ tag ], + 'tags': [tag], 'targetBlank': False, 'title': 'Compare', 'type': 'link', @@ -180,7 +182,7 @@ def add_links(dashboard): 'icon': 'bolt', 'includeVars': True, 'keepTime': True, - 'tags': [ tag ], + 'tags': [tag], 'targetBlank': False, 'title': 'Compare', 'type': 'link', @@ -193,7 +195,7 @@ def add_links(dashboard): 'icon': 'bolt', 'includeVars': True, 'keepTime': True, - 'tags': [ tag ], + 'tags': [tag], 'targetBlank': False, 'title': 'Compare', 'type': 'link', @@ -202,24 +204,30 @@ def add_links(dashboard): print("PostgreSQL Compare link has been added.") dashboard['links'].append(add_item) else: - if (tag in dashboard['tags'] or tag in ['Services','PMM',service_tag]) and tag not in ['Compare','Home','MySQL_HA','MongoDB_HA']: + if (tag in dashboard['tags'] or tag in ['Services', 'PMM', service_tag]) and tag not in ['Compare', + 'Home', + 'MySQL_HA', + 'MongoDB_HA']: add_item = { 'asDropdown': True, - 'includeVars': True if ((tag not in ['Services'] and tag not in ['PMM']) or 'Query Analytics' in dashboard['tags']) else False, + 'includeVars': True if ( + (tag not in ['Services'] and tag not in ['PMM']) or 'Query Analytics' in dashboard[ + 'tags']) else False, 'keepTime': True, - 'tags': [ tag ], + 'tags': [tag], 'targetBlank': False, 'title': tag, 'type': 'dashboards' } dashboard['links'].append(add_item) else: - if (tag == 'MySQL_HA' and ('MySQL' in dashboard['tags'] or service_tag == 'MySQL')) or (tag == 'MongoDB_HA' and ('MongoDB' in dashboard['tags'] or service_tag == 'MongoDB')): + if (tag == 'MySQL_HA' and ('MySQL' in dashboard['tags'] or service_tag == 'MySQL')) or ( + tag == 'MongoDB_HA' and ('MongoDB' in dashboard['tags'] or service_tag == 'MongoDB')): add_item = { 'asDropdown': True, 'includeVars': True, 'keepTime': True, - 'tags': [ tag ], + 'tags': [tag], 'targetBlank': False, 'title': 'HA', 'type': 'dashboards' @@ -341,28 +349,35 @@ def fix_datasource(dashboard): if panel['datasource'] == '${DS_QAN-API}': dashboard['panels'][panel_index]['datasource'] = 'QAN-API' if 'panels' in panel: - if (len(dashboard['panels'][panel_index]['panels']) > 0): - for panelIn_index, panelIn in enumerate(dashboard['panels'][panel_index]['panels']): - if 'datasource' in panelIn: - if dashboard['panels'][panel_index]['panels'][panelIn_index]['datasource'] == '${DS_PROMETHEUS}': - dashboard['panels'][panel_index]['panels'][panelIn_index]['datasource'] = 'Prometheus' - if dashboard['panels'][panel_index]['panels'][panelIn_index]['datasource'] == '${DS_QAN-API}': - dashboard['panels'][panel_index]['panels'][panelIn_index]['datasource'] = 'QAN-API' + if (len(dashboard['panels'][panel_index]['panels']) > 0): + for panelIn_index, panelIn in enumerate(dashboard['panels'][panel_index]['panels']): + if 'datasource' in panelIn: + if dashboard['panels'][panel_index]['panels'][panelIn_index][ + 'datasource'] == '${DS_PROMETHEUS}': + dashboard['panels'][panel_index]['panels'][panelIn_index][ + 'datasource'] = 'Prometheus' + if dashboard['panels'][panel_index]['panels'][panelIn_index][ + 'datasource'] == '${DS_QAN-API}': + dashboard['panels'][panel_index]['panels'][panelIn_index]['datasource'] = 'QAN-API' if 'mappingTypes' in panel: - for mappingTypes_index, mappingTypes in enumerate(dashboard['panels'][panel_index]['mappingTypes']): - if 'datasource' in mappingTypes: - if dashboard['panels'][panel_index]['mappingTypes'][mappingTypes_index]['datasource'] == '${DS_PROMETHEUS}': - dashboard['panels'][panel_index]['mappingTypes'][mappingTypes_index]['datasource'] = 'Prometheus' - if dashboard['panels'][panel_index]['mappingTypes'][mappingTypes_index]['datasource'] == '${DS_QAN-API}': - dashboard['panels'][panel_index]['mappingTypes'][mappingTypes_index]['datasource'] = 'QAN-API' + for mappingTypes_index, mappingTypes in enumerate(dashboard['panels'][panel_index]['mappingTypes']): + if 'datasource' in mappingTypes: + if dashboard['panels'][panel_index]['mappingTypes'][mappingTypes_index][ + 'datasource'] == '${DS_PROMETHEUS}': + dashboard['panels'][panel_index]['mappingTypes'][mappingTypes_index][ + 'datasource'] = 'Prometheus' + if dashboard['panels'][panel_index]['mappingTypes'][mappingTypes_index][ + 'datasource'] == '${DS_QAN-API}': + dashboard['panels'][panel_index]['mappingTypes'][mappingTypes_index][ + 'datasource'] = 'QAN-API' if 'templating' in element: for panel_index, panel in enumerate(dashboard['templating']['list']): - if 'datasource' in list(panel.keys()): - if panel['datasource'] == '${DS_PROMETHEUS}': - dashboard['templating']['list'][panel_index]['datasource'] = 'Prometheus' - if panel['datasource'] == '${DS_QAN-API}': - dashboard['templating']['list'][panel_index]['datasource'] = 'QAN-API' + if 'datasource' in list(panel.keys()): + if panel['datasource'] == '${DS_PROMETHEUS}': + dashboard['templating']['list'][panel_index]['datasource'] = 'Prometheus' + if panel['datasource'] == '${DS_QAN-API}': + dashboard['templating']['list'][panel_index]['datasource'] = 'QAN-API' return dashboard @@ -460,12 +475,12 @@ def add_copyrights_links(dashboard): } dashboard['panels'].append(add_item) add_item = { - 'content': "
\n

MySQL and InnoDB are trademarks of Oracle Corp. Proudly running Percona Server. Copyright (c) 2006-"+year+" Percona LLC.

\n
\n Terms of Use | \n Privacy | \n Copyright | \n Legal\n
\n
\n
\n\n\n", + 'content': "
\n

MySQL and InnoDB are trademarks of Oracle Corp. Proudly running Percona Server. Copyright (c) 2006-" + year + " Percona LLC.

\n
\n Terms of Use | \n Privacy | \n Copyright | \n Legal\n
\n
\n
\n\n\n", 'gridPos': { - 'h': 3, - 'w': 24, - 'x': 0, - 'y': 201 + 'h': 3, + 'w': 24, + 'x': 0, + 'y': 201 }, 'id': 9999, 'links': [], @@ -485,8 +500,10 @@ def main(): dashboard = json.loads(dashboard_file.read()) # registered cleanupers. - CLEANUPERS = [set_title, set_editable, set_hide_timepicker, drop_some_internal_elements, set_time, set_timezone, set_default_refresh_intervals, set_refresh, - fix_datasource, add_annotation, add_links, add_copyrights_links, set_shared_crosshear, set_unique_ids, set_dashboard_id_to_null] + CLEANUPERS = [set_title, set_editable, set_hide_timepicker, drop_some_internal_elements, set_time, set_timezone, + set_default_refresh_intervals, set_refresh, + fix_datasource, add_annotation, add_links, add_copyrights_links, set_shared_crosshear, set_unique_ids, + set_dashboard_id_to_null] for func in CLEANUPERS: dashboard = func(dashboard)