From f21a92165eef360b4d5183204444d446017140db Mon Sep 17 00:00:00 2001 From: kpavic Date: Wed, 20 Jun 2018 14:23:01 +0200 Subject: [PATCH 1/6] Build images in parallel in roles/os_temps/tasks/setup_containers --- .../roles/os_temps/tasks/build_new_app.yml | 49 ++------------ .../roles/os_temps/tasks/check_new_app.yml | 45 +++++++++++++ .../os_temps/tasks/check_os_templates.yml | 24 +++++++ .../roles/os_temps/tasks/setup_containers.yml | 15 ++++- .../os_temps/tasks/setup_os_templates.yml | 65 ++----------------- 5 files changed, 94 insertions(+), 104 deletions(-) create mode 100644 playbooks/roles/os_temps/tasks/check_new_app.yml create mode 100644 playbooks/roles/os_temps/tasks/check_os_templates.yml diff --git a/playbooks/roles/os_temps/tasks/build_new_app.yml b/playbooks/roles/os_temps/tasks/build_new_app.yml index 8b79f8d5..7d22df0f 100644 --- a/playbooks/roles/os_temps/tasks/build_new_app.yml +++ b/playbooks/roles/os_temps/tasks/build_new_app.yml @@ -1,56 +1,17 @@ --- - name: "{{ container_config_name }} :: Imagestream and buildconfig do not exist. Creating..." - shell: "{{ oc_bin }} new-app {{ template_name_file.stdout }} {{ params | join(' ') }}" + shell: "{{ oc_bin }} new-app {{ template_name_files[template_name] }} {{ params | join(' ') }}" args: chdir: "{{ project_dir }}/config" - when: (image_stream_name_check.stdout == "" and build_config_name_check.stdout == "" and build_success|bool == false) + when: (image_stream_name_checks[template_name] == "" and build_config_name_checks[template_name] == "" and build_results[template_name]|bool == false) # Wait container in the pipeline to start building -- name: "{{ container_config_name }} :: Wait for {{ build_config_name_file.stdout }} to be queued" - shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_file.stdout }}'" +- name: "{{ container_config_name }} :: Wait for {{ build_config_name_files[template_name] }} to be queued" + shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_files[template_name] }}'" register: oc_build_result until: oc_build_result.stdout.find(" Running ") != -1 retries: 6 delay: 10 ignore_errors: yes - when: build_success|bool == false + when: build_results[template_name]|bool == false -# Wait container in the pipeline to be finished building -- name: "{{ container_config_name }} :: Wait for {{ build_config_name_file.stdout }} to be built and marked with latest tag" - shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_file.stdout }}'" - register: oc_build_result - until: (oc_build_result.stdout.find(" Running ") == -1 or oc_build_result.stdout.find(" Failed ") != -1) - retries: 300 - delay: 10 - when: build_success|bool == false - -# Set fact if build result is Complete -- set_fact: - build_success: true - when: (build_success|bool == false and oc_build_result is defined and oc_build_result.stdout.find(" Complete ") != -1) - -# Check for failed apps and cleanup -- name: "Check for any failed app on the cluster" - shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1" - register: oc_check_app_status - ignore_errors: yes - when: build_success|bool == false - -- name: "Cleanup any failed dc, bc, routes, svc, and is for {{ oc_check_app_status.stdout }} on the cluster" - shell: "{{ oc_bin }} get all | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" - ignore_errors: yes - when: (build_success|bool == false and oc_check_app_status.stdout != "") - -- name: "Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster" - shell: "{{ oc_bin }} get {{ item }} | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" - ignore_errors: yes - with_items: - - serviceaccounts - - pvc - - rolebindings - when: (build_success|bool == false and oc_check_app_status.stdout != "") - -# If oc_build result is defined publish the outcome -- debug: - msg: "End result of building the container image :: {{ oc_build_result.stdout }}" - when: oc_build_result.stdout is defined diff --git a/playbooks/roles/os_temps/tasks/check_new_app.yml b/playbooks/roles/os_temps/tasks/check_new_app.yml new file mode 100644 index 00000000..9bbfe5bd --- /dev/null +++ b/playbooks/roles/os_temps/tasks/check_new_app.yml @@ -0,0 +1,45 @@ +--- +# Wait container in the pipeline to be finished building +- name: "{{ container_config_name }} :: Wait for {{ build_config_name_files[template_name] }} to be built and marked with latest tag" + shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_files[template_name] }}'" + register: oc_build_result + until: (oc_build_result.stdout.find(" Running ") == -1 or oc_build_result.stdout.find(" Failed ") != -1) + retries: 300 + delay: 10 + when: build_results[template_name]|bool == false + +# Set fact if build result is Complete +- set_fact: + build_results: "{{ build_results | combine( {template_name: true}) }}" + when: (build_results[template_name]|bool == false and oc_build_result is defined and oc_build_result.stdout.find(" Complete ") != -1) + +# Check for failed apps and cleanup +- name: "Check for any failed app on the cluster" + shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1" + register: oc_check_app_status + ignore_errors: yes + when: build_results[template_name]|bool == false + +- name: "Cleanup any failed dc, bc, routes, svc, and is for {{ oc_check_app_status.stdout }} on the cluster" + shell: "{{ oc_bin }} get all | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" + ignore_errors: yes + when: (build_results[template_name]|bool == false and oc_check_app_status.stdout != "") + +- name: "Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster" + shell: "{{ oc_bin }} get {{ item }} | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" + ignore_errors: yes + with_items: + - serviceaccounts + - pvc + - rolebindings + when: (build_results[template_name]|bool == false and oc_check_app_status.stdout != "") + +# If oc_build result is defined publish the outcome +- debug: + msg: "End result of building the container image :: {{ oc_build_result.stdout }}" + when: oc_build_result.stdout is defined + +# Setup tag if container image build successful +- name: Modify tags on images + shell: "{{ oc_bin }} get imagestream | awk '{print $2}' | grep -v DOCKER | sed 's/.*5000\\///g' | grep '{{ build_config_name_files[template_name] }}' | xargs -i {{ oc_bin }} tag {}:latest {}:{{ tag }}" + when: modify_tags|bool == true and build_results[template_name]|bool == true diff --git a/playbooks/roles/os_temps/tasks/check_os_templates.yml b/playbooks/roles/os_temps/tasks/check_os_templates.yml new file mode 100644 index 00000000..4e56081a --- /dev/null +++ b/playbooks/roles/os_temps/tasks/check_os_templates.yml @@ -0,0 +1,24 @@ +--- + +# Start builds for each container defined in the template files +- name: "Start builds for container defined in the template file {{ item.path }}" + include_tasks: "build_new_app.yml template_name={{ item.path }}.processed" + with_items: "{{ os_templates.files }}" + when: (total_build_success|bool == false) + +# Check on build status of each container sequentially +- name: "Check on build status of container {{ item.path }}" + include_tasks: "check_new_app.yml template_name={{ item.path }}.processed" + with_items: "{{ os_templates.files }}" + when: (total_build_success|bool == false) + +- set_fact: + total_build_success: true + +# Check if all builds are marked as succesful +- name: "Check if all builds are marked as succesful" + set_fact: + total_build_success: "{{ total_build_success|bool and template_result.value|bool }}" + with_dict: "{{ build_results }}" + loop_control: + loop_var: template_result diff --git a/playbooks/roles/os_temps/tasks/setup_containers.yml b/playbooks/roles/os_temps/tasks/setup_containers.yml index 81a34c83..101d4c36 100644 --- a/playbooks/roles/os_temps/tasks/setup_containers.yml +++ b/playbooks/roles/os_temps/tasks/setup_containers.yml @@ -29,9 +29,20 @@ shell: "{{ oc_bin }} process -f {{ item.path }}.processed" with_items: "{{ os_templates.files }}" -# Load project_repo templates into OpenShift -- include_tasks: "setup_os_templates.yml template_name={{ item.path }}.processed" +# Set the total build success +- set_fact: + total_build_success: false + +# Setup project_repo templates and store variables for each template +- name: "Setup project_repo templates and store variables for each template" + include_tasks: "setup_os_templates.yml template_name={{ item.path }}.processed" with_items: "{{ os_templates.files }}" when: os_templates.files != "" +# Handle loading of project_repo templates into OpenShift +- name: "Handle loading of project_repo templates into OpenShift" + include_tasks: "check_os_templates.yml" + with_sequence: count=4 + when: os_templates.files != "" + diff --git a/playbooks/roles/os_temps/tasks/setup_os_templates.yml b/playbooks/roles/os_temps/tasks/setup_os_templates.yml index 2c6f7293..24478027 100644 --- a/playbooks/roles/os_temps/tasks/setup_os_templates.yml +++ b/playbooks/roles/os_temps/tasks/setup_os_templates.yml @@ -90,66 +90,15 @@ when: (item.stdout != "" and item.item.val != "") with_items: "{{ check_temp_params.results }}" +# Set all the necessary facts for building of templates and subsequent retries - set_fact: - build_success: false + build_results: "{{ build_results|default({}) | combine( {template_name: false} ) }}" + build_config_name_files: "{{ build_config_name_files|default({}) | combine( {template_name: build_config_name_file.stdout}) }}" + build_config_name_checks: "{{ build_config_name_checks|default({}) | combine( {template_name: build_config_name_check.stdout}) }}" + image_stream_name_checks: "{{ image_stream_name_checks|default({}) | combine( {template_name: image_stream_name_check.stdout}) }}" + template_name_files: "{{ template_name_files|default({}) | combine( {template_name: template_name_file.stdout}) }}" + build_params: "{{ build_params|default({}) | combine( {template_name: params}) }}" -- include_tasks: build_new_app.yml - with_items: - - 1 - - 2 - - 3 - when: build_success|bool == false - -- name: "{{ container_config_name }} :: Imagestream and buildconfig do not exist. Creating..." - shell: "{{ oc_bin }} new-app {{ template_name_file.stdout }} {{ params | join(' ') }}" - when: image_stream_name_check.stdout == "" and build_config_name_check.stdout == "" and build_success|bool == false - -# Wait s2i container template to start building :: FINAL ATTEMPT -- name: "{{ container_config_name }} :: Wait for {{ build_config_name_file.stdout }} to be queued :: FINAL ATTEMPT" - shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_file.stdout }}'" - register: oc_build_result - until: oc_build_result.stdout.find(" Running ") != -1 - retries: 6 - delay: 10 - ignore_errors: yes - when: build_success|bool == false - -# Wait s2i container template to be finished building :: FINAL ATTEMPT -- name: "{{ container_config_name }} :: Wait for {{ build_config_name_file.stdout }} to be built and marked with latest tag :: FINAL ATTEMPT" - shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_file.stdout }}'" - register: oc_build_result - until: (oc_build_result.stdout.find(" Running ") == -1 or oc_build_result.stdout.find(" Failed ") != -1) - retries: 300 - delay: 10 - when: build_success|bool == false - -# Check for failed apps and cleanup -- name: Check for any failed app on the cluster - shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1" - register: oc_check_app_status - ignore_errors: yes - when: build_success|bool == false - -- name: Cleanup any failed dc, bc, routes, svc, and is for {{ oc_check_app_status.stdout }} on the cluster - shell: "{{ oc_bin }} get all | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" - ignore_errors: yes - when: (build_success|bool == false and oc_check_app_status.stdout != "") -- name: Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster - shell: "{{ oc_bin }} get {{ item }} | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" - ignore_errors: yes - with_items: - - serviceaccounts - - pvc - - rolebindings - when: (build_success|bool == false and oc_check_app_status.stdout != "") -# If oc_build result is defined publish the outcome -- debug: - msg: "End result of building the container image :: {{ oc_build_result.stdout }}" - when: oc_build_result.stdout is defined -# Setup tag if container image build successful -- name: Modify tags on images - shell: "{{ oc_bin }} get imagestream | awk '{print $2}' | grep -v DOCKER | sed 's/.*5000\\///g' | grep '{{ build_config_name_file.stdout }}' | xargs -i {{ oc_bin }} tag {}:latest {}:{{ tag }}" - when: modify_tags|bool == true and build_success|bool == true From 9924b0e0dd1e3612b61b5b0cd38e008579f7783e Mon Sep 17 00:00:00 2001 From: kpavic Date: Wed, 20 Jun 2018 18:01:08 +0200 Subject: [PATCH 2/6] Refactoring and minor fixes for parallel builds --- playbooks/roles/os_temps/tasks/check_new_app.yml | 5 ++++- ...ck_os_templates.yml => handle_os_templates.yml} | 14 +++++++++----- .../roles/os_temps/tasks/setup_containers.yml | 6 ++++-- 3 files changed, 17 insertions(+), 8 deletions(-) rename playbooks/roles/os_temps/tasks/{check_os_templates.yml => handle_os_templates.yml} (51%) diff --git a/playbooks/roles/os_temps/tasks/check_new_app.yml b/playbooks/roles/os_temps/tasks/check_new_app.yml index 9bbfe5bd..f3719b79 100644 --- a/playbooks/roles/os_temps/tasks/check_new_app.yml +++ b/playbooks/roles/os_temps/tasks/check_new_app.yml @@ -13,6 +13,9 @@ build_results: "{{ build_results | combine( {template_name: true}) }}" when: (build_results[template_name]|bool == false and oc_build_result is defined and oc_build_result.stdout.find(" Complete ") != -1) +- set_fact: + oc_check_app_status: "" + # Check for failed apps and cleanup - name: "Check for any failed app on the cluster" shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1" @@ -42,4 +45,4 @@ # Setup tag if container image build successful - name: Modify tags on images shell: "{{ oc_bin }} get imagestream | awk '{print $2}' | grep -v DOCKER | sed 's/.*5000\\///g' | grep '{{ build_config_name_files[template_name] }}' | xargs -i {{ oc_bin }} tag {}:latest {}:{{ tag }}" - when: modify_tags|bool == true and build_results[template_name]|bool == true + when: modify_tags|bool == true and build_results[template_name]|bool == true \ No newline at end of file diff --git a/playbooks/roles/os_temps/tasks/check_os_templates.yml b/playbooks/roles/os_temps/tasks/handle_os_templates.yml similarity index 51% rename from playbooks/roles/os_temps/tasks/check_os_templates.yml rename to playbooks/roles/os_temps/tasks/handle_os_templates.yml index 4e56081a..a0221995 100644 --- a/playbooks/roles/os_temps/tasks/check_os_templates.yml +++ b/playbooks/roles/os_temps/tasks/handle_os_templates.yml @@ -1,16 +1,20 @@ --- # Start builds for each container defined in the template files -- name: "Start builds for container defined in the template file {{ item.path }}" - include_tasks: "build_new_app.yml template_name={{ item.path }}.processed" +- name: "Start builds for containers defined in the template files - attempt {{ attempt_number }}" + include_tasks: "build_new_app.yml template_name={{ template_filename.path }}.processed" with_items: "{{ os_templates.files }}" when: (total_build_success|bool == false) + loop_control: + loop_var: template_filename -# Check on build status of each container sequentially -- name: "Check on build status of container {{ item.path }}" - include_tasks: "check_new_app.yml template_name={{ item.path }}.processed" +# Check on build status of each container, finalize if done +- name: "Check on build status of containers, finalize if done - attempt {{ attempt_number }}" + include_tasks: "check_new_app.yml template_name={{ template_filename.path }}.processed" with_items: "{{ os_templates.files }}" when: (total_build_success|bool == false) + loop_control: + loop_var: template_filename - set_fact: total_build_success: true diff --git a/playbooks/roles/os_temps/tasks/setup_containers.yml b/playbooks/roles/os_temps/tasks/setup_containers.yml index 101d4c36..bc52d040 100644 --- a/playbooks/roles/os_temps/tasks/setup_containers.yml +++ b/playbooks/roles/os_temps/tasks/setup_containers.yml @@ -39,10 +39,12 @@ with_items: "{{ os_templates.files }}" when: os_templates.files != "" -# Handle loading of project_repo templates into OpenShift +# Handle loading of project_repo templates into OpenShift, retry 4 times - name: "Handle loading of project_repo templates into OpenShift" - include_tasks: "check_os_templates.yml" + include_tasks: "handle_os_templates.yml" with_sequence: count=4 + loop_control: + loop_var: attempt_number when: os_templates.files != "" From ca96fac87f59d37bf3a90dcf21a9f1c0196204a5 Mon Sep 17 00:00:00 2001 From: kpavic Date: Thu, 21 Jun 2018 16:33:24 +0200 Subject: [PATCH 3/6] Cleanup failed apps in between parallel builds --- .../roles/os_temps/tasks/check_new_app.yml | 24 --------------- .../os_temps/tasks/handle_os_templates.yml | 29 ++++++++++++++++++- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/playbooks/roles/os_temps/tasks/check_new_app.yml b/playbooks/roles/os_temps/tasks/check_new_app.yml index f3719b79..a81afee0 100644 --- a/playbooks/roles/os_temps/tasks/check_new_app.yml +++ b/playbooks/roles/os_temps/tasks/check_new_app.yml @@ -13,30 +13,6 @@ build_results: "{{ build_results | combine( {template_name: true}) }}" when: (build_results[template_name]|bool == false and oc_build_result is defined and oc_build_result.stdout.find(" Complete ") != -1) -- set_fact: - oc_check_app_status: "" - -# Check for failed apps and cleanup -- name: "Check for any failed app on the cluster" - shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1" - register: oc_check_app_status - ignore_errors: yes - when: build_results[template_name]|bool == false - -- name: "Cleanup any failed dc, bc, routes, svc, and is for {{ oc_check_app_status.stdout }} on the cluster" - shell: "{{ oc_bin }} get all | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" - ignore_errors: yes - when: (build_results[template_name]|bool == false and oc_check_app_status.stdout != "") - -- name: "Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster" - shell: "{{ oc_bin }} get {{ item }} | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" - ignore_errors: yes - with_items: - - serviceaccounts - - pvc - - rolebindings - when: (build_results[template_name]|bool == false and oc_check_app_status.stdout != "") - # If oc_build result is defined publish the outcome - debug: msg: "End result of building the container image :: {{ oc_build_result.stdout }}" diff --git a/playbooks/roles/os_temps/tasks/handle_os_templates.yml b/playbooks/roles/os_temps/tasks/handle_os_templates.yml index a0221995..00c9f00b 100644 --- a/playbooks/roles/os_temps/tasks/handle_os_templates.yml +++ b/playbooks/roles/os_temps/tasks/handle_os_templates.yml @@ -1,5 +1,4 @@ --- - # Start builds for each container defined in the template files - name: "Start builds for containers defined in the template files - attempt {{ attempt_number }}" include_tasks: "build_new_app.yml template_name={{ template_filename.path }}.processed" @@ -16,6 +15,34 @@ loop_control: loop_var: template_filename +- set_fact: + oc_check_app_status: + stdout: "" + +# Check for failed apps and cleanup +- name: "Check for all failed app on the cluster" + shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' " + register: oc_check_app_status + ignore_errors: yes + when: total_build_success|bool == false + +- name: "Cleanup all failed dc, bc, routes, svc, and imagestreams on the cluster" + shell: "{{ oc_bin }} get all | grep '{{ failed_container_name }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" + with_items: "{{ oc_check_app_status.stdout_lines }}" + loop_control: + loop_var: failed_container_name + ignore_errors: yes + when: (total_build_success|bool == false and oc_check_app_status.stdout != "") + +- name: "Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster" + shell: "{{ oc_bin }} get {{ item }} | egrep '{{ oc_check_app_status.stdout_lines|join('|') }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" + ignore_errors: yes + with_items: + - serviceaccounts + - pvc + - rolebindings + when: (total_build_success|bool == false and oc_check_app_status.stdout != "") + - set_fact: total_build_success: true From 87a6f8ccdf3f3e10a5062a98d61be87659cbc89d Mon Sep 17 00:00:00 2001 From: kpavic Date: Fri, 22 Jun 2018 09:34:08 +0200 Subject: [PATCH 4/6] Modify tag on image only once --- playbooks/roles/os_temps/tasks/check_new_app.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/playbooks/roles/os_temps/tasks/check_new_app.yml b/playbooks/roles/os_temps/tasks/check_new_app.yml index a81afee0..0c5d149e 100644 --- a/playbooks/roles/os_temps/tasks/check_new_app.yml +++ b/playbooks/roles/os_temps/tasks/check_new_app.yml @@ -1,4 +1,8 @@ --- +# Set current build attempt status +- set_fact: + current_build_succeeded: false + # Wait container in the pipeline to be finished building - name: "{{ container_config_name }} :: Wait for {{ build_config_name_files[template_name] }} to be built and marked with latest tag" shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_files[template_name] }}'" @@ -11,6 +15,7 @@ # Set fact if build result is Complete - set_fact: build_results: "{{ build_results | combine( {template_name: true}) }}" + current_build_succeeded: true when: (build_results[template_name]|bool == false and oc_build_result is defined and oc_build_result.stdout.find(" Complete ") != -1) # If oc_build result is defined publish the outcome @@ -18,7 +23,7 @@ msg: "End result of building the container image :: {{ oc_build_result.stdout }}" when: oc_build_result.stdout is defined -# Setup tag if container image build successful +# Setup tag if current container image build is successful - name: Modify tags on images shell: "{{ oc_bin }} get imagestream | awk '{print $2}' | grep -v DOCKER | sed 's/.*5000\\///g' | grep '{{ build_config_name_files[template_name] }}' | xargs -i {{ oc_bin }} tag {}:latest {}:{{ tag }}" - when: modify_tags|bool == true and build_results[template_name]|bool == true \ No newline at end of file + when: modify_tags|bool == true and current_build_succeeded|bool == true \ No newline at end of file From 3d89a5f80bfe7f36f9489c5b1f517584933032ab Mon Sep 17 00:00:00 2001 From: kpavic Date: Fri, 22 Jun 2018 14:47:27 +0200 Subject: [PATCH 5/6] Cleanup all failed builds before setting up templates --- .../os_temps/tasks/handle_os_templates.yml | 2 +- .../roles/os_temps/tasks/setup_containers.yml | 23 +++++++++++++++++++ .../os_temps/tasks/setup_os_templates.yml | 20 ---------------- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/playbooks/roles/os_temps/tasks/handle_os_templates.yml b/playbooks/roles/os_temps/tasks/handle_os_templates.yml index 00c9f00b..570c8323 100644 --- a/playbooks/roles/os_temps/tasks/handle_os_templates.yml +++ b/playbooks/roles/os_temps/tasks/handle_os_templates.yml @@ -20,7 +20,7 @@ stdout: "" # Check for failed apps and cleanup -- name: "Check for all failed app on the cluster" +- name: "Check for all failed apps on the cluster" shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' " register: oc_check_app_status ignore_errors: yes diff --git a/playbooks/roles/os_temps/tasks/setup_containers.yml b/playbooks/roles/os_temps/tasks/setup_containers.yml index bc52d040..405ffedf 100644 --- a/playbooks/roles/os_temps/tasks/setup_containers.yml +++ b/playbooks/roles/os_temps/tasks/setup_containers.yml @@ -33,6 +33,29 @@ - set_fact: total_build_success: false +# Check for failed apps and cleanup +- name: "Check for all failed apps on the cluster" + shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' " + register: oc_check_app_status + ignore_errors: yes + +- name: "Cleanup all failed dc, bc, routes, svc, and imagestreams on the cluster" + shell: "{{ oc_bin }} get all | grep '{{ failed_container_name }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" + with_items: "{{ oc_check_app_status.stdout_lines }}" + loop_control: + loop_var: failed_container_name + ignore_errors: yes + when: oc_check_app_status.stdout != "" + +- name: "Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster" + shell: "{{ oc_bin }} get {{ item }} | egrep '{{ oc_check_app_status.stdout_lines|join('|') }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" + ignore_errors: yes + with_items: + - serviceaccounts + - pvc + - rolebindings + when: oc_check_app_status.stdout != "" + # Setup project_repo templates and store variables for each template - name: "Setup project_repo templates and store variables for each template" include_tasks: "setup_os_templates.yml template_name={{ item.path }}.processed" diff --git a/playbooks/roles/os_temps/tasks/setup_os_templates.yml b/playbooks/roles/os_temps/tasks/setup_os_templates.yml index 24478027..46a2b7e6 100644 --- a/playbooks/roles/os_temps/tasks/setup_os_templates.yml +++ b/playbooks/roles/os_temps/tasks/setup_os_templates.yml @@ -2,26 +2,6 @@ - set_fact: params: [] -# Check for failed apps and cleanup -- name: Check for any failed apps on the cluster - shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1" - register: oc_check_app_status - ignore_errors: yes - -- name: Cleanup any failed dc, bc, routes, svc, and is for {{ oc_check_app_status.stdout }} on the cluster - shell: "{{ oc_bin }} get all | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" - ignore_errors: yes - when: oc_check_app_status.stdout != "" - -- name: Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster - shell: "{{ oc_bin }} get {{ item }} | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" - ignore_errors: yes - with_items: - - serviceaccounts - - pvc - - rolebindings - when: oc_check_app_status.stdout != "" - - name: "{{ container_config_name }} :: Get template name from the yaml file" shell: "{{ oc_bin }} process -f {{ template_name }} | jq '.items[1].metadata.labels.template' | sed 's/\"//g'" register: "template_name_file" From 08101df775414a296372e2698810cc239e2bd90f Mon Sep 17 00:00:00 2001 From: kpavic Date: Thu, 28 Jun 2018 16:54:02 +0200 Subject: [PATCH 6/6] Minor cleanup fix for templates that have names containing _po_ or _builds_ --- playbooks/roles/os_temps/tasks/handle_os_templates.yml | 4 ++-- playbooks/roles/os_temps/tasks/setup_containers.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/roles/os_temps/tasks/handle_os_templates.yml b/playbooks/roles/os_temps/tasks/handle_os_templates.yml index 570c8323..6cbc7050 100644 --- a/playbooks/roles/os_temps/tasks/handle_os_templates.yml +++ b/playbooks/roles/os_temps/tasks/handle_os_templates.yml @@ -21,13 +21,13 @@ # Check for failed apps and cleanup - name: "Check for all failed apps on the cluster" - shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' " + shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds\/' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' " register: oc_check_app_status ignore_errors: yes when: total_build_success|bool == false - name: "Cleanup all failed dc, bc, routes, svc, and imagestreams on the cluster" - shell: "{{ oc_bin }} get all | grep '{{ failed_container_name }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" + shell: "{{ oc_bin }} get all | grep '{{ failed_container_name }}' | awk '{print $1}' | egrep -v 'builds\/|po\/' | xargs -i {{ oc_bin }} delete {}" with_items: "{{ oc_check_app_status.stdout_lines }}" loop_control: loop_var: failed_container_name diff --git a/playbooks/roles/os_temps/tasks/setup_containers.yml b/playbooks/roles/os_temps/tasks/setup_containers.yml index 405ffedf..647661f6 100644 --- a/playbooks/roles/os_temps/tasks/setup_containers.yml +++ b/playbooks/roles/os_temps/tasks/setup_containers.yml @@ -35,12 +35,12 @@ # Check for failed apps and cleanup - name: "Check for all failed apps on the cluster" - shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' " + shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds\/' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' " register: oc_check_app_status ignore_errors: yes - name: "Cleanup all failed dc, bc, routes, svc, and imagestreams on the cluster" - shell: "{{ oc_bin }} get all | grep '{{ failed_container_name }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" + shell: "{{ oc_bin }} get all | grep '{{ failed_container_name }}' | awk '{print $1}' | egrep -v 'builds\/|po\/' | xargs -i {{ oc_bin }} delete {}" with_items: "{{ oc_check_app_status.stdout_lines }}" loop_control: loop_var: failed_container_name