diff --git a/playbooks/roles/os_temps/tasks/build_new_app.yml b/playbooks/roles/os_temps/tasks/build_new_app.yml index 8b79f8d5..7d22df0f 100644 --- a/playbooks/roles/os_temps/tasks/build_new_app.yml +++ b/playbooks/roles/os_temps/tasks/build_new_app.yml @@ -1,56 +1,17 @@ --- - name: "{{ container_config_name }} :: Imagestream and buildconfig do not exist. Creating..." - shell: "{{ oc_bin }} new-app {{ template_name_file.stdout }} {{ params | join(' ') }}" + shell: "{{ oc_bin }} new-app {{ template_name_files[template_name] }} {{ params | join(' ') }}" args: chdir: "{{ project_dir }}/config" - when: (image_stream_name_check.stdout == "" and build_config_name_check.stdout == "" and build_success|bool == false) + when: (image_stream_name_checks[template_name] == "" and build_config_name_checks[template_name] == "" and build_results[template_name]|bool == false) # Wait container in the pipeline to start building -- name: "{{ container_config_name }} :: Wait for {{ build_config_name_file.stdout }} to be queued" - shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_file.stdout }}'" +- name: "{{ container_config_name }} :: Wait for {{ build_config_name_files[template_name] }} to be queued" + shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_files[template_name] }}'" register: oc_build_result until: oc_build_result.stdout.find(" Running ") != -1 retries: 6 delay: 10 ignore_errors: yes - when: build_success|bool == false + when: build_results[template_name]|bool == false -# Wait container in the pipeline to be finished building -- name: "{{ container_config_name }} :: Wait for {{ build_config_name_file.stdout }} to be built and marked with latest tag" - shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_file.stdout }}'" - register: oc_build_result - until: (oc_build_result.stdout.find(" Running ") == -1 or oc_build_result.stdout.find(" Failed ") != -1) - retries: 300 - delay: 10 - when: build_success|bool == false - -# Set fact if build result is Complete -- set_fact: - build_success: true - when: (build_success|bool == false and oc_build_result is defined and oc_build_result.stdout.find(" Complete ") != -1) - -# Check for failed apps and cleanup -- name: "Check for any failed app on the cluster" - shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1" - register: oc_check_app_status - ignore_errors: yes - when: build_success|bool == false - -- name: "Cleanup any failed dc, bc, routes, svc, and is for {{ oc_check_app_status.stdout }} on the cluster" - shell: "{{ oc_bin }} get all | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" - ignore_errors: yes - when: (build_success|bool == false and oc_check_app_status.stdout != "") - -- name: "Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster" - shell: "{{ oc_bin }} get {{ item }} | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" - ignore_errors: yes - with_items: - - serviceaccounts - - pvc - - rolebindings - when: (build_success|bool == false and oc_check_app_status.stdout != "") - -# If oc_build result is defined publish the outcome -- debug: - msg: "End result of building the container image :: {{ oc_build_result.stdout }}" - when: oc_build_result.stdout is defined diff --git a/playbooks/roles/os_temps/tasks/check_new_app.yml b/playbooks/roles/os_temps/tasks/check_new_app.yml new file mode 100644 index 00000000..0c5d149e --- /dev/null +++ b/playbooks/roles/os_temps/tasks/check_new_app.yml @@ -0,0 +1,29 @@ +--- +# Set current build attempt status +- set_fact: + current_build_succeeded: false + +# Wait container in the pipeline to be finished building +- name: "{{ container_config_name }} :: Wait for {{ build_config_name_files[template_name] }} to be built and marked with latest tag" + shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_files[template_name] }}'" + register: oc_build_result + until: (oc_build_result.stdout.find(" Running ") == -1 or oc_build_result.stdout.find(" Failed ") != -1) + retries: 300 + delay: 10 + when: build_results[template_name]|bool == false + +# Set fact if build result is Complete +- set_fact: + build_results: "{{ build_results | combine( {template_name: true}) }}" + current_build_succeeded: true + when: (build_results[template_name]|bool == false and oc_build_result is defined and oc_build_result.stdout.find(" Complete ") != -1) + +# If oc_build result is defined publish the outcome +- debug: + msg: "End result of building the container image :: {{ oc_build_result.stdout }}" + when: oc_build_result.stdout is defined + +# Setup tag if current container image build is successful +- name: Modify tags on images + shell: "{{ oc_bin }} get imagestream | awk '{print $2}' | grep -v DOCKER | sed 's/.*5000\\///g' | grep '{{ build_config_name_files[template_name] }}' | xargs -i {{ oc_bin }} tag {}:latest {}:{{ tag }}" + when: modify_tags|bool == true and current_build_succeeded|bool == true \ No newline at end of file diff --git a/playbooks/roles/os_temps/tasks/handle_os_templates.yml b/playbooks/roles/os_temps/tasks/handle_os_templates.yml new file mode 100644 index 00000000..6cbc7050 --- /dev/null +++ b/playbooks/roles/os_temps/tasks/handle_os_templates.yml @@ -0,0 +1,55 @@ +--- +# Start builds for each container defined in the template files +- name: "Start builds for containers defined in the template files - attempt {{ attempt_number }}" + include_tasks: "build_new_app.yml template_name={{ template_filename.path }}.processed" + with_items: "{{ os_templates.files }}" + when: (total_build_success|bool == false) + loop_control: + loop_var: template_filename + +# Check on build status of each container, finalize if done +- name: "Check on build status of containers, finalize if done - attempt {{ attempt_number }}" + include_tasks: "check_new_app.yml template_name={{ template_filename.path }}.processed" + with_items: "{{ os_templates.files }}" + when: (total_build_success|bool == false) + loop_control: + loop_var: template_filename + +- set_fact: + oc_check_app_status: + stdout: "" + +# Check for failed apps and cleanup +- name: "Check for all failed apps on the cluster" + shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds\/' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' " + register: oc_check_app_status + ignore_errors: yes + when: total_build_success|bool == false + +- name: "Cleanup all failed dc, bc, routes, svc, and imagestreams on the cluster" + shell: "{{ oc_bin }} get all | grep '{{ failed_container_name }}' | awk '{print $1}' | egrep -v 'builds\/|po\/' | xargs -i {{ oc_bin }} delete {}" + with_items: "{{ oc_check_app_status.stdout_lines }}" + loop_control: + loop_var: failed_container_name + ignore_errors: yes + when: (total_build_success|bool == false and oc_check_app_status.stdout != "") + +- name: "Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster" + shell: "{{ oc_bin }} get {{ item }} | egrep '{{ oc_check_app_status.stdout_lines|join('|') }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" + ignore_errors: yes + with_items: + - serviceaccounts + - pvc + - rolebindings + when: (total_build_success|bool == false and oc_check_app_status.stdout != "") + +- set_fact: + total_build_success: true + +# Check if all builds are marked as succesful +- name: "Check if all builds are marked as succesful" + set_fact: + total_build_success: "{{ total_build_success|bool and template_result.value|bool }}" + with_dict: "{{ build_results }}" + loop_control: + loop_var: template_result diff --git a/playbooks/roles/os_temps/tasks/setup_containers.yml b/playbooks/roles/os_temps/tasks/setup_containers.yml index 81a34c83..647661f6 100644 --- a/playbooks/roles/os_temps/tasks/setup_containers.yml +++ b/playbooks/roles/os_temps/tasks/setup_containers.yml @@ -29,9 +29,45 @@ shell: "{{ oc_bin }} process -f {{ item.path }}.processed" with_items: "{{ os_templates.files }}" -# Load project_repo templates into OpenShift -- include_tasks: "setup_os_templates.yml template_name={{ item.path }}.processed" +# Set the total build success +- set_fact: + total_build_success: false + +# Check for failed apps and cleanup +- name: "Check for all failed apps on the cluster" + shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds\/' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' " + register: oc_check_app_status + ignore_errors: yes + +- name: "Cleanup all failed dc, bc, routes, svc, and imagestreams on the cluster" + shell: "{{ oc_bin }} get all | grep '{{ failed_container_name }}' | awk '{print $1}' | egrep -v 'builds\/|po\/' | xargs -i {{ oc_bin }} delete {}" + with_items: "{{ oc_check_app_status.stdout_lines }}" + loop_control: + loop_var: failed_container_name + ignore_errors: yes + when: oc_check_app_status.stdout != "" + +- name: "Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster" + shell: "{{ oc_bin }} get {{ item }} | egrep '{{ oc_check_app_status.stdout_lines|join('|') }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" + ignore_errors: yes + with_items: + - serviceaccounts + - pvc + - rolebindings + when: oc_check_app_status.stdout != "" + +# Setup project_repo templates and store variables for each template +- name: "Setup project_repo templates and store variables for each template" + include_tasks: "setup_os_templates.yml template_name={{ item.path }}.processed" with_items: "{{ os_templates.files }}" when: os_templates.files != "" +# Handle loading of project_repo templates into OpenShift, retry 4 times +- name: "Handle loading of project_repo templates into OpenShift" + include_tasks: "handle_os_templates.yml" + with_sequence: count=4 + loop_control: + loop_var: attempt_number + when: os_templates.files != "" + diff --git a/playbooks/roles/os_temps/tasks/setup_os_templates.yml b/playbooks/roles/os_temps/tasks/setup_os_templates.yml index 2c6f7293..46a2b7e6 100644 --- a/playbooks/roles/os_temps/tasks/setup_os_templates.yml +++ b/playbooks/roles/os_temps/tasks/setup_os_templates.yml @@ -2,26 +2,6 @@ - set_fact: params: [] -# Check for failed apps and cleanup -- name: Check for any failed apps on the cluster - shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1" - register: oc_check_app_status - ignore_errors: yes - -- name: Cleanup any failed dc, bc, routes, svc, and is for {{ oc_check_app_status.stdout }} on the cluster - shell: "{{ oc_bin }} get all | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" - ignore_errors: yes - when: oc_check_app_status.stdout != "" - -- name: Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster - shell: "{{ oc_bin }} get {{ item }} | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" - ignore_errors: yes - with_items: - - serviceaccounts - - pvc - - rolebindings - when: oc_check_app_status.stdout != "" - - name: "{{ container_config_name }} :: Get template name from the yaml file" shell: "{{ oc_bin }} process -f {{ template_name }} | jq '.items[1].metadata.labels.template' | sed 's/\"//g'" register: "template_name_file" @@ -90,66 +70,15 @@ when: (item.stdout != "" and item.item.val != "") with_items: "{{ check_temp_params.results }}" +# Set all the necessary facts for building of templates and subsequent retries - set_fact: - build_success: false - -- include_tasks: build_new_app.yml - with_items: - - 1 - - 2 - - 3 - when: build_success|bool == false - -- name: "{{ container_config_name }} :: Imagestream and buildconfig do not exist. Creating..." - shell: "{{ oc_bin }} new-app {{ template_name_file.stdout }} {{ params | join(' ') }}" - when: image_stream_name_check.stdout == "" and build_config_name_check.stdout == "" and build_success|bool == false - -# Wait s2i container template to start building :: FINAL ATTEMPT -- name: "{{ container_config_name }} :: Wait for {{ build_config_name_file.stdout }} to be queued :: FINAL ATTEMPT" - shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_file.stdout }}'" - register: oc_build_result - until: oc_build_result.stdout.find(" Running ") != -1 - retries: 6 - delay: 10 - ignore_errors: yes - when: build_success|bool == false - -# Wait s2i container template to be finished building :: FINAL ATTEMPT -- name: "{{ container_config_name }} :: Wait for {{ build_config_name_file.stdout }} to be built and marked with latest tag :: FINAL ATTEMPT" - shell: "{{ oc_bin }} get builds | grep '{{ build_config_name_file.stdout }}'" - register: oc_build_result - until: (oc_build_result.stdout.find(" Running ") == -1 or oc_build_result.stdout.find(" Failed ") != -1) - retries: 300 - delay: 10 - when: build_success|bool == false - -# Check for failed apps and cleanup -- name: Check for any failed app on the cluster - shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1" - register: oc_check_app_status - ignore_errors: yes - when: build_success|bool == false + build_results: "{{ build_results|default({}) | combine( {template_name: false} ) }}" + build_config_name_files: "{{ build_config_name_files|default({}) | combine( {template_name: build_config_name_file.stdout}) }}" + build_config_name_checks: "{{ build_config_name_checks|default({}) | combine( {template_name: build_config_name_check.stdout}) }}" + image_stream_name_checks: "{{ image_stream_name_checks|default({}) | combine( {template_name: image_stream_name_check.stdout}) }}" + template_name_files: "{{ template_name_files|default({}) | combine( {template_name: template_name_file.stdout}) }}" + build_params: "{{ build_params|default({}) | combine( {template_name: params}) }}" -- name: Cleanup any failed dc, bc, routes, svc, and is for {{ oc_check_app_status.stdout }} on the cluster - shell: "{{ oc_bin }} get all | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}" - ignore_errors: yes - when: (build_success|bool == false and oc_check_app_status.stdout != "") -- name: Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster - shell: "{{ oc_bin }} get {{ item }} | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}" - ignore_errors: yes - with_items: - - serviceaccounts - - pvc - - rolebindings - when: (build_success|bool == false and oc_check_app_status.stdout != "") -# If oc_build result is defined publish the outcome -- debug: - msg: "End result of building the container image :: {{ oc_build_result.stdout }}" - when: oc_build_result.stdout is defined -# Setup tag if container image build successful -- name: Modify tags on images - shell: "{{ oc_bin }} get imagestream | awk '{print $2}' | grep -v DOCKER | sed 's/.*5000\\///g' | grep '{{ build_config_name_file.stdout }}' | xargs -i {{ oc_bin }} tag {}:latest {}:{{ tag }}" - when: modify_tags|bool == true and build_success|bool == true