Merge pull request #59 from dirgim/parallel-builds

Build container images in parallel

playbooks/roles/os_temps/tasks/build_new_app.yml

@@ -1,56 +1,17 @@
 ---
 - name: "{{ container_config_name }} :: Imagestream and buildconfig do not exist. Creating..."
-  shell: "{{ oc_bin }} new-app {{ template_name_file.stdout }} {{ params | join(' ') }}"
+  shell: "{{ oc_bin }} new-app {{ template_name_files[template_name] }} {{ params | join(' ') }}"
   args:
     chdir: "{{ project_dir }}/config"
-  when: (image_stream_name_check.stdout == "" and build_config_name_check.stdout == "" and build_success|bool == false)
+  when: (image_stream_name_checks[template_name] == "" and build_config_name_checks[template_name] == "" and build_results[template_name]|bool == false)
 
 # Wait container in the pipeline to start building
-- name: "{{ container_config_name }} :: Wait for {{  build_config_name_file.stdout }} to be queued"
-  shell: "{{ oc_bin }} get builds | grep '{{  build_config_name_file.stdout }}'"
+- name: "{{ container_config_name }} :: Wait for {{  build_config_name_files[template_name] }} to be queued"
+  shell: "{{ oc_bin }} get builds | grep '{{  build_config_name_files[template_name] }}'"
   register: oc_build_result
   until: oc_build_result.stdout.find(" Running ") != -1
   retries: 6
   delay: 10
   ignore_errors: yes
-  when: build_success|bool == false
+  when: build_results[template_name]|bool == false
 
-# Wait container in the pipeline to be finished building
-- name: "{{ container_config_name }} :: Wait for {{  build_config_name_file.stdout }} to be built and marked with latest tag"
-  shell: "{{ oc_bin }} get builds | grep '{{  build_config_name_file.stdout }}'"
-  register: oc_build_result
-  until: (oc_build_result.stdout.find(" Running ") == -1 or oc_build_result.stdout.find(" Failed ") != -1)
-  retries: 300
-  delay: 10
-  when: build_success|bool == false
-
-# Set fact if build result is Complete
-- set_fact:
-    build_success: true
-  when: (build_success|bool == false and oc_build_result is defined and oc_build_result.stdout.find(" Complete ") != -1)
-
-# Check for failed apps and cleanup
-- name: "Check for any failed app on the cluster"
-  shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1"
-  register: oc_check_app_status
-  ignore_errors: yes
-  when: build_success|bool == false
-
-- name: "Cleanup any failed dc, bc, routes, svc, and is for {{ oc_check_app_status.stdout }} on the cluster"
-  shell: "{{ oc_bin }} get all | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}"
-  ignore_errors: yes
-  when: (build_success|bool == false and oc_check_app_status.stdout != "")
-
-- name: "Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster"
-  shell: "{{ oc_bin }} get {{ item }} | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}"
-  ignore_errors: yes
-  with_items:
-    - serviceaccounts
-    - pvc
-    - rolebindings
-  when: (build_success|bool == false and oc_check_app_status.stdout != "")
-
-# If oc_build result is defined publish the outcome
-- debug:
-    msg: "End result of building the container image :: {{ oc_build_result.stdout }}"
-  when: oc_build_result.stdout is defined

playbooks/roles/os_temps/tasks/check_new_app.yml

@@ -0,0 +1,29 @@
+---
+# Set current build attempt status
+- set_fact:
+    current_build_succeeded: false
+
+# Wait container in the pipeline to be finished building
+- name: "{{ container_config_name }} :: Wait for {{  build_config_name_files[template_name] }} to be built and marked with latest tag"
+  shell: "{{ oc_bin }} get builds | grep '{{  build_config_name_files[template_name] }}'"
+  register: oc_build_result
+  until: (oc_build_result.stdout.find(" Running ") == -1 or oc_build_result.stdout.find(" Failed ") != -1)
+  retries: 300
+  delay: 10
+  when: build_results[template_name]|bool == false
+
+# Set fact if build result is Complete
+- set_fact:
+    build_results: "{{ build_results | combine( {template_name: true}) }}"
+    current_build_succeeded: true
+  when: (build_results[template_name]|bool == false and oc_build_result is defined and oc_build_result.stdout.find(" Complete ") != -1)
+
+# If oc_build result is defined publish the outcome
+- debug:
+    msg: "End result of building the container image :: {{ oc_build_result.stdout }}"
+  when: oc_build_result.stdout is defined
+
+# Setup tag if current container image build is successful
+- name: Modify tags on images
+  shell: "{{ oc_bin }} get imagestream | awk '{print $2}' | grep -v DOCKER | sed 's/.*5000\\///g' | grep '{{ build_config_name_files[template_name] }}' | xargs -i {{ oc_bin }} tag {}:latest {}:{{ tag }}"
+  when: modify_tags|bool == true and current_build_succeeded|bool == true

playbooks/roles/os_temps/tasks/handle_os_templates.yml

@@ -0,0 +1,55 @@
+---
+# Start builds for each container defined in the template files
+- name: "Start builds for containers defined in the template files - attempt {{ attempt_number }}"
+  include_tasks: "build_new_app.yml template_name={{ template_filename.path }}.processed"
+  with_items: "{{ os_templates.files }}"
+  when: (total_build_success|bool == false)
+  loop_control:
+    loop_var: template_filename
+
+# Check on build status of each container, finalize if done
+- name: "Check on build status of containers, finalize if done - attempt {{ attempt_number }}"
+  include_tasks: "check_new_app.yml template_name={{ template_filename.path }}.processed"
+  with_items: "{{ os_templates.files }}"
+  when: (total_build_success|bool == false)
+  loop_control:
+    loop_var: template_filename
+
+- set_fact:
+    oc_check_app_status:
+      stdout: ""
+
+# Check for failed apps and cleanup
+- name: "Check for all failed apps on the cluster"
+  shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds\/' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' "
+  register: oc_check_app_status
+  ignore_errors: yes
+  when: total_build_success|bool == false
+
+- name: "Cleanup all failed dc, bc, routes, svc, and imagestreams on the cluster"
+  shell: "{{ oc_bin }} get all | grep '{{ failed_container_name }}' | awk '{print $1}' | egrep -v 'builds\/|po\/' | xargs -i {{ oc_bin }} delete {}"
+  with_items: "{{ oc_check_app_status.stdout_lines }}"
+  loop_control:
+    loop_var: failed_container_name
+  ignore_errors: yes
+  when: (total_build_success|bool == false and oc_check_app_status.stdout != "")
+
+- name: "Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster"
+  shell: "{{ oc_bin }} get {{ item }} | egrep '{{ oc_check_app_status.stdout_lines|join('|') }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}"
+  ignore_errors: yes
+  with_items:
+    - serviceaccounts
+    - pvc
+    - rolebindings
+  when: (total_build_success|bool == false and oc_check_app_status.stdout != "")
+
+- set_fact:
+    total_build_success: true
+
+# Check if all builds are marked as succesful
+- name: "Check if all builds are marked as succesful"
+  set_fact:
+    total_build_success: "{{ total_build_success|bool and template_result.value|bool }}"
+  with_dict: "{{ build_results }}"
+  loop_control:
+    loop_var: template_result

playbooks/roles/os_temps/tasks/setup_containers.yml

@@ -29,9 +29,45 @@
   shell: "{{ oc_bin }} process -f {{ item.path }}.processed"
   with_items: "{{ os_templates.files }}"
 
-# Load project_repo templates into OpenShift
-- include_tasks: "setup_os_templates.yml template_name={{ item.path }}.processed"
+# Set the total build success
+- set_fact:
+    total_build_success: false
+
+# Check for failed apps and cleanup
+- name: "Check for all failed apps on the cluster"
+  shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds\/' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' "
+  register: oc_check_app_status
+  ignore_errors: yes
+
+- name: "Cleanup all failed dc, bc, routes, svc, and imagestreams on the cluster"
+  shell: "{{ oc_bin }} get all | grep '{{ failed_container_name }}' | awk '{print $1}' | egrep -v 'builds\/|po\/' | xargs -i {{ oc_bin }} delete {}"
+  with_items: "{{ oc_check_app_status.stdout_lines }}"
+  loop_control:
+    loop_var: failed_container_name
+  ignore_errors: yes
+  when: oc_check_app_status.stdout != ""
+
+- name: "Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster"
+  shell: "{{ oc_bin }} get {{ item }} | egrep '{{ oc_check_app_status.stdout_lines|join('|') }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}"
+  ignore_errors: yes
+  with_items:
+    - serviceaccounts
+    - pvc
+    - rolebindings
+  when: oc_check_app_status.stdout != ""
+
+# Setup project_repo templates and store variables for each template
+- name: "Setup project_repo templates and store variables for each template"
+  include_tasks: "setup_os_templates.yml template_name={{ item.path }}.processed"
   with_items: "{{ os_templates.files }}"
   when: os_templates.files != ""
 
+# Handle loading of project_repo templates into OpenShift, retry 4 times
+- name: "Handle loading of project_repo templates into OpenShift"
+  include_tasks: "handle_os_templates.yml"
+  with_sequence: count=4
+  loop_control:
+    loop_var: attempt_number
+  when: os_templates.files != ""
+
 

playbooks/roles/os_temps/tasks/setup_os_templates.yml

@@ -2,26 +2,6 @@
 - set_fact:
     params: []
 
-# Check for failed apps and cleanup
-- name: Check for any failed apps on the cluster
-  shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1"
-  register: oc_check_app_status
-  ignore_errors: yes
-
-- name: Cleanup any failed dc, bc, routes, svc, and is for {{ oc_check_app_status.stdout }} on the cluster
-  shell: "{{ oc_bin }} get all | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}"
-  ignore_errors: yes
-  when: oc_check_app_status.stdout != ""
-
-- name: Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster
-  shell: "{{ oc_bin }} get {{ item }} | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}"
-  ignore_errors: yes
-  with_items:
-    - serviceaccounts
-    - pvc
-    - rolebindings
-  when: oc_check_app_status.stdout != ""
-
 - name: "{{ container_config_name }} :: Get template name from the yaml file"
   shell: "{{ oc_bin }} process -f {{ template_name }} | jq '.items[1].metadata.labels.template' | sed 's/\"//g'"
   register: "template_name_file"
@@ -90,66 +70,15 @@
   when: (item.stdout != "" and item.item.val != "")
   with_items: "{{ check_temp_params.results }}"
 
+# Set all the necessary facts for building of templates and subsequent retries
 - set_fact:
-    build_success: false
-
-- include_tasks: build_new_app.yml
-  with_items:
-    - 1
-    - 2
-    - 3
-  when: build_success|bool == false
-
-- name: "{{ container_config_name }} :: Imagestream and buildconfig do not exist. Creating..."
-  shell: "{{ oc_bin }} new-app {{ template_name_file.stdout }} {{ params | join(' ') }}"
-  when: image_stream_name_check.stdout == "" and build_config_name_check.stdout == "" and build_success|bool == false
-
-# Wait s2i container template to start building :: FINAL ATTEMPT
-- name: "{{ container_config_name }} :: Wait for {{  build_config_name_file.stdout }} to be queued :: FINAL ATTEMPT"
-  shell: "{{ oc_bin }} get builds | grep '{{  build_config_name_file.stdout }}'"
-  register: oc_build_result
-  until: oc_build_result.stdout.find(" Running ") != -1
-  retries: 6
-  delay: 10
-  ignore_errors: yes
-  when: build_success|bool == false
-
-# Wait s2i container template to be finished building :: FINAL ATTEMPT
-- name: "{{ container_config_name }} :: Wait for {{  build_config_name_file.stdout }} to be built and marked with latest tag :: FINAL ATTEMPT"
-  shell: "{{ oc_bin }} get builds | grep '{{  build_config_name_file.stdout }}'"
-  register: oc_build_result
-  until: (oc_build_result.stdout.find(" Running ") == -1 or oc_build_result.stdout.find(" Failed ") != -1)
-  retries: 300
-  delay: 10
-  when: build_success|bool == false
-
-# Check for failed apps and cleanup
-- name: Check for any failed app on the cluster
-  shell: "{{ oc_bin }} get all | egrep 'Failed|Error' | egrep 'builds' | awk '{print $1}' | awk -F'/' '{print $2}' | sed 's/-[0-9+]//g' | head -1"
-  register: oc_check_app_status
-  ignore_errors: yes
-  when: build_success|bool == false
+    build_results: "{{ build_results|default({}) | combine( {template_name: false} ) }}"
+    build_config_name_files: "{{ build_config_name_files|default({}) | combine( {template_name: build_config_name_file.stdout}) }}"
+    build_config_name_checks: "{{ build_config_name_checks|default({}) | combine( {template_name: build_config_name_check.stdout}) }}"
+    image_stream_name_checks: "{{ image_stream_name_checks|default({}) | combine( {template_name: image_stream_name_check.stdout}) }}"
+    template_name_files: "{{ template_name_files|default({}) | combine( {template_name: template_name_file.stdout}) }}"
+    build_params: "{{ build_params|default({}) | combine( {template_name: params}) }}"
 
-- name: Cleanup any failed dc, bc, routes, svc, and is for {{ oc_check_app_status.stdout }} on the cluster
-  shell: "{{ oc_bin }} get all | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | egrep -v 'builds|po' | xargs -i {{ oc_bin }} delete {}"
-  ignore_errors: yes
-  when: (build_success|bool == false and oc_check_app_status.stdout != "")
 
-- name: Cleanup any serviceaccounts, pvc, and rolebindings for an app if it exists on the cluster
-  shell: "{{ oc_bin }} get {{ item }} | grep '{{ oc_check_app_status.stdout }}' | awk '{print $1}' | xargs -i {{ oc_bin }} delete {{ item }}/{}"
-  ignore_errors: yes
-  with_items:
-    - serviceaccounts
-    - pvc
-    - rolebindings
-  when: (build_success|bool == false and oc_check_app_status.stdout != "")
 
-# If oc_build result is defined publish the outcome
-- debug:
-    msg: "End result of building the container image :: {{ oc_build_result.stdout }}"
-  when: oc_build_result.stdout is defined
 
-# Setup tag if container image build successful
-- name: Modify tags on images
-  shell: "{{ oc_bin }} get imagestream | awk '{print $2}' | grep -v DOCKER | sed 's/.*5000\\///g' | grep '{{ build_config_name_file.stdout }}' | xargs -i {{ oc_bin }} tag {}:latest {}:{{ tag }}"
-  when: modify_tags|bool == true and build_success|bool == true