diff --git a/functions/app/start_app_prompt.sh b/functions/app/start_app_prompt.sh index 7f47bca5..f2f1ee4d 100644 --- a/functions/app/start_app_prompt.sh +++ b/functions/app/start_app_prompt.sh @@ -36,17 +36,27 @@ start_app_prompt(){ echo -e "Starting ${blue}$app_name${reset}..." - # Check if app is a cnpg instance, or an operator instance output=$(check_filtered_apps "$app_name") - if [[ $output == "${app_name},cnpg" ]]; then - scale_resources "$app_name" 120 "$replica_count" - #TODO: Add a check to ensure the pods are running - echo -e "${yellow}Sent the command to start all pods in: $app_name${reset}" - echo -e "${yellow}However, HeavyScript cannot monitor the new applications${reset}" - echo -e "${yellow}with the new postgres backend to ensure it worked..${reset}" - elif cli -c 'app chart_release scale release_name='\""$app_name"\"\ 'scale_options={"replica_count": '"$replica_count}" &> /dev/null; then + # Initialize a flag + cli_success=true + + if [[ $output == "${app_name},stopAll-on" ]]; then + if ! cli -c 'app chart_release scale release_name='\""$app_name"\"\ 'scale_options={"replica_count": '"1}" > /dev/null; then + cli_success=false + fi + if ! cli -c "app chart_release update chart_release=\"$app_name\" values={\"global\": {\"stopAll\": false}}" > /dev/null; then + cli_success=false + fi + else + if ! cli -c 'app chart_release scale release_name='\""$app_name"\"\ 'scale_options={"replica_count": '"$replica_count}" > /dev/null; then + cli_success=false + fi + fi + + # Check if all cli commands were successful + if $cli_success; then echo -e "${blue}$app_name ${green}Started${reset}" echo -e "${green}Replica count set to ${blue}$replica_count${reset}" else diff --git a/functions/backup_restore/database/cnpg_dump.sh b/functions/backup_restore/database/cnpg_dump.sh index a86344d3..eb82e133 100644 --- a/functions/backup_restore/database/cnpg_dump.sh +++ b/functions/backup_restore/database/cnpg_dump.sh @@ -8,6 +8,44 @@ get_current_replica_counts() { k3s kubectl get deploy -n ix-"$app_name" -o json | jq -r '[.items[] | select(.metadata.labels.cnpg != "true" and (.metadata.name | contains("-cnpg-main-") | not)) | {(.metadata.name): .spec.replicas}] | add' } +wait_for_pods_to_stop() { + local app_name timeout deployment_name + app_name="$1" + timeout="$2" + + SECONDS=0 + while true; do + # If a specific deployment is provided, check only its pods + if ! k3s kubectl get pods -n ix-"$app_name" \ + --field-selector=status.phase!=Succeeded,status.phase!=Failed -o=name \ + | grep -vE -- '-[[:digit:]]$' \ + | rev | cut -d- -f3- | rev \ + | grep -vE -- "-cnpg$|-cnpg-" \ + | grep -qE -- "$deployment_name$"; then + break + fi + if [[ "$SECONDS" -gt $timeout ]]; then + return 1 + fi + sleep 1 + done +} + +scale_deployments() { + local app_name timeout replicas deployment_name + app_name="$1" + timeout="$2" + replicas="${3:-$(pull_replicas "$app_name")}" + deployment_name="$4" + + # Specific deployment passed, scale only this deployment + k3s kubectl scale deployments/"$deployment_name" -n ix-"$app_name" --replicas="$replicas" + + if [[ $replicas -eq 0 ]]; then + wait_for_pods_to_stop "$app_name" "$timeout" "$deployment_name" && return 0 || return 1 + fi +} + dump_database() { app="$1" output_dir="$2/${app}" diff --git a/functions/pod/container_shell_or_logs.sh b/functions/pod/container_shell_or_logs.sh index c86ac7c3..a40cf2d8 100644 --- a/functions/pod/container_shell_or_logs.sh +++ b/functions/pod/container_shell_or_logs.sh @@ -160,7 +160,11 @@ cmd_execute_shell() { read -rsn1 -d ' ' ; echo clear -x title - if ! k3s kubectl exec -n "ix-$app_name" "${pod}" -c "$container" -it -- sh -c '[ -e /bin/bash ] && exec /bin/bash || exec /bin/sh'; then + k3s kubectl exec -n "ix-$app_name" "${pod}" -c "$container" -it -- sh -c '[ -e /bin/bash ] && exec /bin/bash || exec /bin/sh' 2> >(grep -v "command terminated with exit code 130" >&2) + status=$? + if [[ $status -eq 130 ]]; then + echo "Received exit code 130, ignoring it." + elif [[ $status -ne 0 ]]; then echo -e "${red}This container does not accept shell access, try a different one.${reset}" fi break @@ -168,6 +172,8 @@ cmd_execute_shell() { } export -f cmd_execute_shell + + cmd_execute_logs() { local lines while true diff --git a/functions/pvc/mount.sh b/functions/pvc/mount.sh index 6c1ff46b..2b52f4f5 100644 --- a/functions/pvc/mount.sh +++ b/functions/pvc/mount.sh @@ -184,7 +184,8 @@ mount_app_func(){ echo -e "${bold}Unmount Manually with:${reset}\n${blue}zfs set mountpoint=legacy \"$full_path\" && rmdir /mnt/*/mounted_pvc/$data_name${reset}" fi echo - echo -e "Or use the Unmount All option" + echo -e "${bold}Or use the Unmount All option:${reset}" + echo -e "${blue}heavyscript pvc --unmount${reset}" #Ask if user would like to mount something else while true diff --git a/functions/update/commander.sh b/functions/update/commander.sh index 4b42c5fd..6d9a52b4 100644 --- a/functions/update/commander.sh +++ b/functions/update/commander.sh @@ -78,6 +78,19 @@ skip_previously_failed_version() { return 1 } +get_apps_with_status() { + local app_name status + + # Call the existing function and process its output + while IFS=, read -r app_name status; do + # If the status is "stopAll-on" or "operator", append the app_name and status to the apps_with_status array + if [[ "$status" == "stopAll-on" ]] || [[ "$status" == "operator" ]]; then + apps_with_status+=("$app_name,$status") + fi + done < <(check_filtered_apps "${array[@]/,*}") + +} + # Skip if the image update should be ignored skip_image_update() { if [[ $old_full_ver == "$new_full_ver" && $ignore_image_update == true ]]; then @@ -179,8 +192,10 @@ handle_concurrency() { commander() { + apps_with_status=() mapfile -t array < <(get_app_info) echo_updates_header + display_update_status process_apps @@ -190,5 +205,9 @@ commander() { return fi + if [[ $rollback == true ]]; then + get_apps_with_status + fi + handle_concurrency } diff --git a/functions/update/post_process.sh b/functions/update/post_process.sh index 7a60a4de..b313f930 100644 --- a/functions/update/post_process.sh +++ b/functions/update/post_process.sh @@ -46,18 +46,23 @@ handle_rollback() { fi } -check_rollback_availability() { - # Check if app is a cnpg instance, or an operator instance - output=$(check_filtered_apps "$app_name") +failed_rollback() { + echo_array+=("Error: Application did not come up even after a rollback") + echo_array+=("Manual intervention is required\nStopping, then Abandoning") +} - # Check if the output contains the desired namespace and "cnpg" or "operator" - if [[ $output == "${app_name},cnpg" ]]; then - echo_array+=("Error: $app_name contains a cnpg instance, and cannot be rolled back") - return 1 - elif [[ $output == "${app_name},operator" ]]; then +check_rollback_availability() { + if printf '%s\0' "${apps_with_status[@]}" | grep -iFxqz "${app_name},operator"; then echo_array+=("Error: $app_name contains an operator instance, and cannot be rolled back") return 1 fi + if printf '%s\0' "${apps_with_status[@]}" | grep -iFxqz "${app_name},cnpg"; then + echo_array+=("Error: $app_name contains a CNPG deployment, and cannot be rolled back") + echo_array+=("You can attempt a force rollback by shutting down the application with heavyscript") + echo_array+=("Then rolling back from the GUI") + return 1 + fi + return 0 } post_process(){ @@ -92,6 +97,7 @@ post_process(){ SECONDS=0 continue else + failed_rollback update_stop_handler 'Stopping...' break fi diff --git a/functions/update/pre_process.sh b/functions/update/pre_process.sh index 57fc5053..16d40db7 100644 --- a/functions/update/pre_process.sh +++ b/functions/update/pre_process.sh @@ -107,7 +107,11 @@ pre_process() { fi if [[ $rollback == true || "$startstatus" == "STOPPED" ]]; then - if ! check_replicas; then + if printf '%s\0' "${apps_with_status[@]}" | grep -iFxqz "${app_name},stopAll-on"; then + echo_array+=("Stopped") + echo_array + return + elif ! check_replicas; then echo_array return fi diff --git a/utils/check_filtered_apps.sh b/utils/check_filtered_apps.sh index d21f7f23..b4fb0b78 100644 --- a/utils/check_filtered_apps.sh +++ b/utils/check_filtered_apps.sh @@ -4,10 +4,9 @@ check_filtered_apps() { # Define a function to process each app name process_app_name() { - appname=$1 + local app_name=$1 - # Run the command and directly check if the values are true, and include the reason - midclt call chart.release.get_instance "$appname" | jq -r ' + midclt call chart.release.get_instance "$app_name" | jq -r ' if .config.operator.enabled == true then .name + ",operator" else @@ -17,6 +16,13 @@ check_filtered_apps() { .name + ",cnpg" else empty + end, + if .config.global.stopAll == true then + .name + ",stopAll-on" + elif .config.global.stopAll == false then + .name + ",stopAll-off" + else + empty end | select(length > 0) ' @@ -48,9 +54,9 @@ check_filtered_apps() { fi # Process the app names with a maximum of 5 concurrent processes - for appname in "${app_names[@]}"; do + for app_name in "${app_names[@]}"; do wait_for_slot - process_app_name "$appname" & + process_app_name "$app_name" & done # Wait for any remaining jobs to finish diff --git a/utils/resources.sh b/utils/resources.sh index 643053d7..39a866e9 100644 --- a/utils/resources.sh +++ b/utils/resources.sh @@ -7,33 +7,13 @@ pull_replicas() { midclt call chart.release.get_instance "$app_name" | jq '.config.controller.replicas // .config.workload.main.replicas // .pod_status.desired' } -scale_resources() { - local app_name timeout replicas deployment_name - app_name="$1" - timeout="$2" - replicas="${3:-$(pull_replicas "$app_name")}" - deployment_name="$4" - - if [[ -z "$deployment_name" ]]; then - # No specific deployment passed, scale all applicable deployments in the app - k3s kubectl get deployments -n ix-"$app_name" -o json | jq -r '.items[] | select(.metadata.name | contains("-cnpg-main-") | not) | .metadata.name' | xargs -r -I{} k3s kubectl scale deployments/{} -n ix-"$app_name" --replicas="$replicas" - else - # Specific deployment passed, scale only this deployment - k3s kubectl scale deployments/"$deployment_name" -n ix-"$app_name" --replicas="$replicas" - fi - - if [[ $replicas -eq 0 ]]; then - wait_for_pods_to_stop "$app_name" "$timeout" "$deployment_name" && return 0 || return 1 - fi -} - - restart_app(){ # There are no good labels to use to identify the deployment, so we have to simply filter out the cnpg deployment for now - dep_name=$(k3s kubectl -n ix-"$app_name" get deploy | grep -vE -- '(-cnpg-)' | sed -e '1d' -e 's/ .*//') - if k3s kubectl -n ix-"$app_name" rollout restart deploy "$dep_name" &>/dev/null; then - return 0 - else - return 1 - fi -} \ No newline at end of file + dep_names=$(k3s kubectl -n ix-"$app_name" get deploy | grep -vE -- '(-cnpg-)' | sed -e '1d' -e 's/ .*//') + for dep_name in $dep_names; do + if ! k3s kubectl -n ix-"$app_name" rollout restart deploy "$dep_name" &>/dev/null; then + return 1 + fi + done + return 0 +} diff --git a/utils/stop_app.sh b/utils/stop_app.sh index 1ba632f4..fcf8d851 100644 --- a/utils/stop_app.sh +++ b/utils/stop_app.sh @@ -1,41 +1,5 @@ #!/bin/bash - -wait_for_pods_to_stop() { - local app_name timeout deployment_name - app_name="$1" - timeout="$2" - deployment_name="$3" - - SECONDS=0 - while true; do - if [[ -n "$deployment_name" ]]; then - # If a specific deployment is provided, check only its pods - if ! k3s kubectl get pods -n ix-"$app_name" \ - --field-selector=status.phase!=Succeeded,status.phase!=Failed -o=name \ - | grep -vE -- '-[[:digit:]]$' \ - | rev | cut -d- -f3- | rev \ - | grep -vE -- "-cnpg$|-cnpg-" \ - | grep -qE -- "$deployment_name$"; then - break - fi - else - # If no specific deployment is provided, check any non-CNPG pods - if ! k3s kubectl get pods -n ix-"$app_name" \ - --field-selector=status.phase!=Succeeded,status.phase!=Failed -o=name \ - | grep -v -- '-cnpg-' \ - | grep -vqE -- '-[[:digit:]]$'; then - break - fi - fi - if [[ "$SECONDS" -gt $timeout ]]; then - return 1 - fi - sleep 1 - done -} - - get_app_status() { local app_name stop_type app_name="$1" @@ -46,7 +10,8 @@ get_app_status() { else cli -m csv -c 'app chart_release query name,status' | \ grep -- "^$app_name," | \ - awk -F ',' '{print $2}' + awk -F ',' '{print $2}' | \ + tr -d " \t\r" fi } @@ -84,30 +49,37 @@ stop_app() { app_name="$2" timeout="150" - # Check if app is a cnpg instance, or an operator instance - output=$(check_filtered_apps "$app_name") - - # Check if the output contains the desired namespace and "cnpg" or "operator" - if [[ $output == "${app_name},cnpg" ]]; then - scale_resources "$app_name" "$timeout" 0 && return 0 || return 1 - elif [[ $output == "${app_name},operator" ]]; then - return 3 - fi - + handle_timeout() { + local timeout_result=$1 + if [[ $timeout_result -eq 0 ]]; then + return 0 + elif [[ $timeout_result -eq 124 ]]; then + return 2 + else + return 1 + fi + } status=$(get_app_status "$app_name" "$stop_type") + if [[ "$status" == "STOPPED" ]]; then return 0 fi - timeout "${timeout}s" cli -c 'app chart_release scale release_name='\""$app_name"\"\ 'scale_options={"replica_count": 0}' &> /dev/null - timeout_result=$? - - if [[ $timeout_result -eq 0 ]]; then - return 0 - elif [[ $timeout_result -eq 124 ]]; then - return 2 - fi + output=$(check_filtered_apps "$app_name") - return 1 + # Check if the output contains the desired namespace and "cnpg" or "operator" + case $output in + "${app_name},stopAll-on" | "${app_name},stopAll-off") + timeout "${timeout}s" cli -c "app chart_release update chart_release=\"$app_name\" values={\"global\": {\"stopAll\": true}}" > /dev/null + handle_timeout $? + ;; + "${app_name},operator") + return 3 + ;; + *) + timeout "${timeout}s" cli -c 'app chart_release scale release_name='\""$app_name"\"\ 'scale_options={"replica_count": 0}' > /dev/null + handle_timeout $? + ;; + esac } \ No newline at end of file