From 924f6c0a5ec3a02eb05580c26e45e03a4f43311a Mon Sep 17 00:00:00 2001 From: Timothy Smith Date: Mon, 29 Jan 2024 11:14:57 -0500 Subject: [PATCH] rebasish --- test/integration/global_deploy_test.rb | 191 ++++++++ test/integration/krane_test.rb | 147 ++++++ test/integration/render_task_test.rb | 415 +++++++++++++++++ test/integration/restart_task_test.rb | 419 ++++++++++++++++++ test/integration/runner_task_test.rb | 317 +++++++++++++ .../integration/task_config_validator_test.rb | 60 +++ test/test_helper.rb | 2 +- 7 files changed, 1550 insertions(+), 1 deletion(-) create mode 100644 test/integration/global_deploy_test.rb create mode 100644 test/integration/krane_test.rb create mode 100644 test/integration/render_task_test.rb create mode 100644 test/integration/restart_task_test.rb create mode 100644 test/integration/runner_task_test.rb create mode 100644 test/integration/task_config_validator_test.rb diff --git a/test/integration/global_deploy_test.rb b/test/integration/global_deploy_test.rb new file mode 100644 index 000000000..b82e0a468 --- /dev/null +++ b/test/integration/global_deploy_test.rb @@ -0,0 +1,191 @@ +# frozen_string_literal: true +require 'integration_test_helper' + +class GlobalDeployTest < Krane::IntegrationTest + def test_global_deploy_task_success + assert_deploy_success(deploy_global_fixtures('globals')) + + assert_logs_match_all([ + "Phase 1: Initializing deploy", + "Using resource selector app=krane,test=", + "All required parameters and files are present", + "Discovering resources:", + " - StorageClass/#{storage_class_name}", + "Phase 2: Checking initial resource statuses", + %r{StorageClass\/#{storage_class_name}\s+Not Found}, + "Phase 3: Deploying all resources", + "Deploying resources:", + %r{StorageClass\/#{storage_class_name} \(timeout: 300s\)}, + %r{PriorityClass/#{priority_class_name} \(timeout: 300s\)}, + "Don't know how to monitor resources of type StorageClass.", + %r{Assuming StorageClass\/#{storage_class_name} deployed successfully.}, + %r{Successfully deployed in [\d.]+s: PriorityClass/#{priority_class_name}, StorageClass\/#{storage_class_name}}, + "Result: SUCCESS", + "Successfully deployed 2 resources", + "Successful resources", + "StorageClass/#{storage_class_name}", + "PriorityClass/#{priority_class_name}", + ]) + end + + def test_global_deploy_task_success_timeout + assert_deploy_failure(deploy_global_fixtures('globals', global_timeout: 0), :timed_out) + + assert_logs_match_all([ + "Phase 1: Initializing deploy", + "Using resource selector app=krane,test=", + "All required parameters and files are present", + "Discovering resources:", + " - StorageClass/#{storage_class_name}", + "Phase 2: Checking initial resource statuses", + %r{StorageClass\/#{storage_class_name}\s+Not Found}, + "Phase 3: Deploying all resources", + "Deploying resources:", + "Result: TIMED OUT", + "Timed out waiting for 2 resources to deploy", + %r{StorageClass\/#{storage_class_name}: GLOBAL WATCH TIMEOUT \(0 seconds\)}, + "If you expected it to take longer than 0 seconds for your deploy to roll out, increase --global-timeout.", + ]) + end + + def test_global_deploy_task_success_verify_false + assert_deploy_success(deploy_global_fixtures('globals', verify_result: false)) + + assert_logs_match_all([ + "Phase 1: Initializing deploy", + "Using resource selector app=krane,test=", + "All required parameters and files are present", + "Discovering resources:", + " - StorageClass/#{storage_class_name}", + " - PriorityClass/#{priority_class_name}", + "Phase 2: Checking initial resource statuses", + %r{StorageClass\/#{storage_class_name}\s+Not Found}, + %r{PriorityClass/#{priority_class_name}\s+Not Found}, + "Phase 3: Deploying all resources", + "Deploying resources:", + %r{StorageClass\/#{storage_class_name} \(timeout: 300s\)}, + %r{PriorityClass/#{priority_class_name} \(timeout: 300s\)}, + "Result: SUCCESS", + "Deployed 2 resources", + "Deploy result verification is disabled for this deploy.", + "This means the desired changes were communicated to Kubernetes, but the"\ + " deploy did not make sure they actually succeeded.", + ]) + end + + def test_global_deploy_task_empty_selector_validation_failure + assert_deploy_failure(deploy_global_fixtures('globals', selector: false)) + assert_logs_match_all([ + "Phase 1: Initializing deploy", + "Result: FAILURE", + "Configuration invalid", + "- Selector is required", + ]) + end + + def test_global_deploy_task_success_selector + selector = "extraSelector=krane2" + assert_deploy_success(deploy_global_fixtures('globals', selector: selector)) + + assert_logs_match_all([ + "Phase 1: Initializing deploy", + "Using resource selector #{selector}", # there are more, but this one should be listed first + "All required parameters and files are present", + "Discovering resources:", + " - StorageClass/#{storage_class_name}", + "Phase 2: Checking initial resource statuses", + %r{StorageClass\/#{storage_class_name}\s+Not Found}, + "Phase 3: Deploying all resources", + "Deploying resources:", + %r{PriorityClass/#{priority_class_name} \(timeout: 300s\)}, + %r{StorageClass\/#{storage_class_name} \(timeout: 300s\)}, + "Don't know how to monitor resources of type StorageClass.", + "Assuming StorageClass/#{storage_class_name} deployed successfully.", + /Successfully deployed in [\d.]+s/, + "Result: SUCCESS", + "Successfully deployed 2 resources", + "Successful resources", + "StorageClass/#{storage_class_name}", + "PriorityClass/#{priority_class_name}", + ]) + end + + def test_global_deploy_task_failure + result = deploy_global_fixtures('globals') do |fixtures| + fixtures.dig("storage_classes.yml", "StorageClass").first["metadata"]['badField'] = "true" + end + assert_deploy_failure(result) + + assert_logs_match_all([ + "Phase 1: Initializing deploy", + "Using resource selector app=krane,test=", + "All required parameters and files are present", + "Discovering resources:", + " - StorageClass/#{storage_class_name}", + "Result: FAILURE", + "Invalid template", + ]) + end + + def test_global_deploy_prune_success + selector = 'extraSelector=prune1' + assert_deploy_success(deploy_global_fixtures('globals', selector: selector)) + reset_logger + assert_deploy_success(deploy_global_fixtures('globals', subset: 'storage_classes.yml', selector: selector)) + + assert_logs_match_all([ + "Phase 1: Initializing deploy", + "Using resource selector #{selector}", + "All required parameters and files are present", + "Discovering resources:", + " - StorageClass/#{storage_class_name}", + "Phase 2: Checking initial resource statuses", + %r{StorageClass\/#{storage_class_name}\s+Exists}, + "Phase 3: Deploying all resources", + %r{Deploying StorageClass\/#{storage_class_name} \(timeout: 300s\)}, + "The following resources were pruned: priorityclass.scheduling.k8s.io/#{priority_class_name}", + %r{Successfully deployed in [\d.]+s: StorageClass\/#{storage_class_name}}, + "Result: SUCCESS", + "Pruned 1 resource and successfully deployed 1 resource", + "Successful resources", + "StorageClass/#{storage_class_name}", + ]) + end + + def test_no_prune_global_deploy_success + selector = 'extraSelector=prune2' + assert_deploy_success(deploy_global_fixtures('globals', selector: selector)) + reset_logger + assert_deploy_success(deploy_global_fixtures('globals', subset: 'storage_classes.yml', + selector: selector, prune: false)) + assert_logs_match_all([ + "Phase 1: Initializing deploy", + "Using resource selector #{selector}", + "All required parameters and files are present", + "Discovering resources:", + " - StorageClass/#{storage_class_name}", + "Phase 2: Checking initial resource statuses", + %r{StorageClass\/#{storage_class_name}\s+Exists}, + "Phase 3: Deploying all resources", + %r{Deploying StorageClass\/#{storage_class_name} \(timeout: 300s\)}, + %r{Successfully deployed in [\d.]+s: StorageClass\/#{storage_class_name}}, + "Result: SUCCESS", + "Successfully deployed 1 resource", + "Successful resources", + "StorageClass/#{storage_class_name}", + ]) + refute_logs_match(/[pP]runed/) + refute_logs_match(priority_class_name) + assert_deploy_success(deploy_global_fixtures('globals', selector: selector)) + end + + private + + def storage_class_name + @storage_class_name ||= add_unique_prefix_for_test("testing-storage-class") + end + + def priority_class_name + @priority_class_name ||= add_unique_prefix_for_test("testing-priority-class") + end +end diff --git a/test/integration/krane_test.rb b/test/integration/krane_test.rb new file mode 100644 index 000000000..2df9b7ccb --- /dev/null +++ b/test/integration/krane_test.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true +require 'integration_test_helper' + +class KraneTest < Krane::IntegrationTest + def test_restart_black_box + assert_deploy_success( + deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "web.yml.erb", "redis.yml"], render_erb: true) + ) + refute(fetch_restarted_at("web"), "restart annotation on fresh deployment") + refute(fetch_restarted_at("redis"), "restart annotation on fresh deployment") + + out, err, status = krane_black_box("restart", "#{@namespace} #{KubeclientHelper::TEST_CONTEXT} --deployments web") + assert_empty(out) + assert_match("Success", err) + assert_predicate(status, :success?) + + assert(fetch_restarted_at("web"), "no restart annotation is present after the restart") + refute(fetch_restarted_at("redis"), "restart annotation is present") + end + + def test_run_success_black_box + assert_empty(task_runner_pods) + assert_deploy_success(deploy_fixtures("hello-cloud", subset: ["template-runner.yml", "configmap-data.yml"])) + template = "hello-cloud-template-runner" + out, err, status = krane_black_box("run", + "#{@namespace} #{KubeclientHelper::TEST_CONTEXT} --template #{template} --command ls --arguments '-a /'") + assert_match("Success", err) + assert_empty(out) + assert_predicate(status, :success?) + assert_equal(1, task_runner_pods.count) + end + + def test_render_black_box + # Ordered so that template requiring bindings comes first + paths = ["test/fixtures/test-partials/partials/independent-configmap.yml.erb", + "test/fixtures/hello-cloud/web.yml.erb"] + data_value = rand(10_000).to_s + bindings = "data=#{data_value}" + test_sha = rand(10_000).to_s + + out, err, status = krane_black_box("render", + "-f #{paths.join(' ')} --bindings #{bindings} --current-sha #{test_sha}") + + assert_predicate(status, :success?) + assert_match("Success", err) + assert_match(test_sha, out) + assert_match(data_value, out) + + out, err, status = krane_black_box("render", "-f #{paths.join(' ')} --current-sha #{test_sha}") + + refute_predicate(status, :success?) + assert_match("FAILURE", err) + refute_match(data_value, out) + assert_match(test_sha, out) + end + + def test_render_black_box_stdin + file = "test/fixtures/branched/web.yml.erb" + template = File.read(file) + data_value = rand(10_000).to_s + bindings = "branch=#{data_value}" + test_sha = rand(10_000).to_s + + out, err, status = krane_black_box("render", + "--filenames - --bindings #{bindings} --current-sha #{test_sha}", stdin: template) + + assert_predicate(status, :success?) + assert_match("Success", err) + assert_match(test_sha, out) + assert_match(data_value, out) + end + + def test_render_current_sha_cant_be_blank + paths = ["test/fixtures/test-partials/partials/independent-configmap.yml.erb"] + _, err, status = krane_black_box("render", "-f #{paths.join(' ')} --current-sha") + refute_predicate(status, :success?) + assert_match("FAILURE", err) + assert_match("current-sha is optional but can not be blank", err) + end + + def test_deploy_black_box_success + setup_template_dir("hello-cloud", subset: %w(bare_replica_set.yml)) do |target_dir| + flags = "-f #{target_dir}" + out, err, status = krane_black_box("deploy", "#{@namespace} #{KubeclientHelper::TEST_CONTEXT} #{flags}") + assert_empty(out) + assert_match("Success", err) + assert_predicate(status, :success?) + end + end + + def test_deploy_black_box_success_stdin + render_out, _, render_status = krane_black_box("render", + "-f #{fixture_path('hello-cloud')} --bindings deployment_id=1 current_sha=123") + assert_predicate(render_status, :success?) + + out, err, status = krane_black_box("deploy", "#{@namespace} #{KubeclientHelper::TEST_CONTEXT} --filenames -", + stdin: render_out) + assert_empty(out) + assert_match("Success", err) + assert_predicate(status, :success?) + end + + def test_deploy_black_box_failure + out, err, status = krane_black_box("deploy", "#{@namespace} #{KubeclientHelper::TEST_CONTEXT}") + assert_empty(out) + assert_match("--filenames must be set and not empty", err) + refute_predicate(status, :success?) + assert_equal(status.exitstatus, 1) + end + + def test_deploy_black_box_timeout + setup_template_dir("hello-cloud", subset: %w(bare_replica_set.yml)) do |target_dir| + flags = "-f #{target_dir} --global-timeout=0.1s" + out, err, status = krane_black_box("deploy", "#{@namespace} #{KubeclientHelper::TEST_CONTEXT} #{flags}") + assert_empty(out) + assert_match("TIMED OUT", err) + refute_predicate(status, :success?) + assert_equal(status.exitstatus, 70) + end + end + + # test_global_deploy_black_box_success and test_global_deploy_black_box_timeout + # are in test/integration-serial/serial_deploy_test.rb because they modify + # global state + + def test_global_deploy_black_box_failure + setup_template_dir("resource-quota") do |target_dir| + flags = "-f #{target_dir} --selector app=krane" + out, err, status = krane_black_box("global-deploy", "#{KubeclientHelper::TEST_CONTEXT} #{flags}") + assert_empty(out) + assert_match("FAILURE", err) + refute_predicate(status, :success?) + assert_equal(status.exitstatus, 1) + end + end + + private + + def task_runner_pods + kubeclient.get_pods(namespace: @namespace, label_selector: "name=runner,app=hello-cloud") + end + + def fetch_restarted_at(deployment_name) + deployment = apps_v1_kubeclient.get_deployment(deployment_name, @namespace) + deployment.spec.template.metadata.annotations&.dig(Krane::RestartTask::RESTART_TRIGGER_ANNOTATION) + end +end diff --git a/test/integration/render_task_test.rb b/test/integration/render_task_test.rb new file mode 100644 index 000000000..2d94c386a --- /dev/null +++ b/test/integration/render_task_test.rb @@ -0,0 +1,415 @@ +# frozen_string_literal: true +require 'test_helper' +require 'krane/render_task' + +class RenderTaskTest < Krane::TestCase + include FixtureDeployHelper + + def test_render_task + render = build_render_task( + File.join(fixture_path('hello-cloud'), 'configmap-data.yml') + ) + + assert_render_success(render.run(stream: mock_output_stream)) + + stdout_assertion do |output| + assert_equal(output, <<~RENDERED) + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: hello-cloud-configmap-data + labels: + name: hello-cloud-configmap-data + app: hello-cloud + data: + datapoint1: value1 + datapoint2: value2 + RENDERED + end + end + + def test_render_task_multiple_templates + SecureRandom.expects(:hex).with(4).returns('aaaa') + SecureRandom.expects(:hex).with(6).returns('bbbbbb') + render = build_render_task([ + File.join(fixture_path('hello-cloud'), 'configmap-data.yml'), + File.join(fixture_path('hello-cloud'), 'unmanaged-pod-1.yml.erb'), + ]) + assert_render_success(render.run(stream: mock_output_stream)) + + stdout_assertion do |output| + expected = <<~RENDERED + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: hello-cloud-configmap-data + labels: + name: hello-cloud-configmap-data + app: hello-cloud + data: + datapoint1: value1 + datapoint2: value2 + --- + apiVersion: v1 + kind: Pod + metadata: + name: unmanaged-pod-1-kbbbbbb-aaaa + annotations: + krane.shopify.io/timeout-override: 60s + labels: + type: unmanaged-pod + name: unmanaged-pod-1-kbbbbbb-aaaa + app: hello-cloud + spec: + activeDeadlineSeconds: 60 + restartPolicy: Never + containers: + - name: hello-cloud + image: busybox + imagePullPolicy: IfNotPresent + command: ["sh", "-c", "echo 'Hello from the command runner!' && test 1 -eq 1"] + env: + - name: CONFIG + valueFrom: + configMapKeyRef: + name: hello-cloud-configmap-data + key: datapoint2 + RENDERED + assert_equal(expected, output) + end + end + + def test_render_task_with_partials_and_bindings + render = build_render_task( + File.join(fixture_path('test-partials'), 'deployment.yaml.erb'), + 'supports_partials': 'yep' + ) + + assert_render_success(render.run(stream: mock_output_stream)) + stdout_assertion do |output| + expected = <<~RENDERED + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: web + labels: + name: web + app: test-partials + spec: + replicas: 1 + selector: + matchLabels: + name: web + app: test-partials + template: + metadata: + labels: + name: web + app: test-partials + spec: {"containers":[{"name":"sleepy-guy","image":"busybox","imagePullPolicy":"IfNotPresent","command":["sleep","8000"]}]} + --- + apiVersion: v1 + kind: Pod + metadata: + name: pod1 + spec: + restartPolicy: "Never" + activeDeadlineSeconds: 60 + containers: + - name: pod1 + image: busybox + args: ["echo", "log from pod1"] + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: config-for-pod1 + data: + supports_partials: "yep" + + # This is valid + --- # leave this whitespace + apiVersion: v1 + kind: ConfigMap + metadata: + name: independent-configmap + data: + value: "renderer test" + + --- + apiVersion: v1 + kind: Pod + metadata: + name: pod2 + spec: + restartPolicy: "Never" + activeDeadlineSeconds: 60 + containers: + - name: pod2 + image: busybox + args: ["echo", "log from pod2"] + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: config-for-pod2 + data: + supports_partials: "yep" + + RENDERED + assert_equal(expected, output) + end + end + + def test_render_task_rendering_all_files + render = build_render_task(fixture_path('hello-cloud')) + + assert_render_success(render.run(stream: mock_output_stream)) + stdout_assertion do |output| + assert_match(/name: bare-replica-set/, output) + assert_match(/name: hello-cloud-configmap-data/, output) + assert_match(/name: ds-app/, output) + assert_match(/kind: PodDisruptionBudget/, output) + assert_match(/name: hello-job/, output) + assert_match(/name: redis/, output) + assert_match(/name: role-binding/, output) + assert_match(/name: resource-quotas/, output) + assert_match(/name: allow-all-network-policy/, output) + assert_match(/name: build-robot/, output) + assert_match(/name: stateful-busybox/, output) + assert_match(/name: hello-cloud-template-runner/, output) + assert_match(/name: unmanaged-pod-\w+/, output) + assert_match(/name: web/, output) + end + end + + def test_render_task_multiple_templates_with_middle_failure + render = build_render_task([ + File.join(fixture_path('some-invalid'), 'configmap-data.yml'), + File.join(fixture_path('some-invalid'), 'yaml-error.yml'), + File.join(fixture_path('some-invalid'), 'stateful_set.yml'), + ]) + assert_render_failure(render.run(stream: mock_output_stream)) + + stdout_assertion do |output| + assert_match(/name: hello-cloud-configmap-data/, output) + assert_match(/name: stateful-busybox/, output) + end + + logging_assertion do |logs| + assert_match(/Invalid template: yaml-error.yml/, logs) + end + end + + def test_render_invalid_binding + render = build_render_task( + File.join(fixture_path('test-partials'), 'deployment.yaml.erb'), + 'a': 'binding-a', + 'b': 'binding-b' + ) + assert_render_failure(render.run(stream: mock_output_stream)) + assert_logs_match_all([ + /Invalid template: .*deployment.yaml.erb/, + "> Error message:", + /undefined local variable or method `supports_partials'/, + "> Template content:", + 'supports_partials: "<%= supports_partials %>"', + ], in_order: true) + end + + def test_render_runtime_error_when_rendering + render = build_render_task( + File.join(fixture_path('invalid'), 'raise_inside.yml.erb') + ) + assert_render_failure(render.run(stream: mock_output_stream)) + assert_logs_match_all([ + /Invalid template: .*raise_inside.yml.erb/, + "> Error message:", + /mock error when evaluating erb/, + "> Template content:", + 'datapoint1: <% raise RuntimeError, "mock error when evaluating erb" %>', + ], in_order: true) + end + + def test_render_empty_template_dir + tmp_dir = Dir.mktmpdir + render = build_render_task(tmp_dir) + assert_render_failure(render.run(stream: mock_output_stream)) + assert_logs_match_all([ + "Template directory #{tmp_dir} does not contain any valid templates", + ]) + end + + def test_render_invalid_yaml + render = build_render_task( + File.join(fixture_path('invalid'), 'yaml-error.yml'), + data: "data" + ) + assert_render_failure(render.run(stream: mock_output_stream)) + assert_logs_match_all([ + /Invalid template: .*yaml-error.yml/, + "> Error message:", + /mapping values are not allowed/, + ], in_order: true) + end + + def test_render_valid_fixtures + load_fixtures('hello-cloud', nil).each do |basename, _docs| + render = build_render_task( + File.join(fixture_path('hello-cloud'), basename) + ) + assert_render_success(render.run(stream: mock_output_stream)) + stdout_assertion do |output| + assert(!output.empty?) + end + end + end + + def test_render_only_adds_initial_doc_separator_when_missing + render = build_render_task([ + File.join(fixture_path('partials'), 'no-doc-separator.yml.erb'), + File.join(fixture_path('partials'), 'no-doc-separator.yml.erb'), + ]) + expected = "---\n# The first doc has no yaml separator\nkey1: foo\n---\nkey2: bar\n" + + assert_render_success(render.run(stream: mock_output_stream)) + stdout_assertion do |output| + assert_equal("#{expected}#{expected}", output) + end + + mock_output_stream.rewind + render = build_render_task( + File.join(fixture_path('test-partials/partials'), 'independent-configmap.yml.erb'), + data: "data" + ) + expected = <<~RENDERED + # This is valid + --- # leave this whitespace + apiVersion: v1 + kind: ConfigMap + metadata: + name: independent-configmap + data: + value: "data" + RENDERED + + assert_render_success(render.run(stream: mock_output_stream)) + stdout_assertion do |output| + assert_equal(expected, output) + end + end + + def test_render_preserves_duplicate_keys + render = build_render_task( + File.join(fixture_path('partials'), 'duplicate-keys.yml.erb') + ) + expected = "---\nkey1: \"0\"\nkey1: \"1\"\nkey1: \"2\"\n" + + assert_render_success(render.run(stream: mock_output_stream)) + stdout_assertion do |output| + assert_equal(expected, output) + end + end + + def test_render_does_not_generate_extra_blank_documents_when_file_is_empty + render = build_render_task( + File.join(fixture_path('collection-with-erb'), 'effectively_empty.yml.erb') + ) + assert_render_success(render.run(stream: mock_output_stream)) + stdout_assertion do |output| + assert_equal("", output.strip) + end + assert_logs_match("Rendered effectively_empty.yml.erb successfully, but the result was blank") + end + + def test_render_errors_empty_sha + render = Krane::RenderTask.new(logger: logger, current_sha: "", bindings: {}, + filenames: [fixture_path('test-partials')]) + + assert_render_failure(render.run(stream: mock_output_stream)) + assert_logs_match_all([ + "Result: FAILURE", + "Configuration invalid", + "- current-sha is optional but can not be blank", + ], in_order: true) + end + + def test_render_k8s_compatibility + render = build_render_task( + File.join(fixture_path('k8s-compatibility'), 'pod_with_e_notation_env.yml') + ) + + assert_render_success(render.run(stream: mock_output_stream)) + + stdout_assertion do |output| + assert_equal(output, <<~RENDERED) + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: web + annotations: + shipit.shopify.io/restart: "true" + labels: + name: web + app: hello-cloud + spec: + replicas: 1 + selector: + matchLabels: + name: web + app: hello-cloud + progressDeadlineSeconds: 60 + template: + metadata: + labels: + name: web + app: hello-cloud + spec: + containers: + - name: app + image: busybox + imagePullPolicy: IfNotPresent + command: ["tail", "-f", "/dev/null"] + ports: + - containerPort: 80 + name: http + env: + - name: FOO1 + value: "1e3" + - name: FOO2 + value: "1e+3" + - name: FOO2 + value: "1e-3" + RENDERED + end + end + + private + + def build_render_task(filenames, bindings = {}) + Krane::RenderTask.new( + logger: logger, + current_sha: "k#{SecureRandom.hex(6)}", + bindings: bindings, + filenames: Array(filenames) + ) + end + + def assert_render_success(result) + assert_equal(true, result, "Render failed when it was expected to succeed.#{logs_message_if_captured}") + logging_assertion do |logs| + assert_match(Regexp.new("Result: SUCCESS"), logs, "'Result: SUCCESS' not found in the following logs:\n#{logs}") + end + end + + def assert_render_failure(result) + assert_equal(false, result, "Render succeeded when it was expected to fail.#{logs_message_if_captured}") + logging_assertion do |logs| + assert_match(Regexp.new("Result: FAILURE"), logs, "'Result: FAILURE' not found in the following logs:\n#{logs}") + end + end +end diff --git a/test/integration/restart_task_test.rb b/test/integration/restart_task_test.rb new file mode 100644 index 000000000..d13a78193 --- /dev/null +++ b/test/integration/restart_task_test.rb @@ -0,0 +1,419 @@ +# frozen_string_literal: true +require 'integration_test_helper' +require 'krane/restart_task' + +class RestartTaskTest < Krane::IntegrationTest + def test_restart_by_annotation + assert_deploy_success(deploy_fixtures("hello-cloud", + subset: ["configmap-data.yml", "web.yml.erb", "redis.yml", "stateful_set.yml", "daemon_set.yml"], + render_erb: true)) + + refute(fetch_restarted_at("web"), "no restart annotation on fresh deployment") + refute(fetch_restarted_at("stateful-busybox", kind: :statefulset), "no restart annotation on fresh stateful set") + refute(fetch_restarted_at("ds-app", kind: :daemonset), "no restart annotation on fresh daemon set") + refute(fetch_restarted_at("redis"), "no restart annotation on fresh deployment") + + restart = build_restart_task + assert_restart_success(restart.perform) + + assert_logs_match_all([ + "Configured to restart all workloads with the `shipit.shopify.io/restart` annotation", + "Triggered `Deployment/web` restart", + "Waiting for rollout", + %r{Successfully restarted in \d+\.\d+s: Deployment/web}, + "Result: SUCCESS", + "Successfully restarted 3 resources", + %r{Deployment/web.*1 availableReplica}, + %r{StatefulSet/stateful-busybox.* 2 replicas}, + %r{DaemonSet/ds-app.* 1 updatedNumberScheduled}, + ], + in_order: true) + + assert(fetch_restarted_at("web"), "restart annotation is present after the restart") + assert(fetch_restarted_at("stateful-busybox", kind: :statefulset), "restart annotation on fresh stateful set") + assert(fetch_restarted_at("ds-app", kind: :daemonset), "restart annotation on fresh daemon set") + refute(fetch_restarted_at("redis"), "no restart annotation env on fresh deployment") + end + + def test_restart_statefulset_on_delete_restarts_child_pods + result = deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "unmanaged-pod-1.yml.erb", + "stateful_set.yml"], render_erb: true) do |fixtures| + statefulset = fixtures["stateful_set.yml"]["StatefulSet"].first + statefulset["spec"]["updateStrategy"] = { "type" => "OnDelete" } + end + assert_deploy_success(result) + before_pods = kubeclient.get_pods(namespace: @namespace, label_selector: "name=stateful-busybox").map do |p| + p.metadata.uid + end + + restart = build_restart_task + assert_restart_success(restart.perform) + after_pods = kubeclient.get_pods(namespace: @namespace, label_selector: "name=stateful-busybox").map do |p| + p.metadata.uid + end + refute_equal(before_pods.sort, after_pods.sort) + + assert_logs_match_all([ + "Configured to restart all workloads with the `shipit.shopify.io/restart` annotation", + "Triggered `StatefulSet/stateful-busybox` restart", + "`StatefulSet/stateful-busybox` has updateStrategy: OnDelete, Restarting by forcefully deleting child pods", + "Waiting for rollout", + "Result: SUCCESS", + "Successfully restarted 1 resource", + %r{StatefulSet/stateful-busybox.* 2 replicas}, + ], + in_order: true) + end + + def test_restart_by_selector + assert_deploy_success(deploy_fixtures("branched", + bindings: { "branch" => "master" }, + selector: Krane::LabelSelector.parse("branch=master"), + render_erb: true)) + assert_deploy_success(deploy_fixtures("branched", + bindings: { "branch" => "staging" }, + selector: Krane::LabelSelector.parse("branch=staging"), + render_erb: true)) + + refute(fetch_restarted_at("master-web"), "no restart annotation on fresh deployment") + refute(fetch_restarted_at("staging-web"), "no restart annotation on fresh deployment") + refute(fetch_restarted_at("master-stateful-busybox", kind: :statefulset), + "no restart annotation on fresh stateful set") + refute(fetch_restarted_at("staging-stateful-busybox", kind: :statefulset), + "no restart annotation on fresh stateful set") + refute(fetch_restarted_at("master-ds-app", kind: :daemonset), "no restart annotation on fresh daemon set") + refute(fetch_restarted_at("staging-ds-app", kind: :daemonset), "no restart annotation on fresh daemon set") + + restart = build_restart_task + assert_restart_success(restart.perform(selector: Krane::LabelSelector.parse("branch=staging"))) + + assert_logs_match_all([ + "Configured to restart all workloads with the `shipit.shopify.io/restart` annotation " \ + "and branch=staging selector", + "Triggered `Deployment/staging-web` restart", + "Triggered `StatefulSet/staging-stateful-busybox` restart", + "Triggered `DaemonSet/staging-ds-app` restart", + "Waiting for rollout", + %r{Successfully restarted in \d+\.\ds: Deployment/staging-web}, + "Result: SUCCESS", + "Successfully restarted 3 resources", + %r{Deployment/staging-web.*1 availableReplica}, + ], + in_order: true) + + assert(fetch_restarted_at("staging-web"), "restart annotation is present after the restart") + refute(fetch_restarted_at("master-web"), "no restart annotation on fresh deployment") + assert(fetch_restarted_at("staging-stateful-busybox", kind: :statefulset), + "restart annotation is present after the restart") + refute(fetch_restarted_at("master-stateful-busybox", kind: :statefulset), + "no restart annotation on fresh stateful set") + assert(fetch_restarted_at("staging-ds-app", kind: :daemonset), "restart annotation is present after the restart") + refute(fetch_restarted_at("master-ds-app", kind: :daemonset), "no restart annotation on fresh daemon set") + end + + def test_restart_by_annotation_none_found + restart = build_restart_task + assert_restart_failure(restart.perform) + assert_logs_match_all([ + "Configured to restart all workloads with the `shipit.shopify.io/restart` annotation", + "Result: FAILURE", + %r{No deployments, statefulsets, or daemonsets, with the `shipit\.shopify\.io/restart` annotation found}, + ], + in_order: true) + end + + def test_restart_named_workloads_twice + assert_deploy_success(deploy_fixtures("hello-cloud", + subset: ["configmap-data.yml", "web.yml.erb", "stateful_set.yml", "daemon_set.yml"], + render_erb: true)) + + refute(fetch_restarted_at("web"), "no restart annotation on fresh deployment") + + restart = build_restart_task + assert_restart_success( + restart.perform(deployments: %w(web), statefulsets: %w(stateful-busybox), daemonsets: %w(ds-app)) + ) + + assert_logs_match_all([ + "Configured to restart deployments by name: web", + "Configured to restart statefulsets by name: stateful-busybox", + "Configured to restart daemonsets by name: ds-app", + "Triggered `Deployment/web` restart", + "Triggered `StatefulSet/stateful-busybox` restart", + "Triggered `DaemonSet/ds-app` restart", + "Waiting for rollout", + %r{Successfully restarted in \d+\.\d+s: Deployment/web}, + "Result: SUCCESS", + "Successfully restarted 3 resources", + %r{Deployment/web.*1 availableReplica}, + ], + in_order: true) + + first_restarted_at_deploy = fetch_restarted_at("web") + first_restarted_at_statefulset = fetch_restarted_at("stateful-busybox", kind: :statefulset) + first_restarted_at_daemonset = fetch_restarted_at("ds-app", kind: :daemonset) + assert(first_restarted_at_deploy, "restart annotation is present after first restart") + assert(first_restarted_at_statefulset, "restart annotation is present after first restart") + assert(first_restarted_at_daemonset, "restart annotation is present after first restart") + + Timecop.freeze(1.second.from_now) do + assert_restart_success( + restart.perform(deployments: %w(web), statefulsets: %w(stateful-busybox), daemonsets: %w(ds-app)) + ) + end + + second_restarted_at_deploy = fetch_restarted_at("web") + second_restarted_at_statefulset = fetch_restarted_at("stateful-busybox", kind: :statefulset) + second_restarted_at_daemonset = fetch_restarted_at("ds-app", kind: :daemonset) + assert(second_restarted_at_deploy, "restart annotation is present after second restart") + assert(second_restarted_at_statefulset, "restart annotation is present after second restart") + assert(second_restarted_at_daemonset, "restart annotation is present after second restart") + refute_equal(first_restarted_at_deploy, second_restarted_at_deploy) + refute_equal(first_restarted_at_statefulset, second_restarted_at_statefulset) + refute_equal(first_restarted_at_daemonset, second_restarted_at_daemonset) + end + + def test_restart_with_same_resource_twice + assert_deploy_success(deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "web.yml.erb"], + render_erb: true)) + + refute(fetch_restarted_at("web"), "no restart annotation on fresh deployment") + + restart = build_restart_task + assert_restart_success(restart.perform(deployments: %w(web web))) + + assert_logs_match_all([ + "Configured to restart deployments by name: web", + "Triggered `Deployment/web` restart", + "Result: SUCCESS", + "Successfully restarted 1 resource", + %r{Deployment/web.*1 availableReplica}, + ], + in_order: true) + + assert(fetch_restarted_at("web"), "restart annotation is present after the restart") + end + + def test_restart_not_existing_deployment + restart = build_restart_task + assert_restart_failure(restart.perform(deployments: %w(web))) + assert_logs_match_all([ + "Configured to restart deployments by name: web", + "Result: FAILURE", + "Deployment `web` not found in namespace", + ], + in_order: true) + end + + def test_restart_one_not_existing_deployment + assert(deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "web.yml.erb"], render_erb: true)) + + restart = build_restart_task + assert_restart_failure(restart.perform(deployments: %w(walrus web))) + + refute(fetch_restarted_at("web"), "no restart annotation after failed restart task") + assert_logs_match_all([ + "Configured to restart deployments by name: walrus, web", + "Result: FAILURE", + "Deployment `walrus` not found in namespace", + ], + in_order: true) + end + + def test_restart_deployments_and_selector + restart = build_restart_task + assert_restart_failure(restart.perform(deployments: %w(web), selector: Krane::LabelSelector.parse("app=web"))) + assert_logs_match_all([ + "Result: FAILURE", + "Can't specify workload names and selector at the same time", + ], + in_order: true) + end + + def test_restart_not_existing_context + restart = Krane::RestartTask.new( + context: "walrus", + namespace: @namespace, + logger: logger + ) + assert_restart_failure(restart.perform(deployments: %w(web))) + assert_logs_match_all([ + "Result: FAILURE", + /- Context walrus missing from your kubeconfig file\(s\)/, + ], + in_order: true) + end + + def test_restart_not_existing_namespace + restart = Krane::RestartTask.new( + context: KubeclientHelper::TEST_CONTEXT, + namespace: "walrus", + logger: logger + ) + assert_restart_failure(restart.perform(deployments: %w(web))) + assert_logs_match_all([ + "Result: FAILURE", + "- Could not find Namespace: walrus in Context: #{KubeclientHelper::TEST_CONTEXT}", + ], + in_order: true) + end + + def test_restart_failure + success = deploy_fixtures("downward_api", subset: ["configmap-data.yml", "web.yml.erb"], + render_erb: true) do |fixtures| + deployment = fixtures["web.yml.erb"]["Deployment"].first + deployment["spec"]["progressDeadlineSeconds"] = 30 + container = deployment["spec"]["template"]["spec"]["containers"].first + container["readinessProbe"] = { + "failureThreshold" => 1, + "periodSeconds" => 1, + "initialDelaySeconds" => 0, + "exec" => { + "command" => [ + "/bin/sh", + "-c", + "test $(cat /etc/podinfo/annotations | grep -s kubectl.kubernetes.io/restartedAt -c) -eq 0", + ], + }, + } + end + assert_deploy_success(success) + + restart = build_restart_task + assert_raises(Krane::DeploymentTimeoutError) { restart.perform!(deployments: %w(web)) } + + assert_logs_match_all([ + "Triggered `Deployment/web` restart", + "Deployment/web rollout timed out", + "Result: TIMED OUT", + "Timed out waiting for 1 resource to restart", + "Deployment/web: TIMED OUT", + "The following containers have not passed their readiness probes", + "app must exit 0 from the following command", + "Final status: 2 replicas, 1 updatedReplica, 1 availableReplica, 1 unavailableReplica", + "Unhealthy: Readiness probe failed", + ], + in_order: true) + end + + def test_restart_successful_with_partial_availability + result = deploy_fixtures("slow-cloud", subset: %w(web-deploy-1.yml)) do |fixtures| + web = fixtures["web-deploy-1.yml"]["Deployment"].first + web["spec"]["strategy"]['rollingUpdate']['maxUnavailable'] = '50%' + container = web["spec"]["template"]["spec"]["containers"].first + container["readinessProbe"] = { + "exec" => { "command" => %w(sleep 5) }, + "timeoutSeconds" => 6, + } + end + assert_deploy_success(result) + + restart = build_restart_task + assert_restart_success(restart.perform(deployments: %w(web))) + + pods = kubeclient.get_pods(namespace: @namespace, label_selector: 'name=web,app=slow-cloud') + new_pods = pods.select do |pod| + pod.metadata.annotations&.dig(Krane::RestartTask::RESTART_TRIGGER_ANNOTATION) + end + assert(new_pods.length >= 1, "Expected at least one new pod, saw #{new_pods.length}") + + new_ready_pods = new_pods.select do |pod| + pod.status.phase == "Running" && + pod.status.conditions.any? { |condition| condition["type"] == "Ready" && condition["status"] == "True" } + end + assert_equal(1, new_ready_pods.length, "Expected exactly one new pod to be ready, saw #{new_ready_pods.length}") + + assert(fetch_restarted_at("web"), "restart annotation is present after the restart") + end + + def test_verify_result_false_succeeds + assert_deploy_success(deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "web.yml.erb", "redis.yml"], + render_erb: true)) + + refute(fetch_restarted_at("web"), "no restart annotation on fresh deployment") + refute(fetch_restarted_at("redis"), "no restart annotation on fresh deployment") + + restart = build_restart_task + assert_restart_success(restart.perform(verify_result: false)) + + assert_logs_match_all([ + "Configured to restart all workloads with the `shipit.shopify.io/restart` annotation", + "Triggered `Deployment/web` restart", + "Result: SUCCESS", + "Result verification is disabled for this task", + ], + in_order: true) + + assert(fetch_restarted_at("web"), "restart annotation is present after the restart") + refute(fetch_restarted_at("redis"), "no restart annotation on fresh deployment") + end + + def test_verify_result_false_fails_on_config_checks + restart = build_restart_task + assert_restart_failure(restart.perform(verify_result: false)) + assert_logs_match_all([ + "Configured to restart all workloads with the `shipit.shopify.io/restart` annotation", + "Result: FAILURE", + %r{No deployments, statefulsets, or daemonsets, with the `shipit\.shopify\.io/restart` annotation found}, + ], + in_order: true) + end + + def test_verify_result_false_succeeds_quickly_when_verification_would_timeout + success = deploy_fixtures("downward_api", + subset: ["configmap-data.yml", "web.yml.erb", "daemon_set.yml", "stateful_set.yml"], + render_erb: true) do |fixtures| + deployment = fixtures["web.yml.erb"]["Deployment"].first + deployment["spec"]["progressDeadlineSeconds"] = 60 + container = deployment["spec"]["template"]["spec"]["containers"].first + container["readinessProbe"] = { + "failureThreshold" => 1, + "periodSeconds" => 1, + "initialDelaySeconds" => 0, + "exec" => { + "command" => [ + "/bin/sh", + "-c", + "test $(cat /etc/podinfo/annotations | grep -s kubectl.kubernetes.io/restartedAt -c) -eq 0", + ], + }, + } + end + assert_deploy_success(success) + + restart = build_restart_task + restart.perform!(deployments: %w(web), statefulsets: %w(stateful-busybox), daemonsets: %w(ds-app), + verify_result: false) + + assert_logs_match_all([ + "Triggered `Deployment/web` restart", + "Triggered `StatefulSet/stateful-busybox` restart", + "Triggered `DaemonSet/ds-app` restart", + "Result: SUCCESS", + "Result verification is disabled for this task", + ], + in_order: true) + end + + private + + def build_restart_task + Krane::RestartTask.new( + context: KubeclientHelper::TEST_CONTEXT, + namespace: @namespace, + logger: logger + ) + end + + def fetch_restarted_at(name, kind: :deployment) + resource = case kind + when :deployment + apps_v1_kubeclient.get_deployment(name, @namespace) + when :statefulset + apps_v1_kubeclient.get_stateful_set(name, @namespace) + when :daemonset + apps_v1_kubeclient.get_daemon_set(name, @namespace) + end + resource.spec.template.metadata.annotations&.dig(Krane::RestartTask::RESTART_TRIGGER_ANNOTATION) + end +end diff --git a/test/integration/runner_task_test.rb b/test/integration/runner_task_test.rb new file mode 100644 index 000000000..cdcf1748b --- /dev/null +++ b/test/integration/runner_task_test.rb @@ -0,0 +1,317 @@ +# frozen_string_literal: true +require 'integration_test_helper' + +class RunnerTaskTest < Krane::IntegrationTest + include TaskRunnerTestHelper + + def test_run_without_verify_result_succeeds_as_soon_as_pod_is_successfully_created + deploy_unschedulable_template + + task_runner = build_task_runner + assert_nil(task_runner.pod_name) + result = task_runner.run(**run_params(verify_result: false)) + assert_task_run_success(result) + + assert_logs_match_all([ + /Creating pod 'task-runner-\w+'/, + "Pod creation succeeded", + "Result: SUCCESS", + "Result verification is disabled for this task", + "The following status was observed immediately after pod creation:", + %r{Pod/task-runner-\w+\s+(Pending|Running)}, + ], in_order: true) + + pods = kubeclient.get_pods(namespace: @namespace) + assert_equal(1, pods.length, "Expected 1 pod to exist, found #{pods.length}") + assert_equal(task_runner.pod_name, pods.first.metadata.name, "Pod name should be available after run") + end + + def test_run_global_timeout_with_global_timeout + deploy_task_template + timeout = 5 # seconds + + task_runner = build_task_runner(global_timeout: timeout) + result = task_runner.run(**run_params(log_lines: timeout * 10, log_interval: 1)) + assert_task_run_failure(result, :timed_out) + + assert_logs_match_all([ + "Result: TIMED OUT", + "Timed out waiting for 1 resource to run", + %r{Pod/task-runner-\w+: GLOBAL WATCH TIMEOUT \(#{timeout} seconds\)}, + /Final status\: (Pending|Running)/, + ], in_order: true) + end + + def test_run_with_verify_result_failure + deploy_task_template + + task_runner = build_task_runner + result = task_runner.run(**run_params.merge(arguments: ["/not/a/command"])) + assert_task_run_failure(result) + + assert_logs_match_all([ + "Streaming logs", + "/bin/sh: /not/a/command: not found", + %r{Pod/task-runner-\w+ failed to run after \d+.\ds}, + "Result: FAILURE", + "Pod status: Failed", + ], in_order: true) + refute_logs_match("Logs: None found") + + pods = kubeclient.get_pods(namespace: @namespace) + assert_equal(1, pods.length, "Expected 1 pod to exist, found #{pods.length}") + assert_equal(task_runner.pod_name, pods.first.metadata.name, "Pod name should be available after run") + end + + def test_run_with_verify_result_success + deploy_task_template + + task_runner = build_task_runner + assert_nil(task_runner.pod_name) + result = task_runner.run(**run_params(log_lines: 8, log_interval: 0.25)) + assert_task_run_success(result) + + assert_logs_match_all([ + "Initializing task", + /Using namespace 'k8sdeploy-test-run-with-verify-result-success-\w+' in context '[\w-]+'/, + "Using template 'hello-cloud-template-runner'", + "Running pod", + /Creating pod 'task-runner-\w+'/, + "Pod creation succeeded", + "Streaming logs", + "Line 1", + "Line 8", + "Result: SUCCESS", + %r{Pod/task-runner-\w+\s+Succeeded}, + ]) + pods = kubeclient.get_pods(namespace: @namespace) + assert_equal(1, pods.length, "Expected 1 pod to exist, found #{pods.length}") + assert_equal(task_runner.pod_name, pods.first.metadata.name, "Pod name should be available after run") + end + + def test_run_with_verify_result_fails_quickly_if_the_pod_is_deleted_out_of_band + deploy_task_template + + task_runner = build_task_runner + deleter_thread = Thread.new do + loop do + if task_runner.pod_name.present? + begin + kubeclient.delete_pod(task_runner.pod_name, @namespace) + break + rescue Kubeclient::ResourceNotFoundError + sleep(0.1) + retry + end + end + sleep(0.1) + end + end + deleter_thread.abort_on_exception = true + + result = task_runner.run(**run_params(log_lines: 20, log_interval: 1)) + assert_task_run_failure(result) + + assert_logs_match_all([ + "Pod creation succeeded", + "Result: FAILURE", + /Pod status\: (Terminating|Disappeared)/, + ]) + ensure + deleter_thread&.kill + end + + def test_run_with_verify_result_neither_misses_nor_duplicates_logs_across_pollings + deploy_task_template + task_runner = build_task_runner + result = task_runner.run(**run_params(log_lines: 5_000, log_interval: 0.0005)) + assert_task_run_success(result) + + logging_assertion do |all_logs| + nums_printed = all_logs.scan(/Line (\d+)$/).flatten + + first_num_printed = nums_printed[0].to_i + expected_nums = (first_num_printed..5_000).map(&:to_s) + missing_nums = expected_nums - nums_printed.uniq + assert(missing_nums.empty?, "Some lines were not streamed: #{missing_nums}") + + num_lines_duplicated = nums_printed.length - nums_printed.uniq.length + assert(num_lines_duplicated.zero?, "#{num_lines_duplicated} lines were duplicated") + end + end + + def test_run_with_bad_restart_policy + deploy_task_template do |fixtures| + fixtures["template-runner.yml"]["PodTemplate"].first["template"]["spec"]["restartPolicy"] = "OnFailure" + end + + task_runner = build_task_runner + assert_task_run_success(task_runner.run(**run_params)) + + assert_logs_match_all([ + "Phase 1: Initializing task", + "Using template 'hello-cloud-template-runner'", + "Changed Pod RestartPolicy from 'OnFailure' to 'Never'. Disable result verification to use 'OnFailure'.", + "Phase 2: Running pod", + ]) + end + + def test_run_bang_raises_exceptions_as_well_as_printing_failure + deploy_task_template + + task_runner = build_task_runner + assert_raises(Krane::FatalDeploymentError) do + task_runner.run!(**run_params.merge(arguments: ["/not/a/command"])) + end + + assert_logs_match_all([ + "Streaming logs", + "/bin/sh: /not/a/command: not found", + %r{Pod/task-runner-\w+ failed to run after \d+.\ds}, + "Result: FAILURE", + "Pod status: Failed", + ], in_order: true) + + pods = kubeclient.get_pods(namespace: @namespace) + assert_equal(1, pods.length, "Expected 1 pod to exist, found #{pods.length}") + end + + def test_run_fails_if_context_is_invalid + task_runner = build_task_runner(context: "unknown") + assert_task_run_failure(task_runner.run(**run_params)) + + assert_logs_match_all([ + "Initializing task", + "Validating configuration", + "Result: FAILURE", + "Configuration invalid", + "Context unknown missing from your kubeconfig file(s)", + ], in_order: true) + end + + def test_run_fails_if_namespace_is_missing + task_runner = build_task_runner(ns: "missing") + assert_task_run_failure(task_runner.run(**run_params)) + + assert_logs_match_all([ + "Initializing task", + "Validating configuration", + "Result: FAILURE", + "Configuration invalid", + "Could not find Namespace:", + ], in_order: true) + end + + def test_run_fails_if_template_is_blank + task_runner = build_task_runner + result = task_runner.run(template: '', + command: ['/bin/sh', '-c'], + arguments: nil, + env_vars: ["MY_CUSTOM_VARIABLE=MITTENS"]) + assert_task_run_failure(result) + + assert_logs_match_all([ + "Initializing task", + "Validating configuration", + "Result: FAILURE", + "Configuration invalid", + "Task template name can't be nil", + ], in_order: true) + end + + def test_run_bang_fails_if_template_is_invalid + task_runner = build_task_runner + assert_raises(Krane::TaskConfigurationError) do + task_runner.run!(template: '', + command: ['/bin/sh', '-c'], + arguments: nil, + env_vars: ["MY_CUSTOM_VARIABLE=MITTENS"]) + end + end + + def test_run_with_template_missing + task_runner = build_task_runner + assert_task_run_failure(task_runner.run(**run_params)) + message = "Pod template `hello-cloud-template-runner` not found in namespace `#{@namespace}`, " \ + "context `#{KubeclientHelper::TEST_CONTEXT}`" + assert_logs_match_all([ + "Result: FAILURE", + message, + ], in_order: true) + + assert_raises_message(Krane::RunnerTask::TaskTemplateMissingError, message) do + task_runner.run!(**run_params) + end + end + + def test_run_with_pod_spec_template_missing + deploy_task_template do |fixtures| + template = fixtures["template-runner.yml"]["PodTemplate"].first["template"] + template["spec"]["containers"].first["name"] = "bad-name" + end + + task_runner = build_task_runner + assert_task_run_failure(task_runner.run(**run_params)) + message = "Pod spec does not contain a template container called 'task-runner'" + + assert_raises_message(Krane::TaskConfigurationError, message) do + task_runner.run!(**run_params) + end + + assert_logs_match_all([ + "Result: FAILURE", + message, + ], in_order: true) + end + + def test_run_adds_env_vars_provided_to_the_task_container + deploy_task_template + + task_runner = build_task_runner + result = task_runner.run( + template: 'hello-cloud-template-runner', + command: ['/bin/sh', '-c'], + arguments: ['echo "The value is: $MY_CUSTOM_VARIABLE"'], + env_vars: ["MY_CUSTOM_VARIABLE=MITTENS"] + ) + assert_task_run_success(result) + + assert_logs_match_all([ + "Streaming logs", + "The value is: MITTENS", + ], in_order: true) + end + + def test_run_adds_custom_image_tag_provided_to_the_task_container + deploy_task_template + + task_runner = build_task_runner + result = task_runner.run( + template: 'hello-cloud-template-runner', + command: ['/bin/sh', '-c'], + arguments: ['echo "Hello World"'], + image_tag: 'latest' + ) + assert_task_run_success(result) + + pods = kubeclient.get_pods(namespace: @namespace) + assert_equal(1, pods.length, "Expected 1 pod to exist, found #{pods.length}") + container = pods.first.spec.containers.find { |cont| cont.name == 'task-runner' } + assert_equal('busybox:latest', container.image, "Container image should have been upadted") + end + + private + + def deploy_unschedulable_template + deploy_task_template do |fixtures| + way_too_fat = { + "requests" => { + "cpu" => 1000, + "memory" => "100Gi", + }, + } + template = fixtures["template-runner.yml"]["PodTemplate"].first["template"] + template["spec"]["containers"].first["resources"] = way_too_fat + end + end +end diff --git a/test/integration/task_config_validator_test.rb b/test/integration/task_config_validator_test.rb new file mode 100644 index 000000000..50c1df8dd --- /dev/null +++ b/test/integration/task_config_validator_test.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true +require 'integration_test_helper' + +class TaskConfigValidatorTest < Krane::IntegrationTest + def test_valid_configuration + assert_predicate(validator(context: KubeclientHelper::TEST_CONTEXT, namespace: 'default'), :valid?) + end + + def test_only_is_respected + assert_predicate(validator(only: []), :valid?) + end + + def test_invalid_kubeconfig + bad_file = "/IM_NOT_A_REAL_FILE.yml" + builder = Krane::KubeclientBuilder.new(kubeconfig: bad_file) + assert_match("Kubeconfig not found at #{bad_file}", + validator(kubeclient_builder: builder, only: [:validate_kubeconfig]).errors.join("\n")) + end + + def test_context_does_not_exists_in_kubeconfig + fake_context = "fake-context" + assert_match(/Context #{fake_context} missing from your kubeconfig file/, + validator(context: fake_context).errors.join("\n")) + end + + def test_context_not_reachable + fake_context = "fake-context" + assert_match(/Something went wrong connecting to #{fake_context}/, + validator(context: fake_context, only: [:validate_context_reachable]).errors.join("\n")) + end + + def test_namespace_does_not_exists + assert_match(/Could not find Namespace: test-namespace in Context: #{KubeclientHelper::TEST_CONTEXT}/, + validator(context: KubeclientHelper::TEST_CONTEXT).errors.join("\n")) + end + + def test_invalid_server_version + old_min_version = Krane::MIN_KUBE_VERSION + new_min_version = "99999" + Krane.send(:remove_const, :MIN_KUBE_VERSION) + Krane.const_set(:MIN_KUBE_VERSION, new_min_version) + validator(context: KubeclientHelper::TEST_CONTEXT, namespace: 'default', logger: @logger).valid? + assert_logs_match_all([ + "Minimum cluster version requirement of #{new_min_version} not met.", + ]) + ensure + Krane.const_set(:MIN_KUBE_VERSION, old_min_version) + end + + private + + def validator(context: nil, namespace: nil, logger: nil, kubeclient_builder: nil, only: nil) + context ||= "test-context" + namespace ||= "test-namespace" + config = task_config(context: context, namespace: namespace, logger: logger) + kubectl = Krane::Kubectl.new(task_config: config, log_failure_by_default: true) + kubeclient_builder ||= Krane::KubeclientBuilder.new + Krane::TaskConfigValidator.new(config, kubectl, kubeclient_builder, only: only) + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb index e8f20898f..370a9b79f 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true -# require 'pry' +require 'pry' if ENV["COVERAGE"] require 'simplecov'