diff --git a/boilerplate/flyte/end2end/Makefile b/boilerplate/flyte/end2end/Makefile index b0eb945..7f080e9 100644 --- a/boilerplate/flyte/end2end/Makefile +++ b/boilerplate/flyte/end2end/Makefile @@ -5,8 +5,9 @@ .PHONY: end2end_execute end2end_execute: - ./boilerplate/flyte/end2end/end2end.sh - + # TODO: These arguments could come from environment variables + ./boilerplate/flyte/end2end/end2end.sh ./boilerplate/flyte/end2end/functional-test.config --return_non_zero_on_failure + .PHONY: k8s_integration_execute k8s_integration_execute: - echo "pass" \ No newline at end of file + echo "pass" diff --git a/boilerplate/flyte/end2end/end2end.sh b/boilerplate/flyte/end2end/end2end.sh index 60109c6..736e3bb 100755 --- a/boilerplate/flyte/end2end/end2end.sh +++ b/boilerplate/flyte/end2end/end2end.sh @@ -4,16 +4,13 @@ # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY: # # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst - set -e -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" +CONFIG_FILE=$1; shift +EXTRA_FLAGS=( "$@" ) -OUT="${DIR}/tmp" -rm -rf ${OUT} -git clone https://github.com/flyteorg/flyte.git "${OUT}" +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -pushd ${OUT} +LATEST_VERSION=$(curl --silent "https://api.github.com/repos/flyteorg/flytesnacks/releases/latest" | jq -r .tag_name) -make end2end_execute -popd +python ./boilerplate/flyte/end2end/run-tests.py $LATEST_VERSION P0,P1 $CONFIG_FILE ${EXTRA_FLAGS[@]} diff --git a/boilerplate/flyte/end2end/functional-test.config b/boilerplate/flyte/end2end/functional-test.config new file mode 100644 index 0000000..f2a867a --- /dev/null +++ b/boilerplate/flyte/end2end/functional-test.config @@ -0,0 +1,3 @@ +[platform] +url=127.0.0.1:30081 +insecure=True diff --git a/boilerplate/flyte/end2end/run-tests.py b/boilerplate/flyte/end2end/run-tests.py new file mode 100755 index 0000000..dbae8a7 --- /dev/null +++ b/boilerplate/flyte/end2end/run-tests.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 + +import click +import json +import sys +import time +import traceback +import requests +from typing import List, Mapping, Tuple, Dict +from flytekit.remote import FlyteRemote +from flytekit.models.core.execution import WorkflowExecutionPhase +from flytekit.configuration import Config, ImageConfig, SerializationSettings + + +WAIT_TIME = 10 +MAX_ATTEMPTS = 60 + +# This dictionary maps the names found in the flytesnacks manifest to a list of workflow names and +# inputs. This is so we can progressively cover all priorities in the original flytesnacks manifest, +# starting with "core". +FLYTESNACKS_WORKFLOW_GROUPS: Mapping[str, List[Tuple[str, dict]]] = { + "core": [ + ("core.control_flow.chain_tasks.chain_tasks_wf", {}), + ("core.control_flow.dynamics.wf", {"s1": "Pear", "s2": "Earth"}), + ("core.control_flow.map_task.my_map_workflow", {"a": [1, 2, 3, 4, 5]}), + # Workflows that use nested executions cannot be launched via flyteremote. + # This issue is being tracked in https://github.com/flyteorg/flyte/issues/1482. + # ("core.control_flow.run_conditions.multiplier", {"my_input": 0.5}), + # ("core.control_flow.run_conditions.multiplier_2", {"my_input": 10}), + # ("core.control_flow.run_conditions.multiplier_3", {"my_input": 5}), + # ("core.control_flow.run_conditions.basic_boolean_wf", {"seed": 5}), + # ("core.control_flow.run_conditions.bool_input_wf", {"b": True}), + # ("core.control_flow.run_conditions.nested_conditions", {"my_input": 0.4}), + # ("core.control_flow.run_conditions.consume_outputs", {"my_input": 0.4, "seed": 7}), + # ("core.control_flow.run_merge_sort.merge_sort", {"numbers": [5, 4, 3, 2, 1], "count": 5}), + ("core.control_flow.subworkflows.parent_wf", {"a": 3}), + ("core.control_flow.subworkflows.nested_parent_wf", {"a": 3}), + ("core.flyte_basics.basic_workflow.my_wf", {"a": 50, "b": "hello"}), + # Getting a 403 for the wikipedia image + # ("core.flyte_basics.files.rotate_one_workflow", {"in_image": "https://upload.wikimedia.org/wikipedia/commons/d/d2/Julia_set_%28C_%3D_0.285%2C_0.01%29.jpg"}), + ("core.flyte_basics.folders.download_and_rotate", {}), + ("core.flyte_basics.hello_world.my_wf", {}), + ("core.flyte_basics.lp.my_wf", {"val": 4}), + ("core.flyte_basics.lp.go_greet", {"day_of_week": "5", "number": 3, "am": True}), + ("core.flyte_basics.named_outputs.my_wf", {}), + # # Getting a 403 for the wikipedia image + # # ("core.flyte_basics.reference_task.wf", {}), + ("core.type_system.custom_objects.wf", {"x": 10, "y": 20}), + # Enums are not supported in flyteremote + # ("core.type_system.enums.enum_wf", {"c": "red"}), + ("core.type_system.schema.df_wf", {"a": 42}), + ("core.type_system.typed_schema.wf", {}), + ("my.imperative.workflow.example", {"in1": "hello", "in2": "foo"}), + ], +} + + +def run_launch_plan(remote, version, workflow_name, inputs): + print(f"Fetching workflow={workflow_name} and version={version}") + lp = remote.fetch_workflow(name=workflow_name, version=version) + return remote.execute(lp, inputs=inputs, wait=False) + + +def schedule_workflow_group( + tag: str, + workflow_group: str, + remote: FlyteRemote, + terminate_workflow_on_failure: bool, +) -> bool: + """ + Schedule all workflows executions and return True if all executions succeed, otherwise + return False. + """ + workflows = FLYTESNACKS_WORKFLOW_GROUPS.get(workflow_group, []) + + launch_plans = [ + run_launch_plan(remote, tag, workflow[0], workflow[1]) for workflow in workflows + ] + + # Wait for all launch plans to finish + attempt = 0 + while attempt == 0 or ( + not all([lp.is_complete for lp in launch_plans]) and attempt < MAX_ATTEMPTS + ): + attempt += 1 + print( + f"Not all executions finished yet. Sleeping for some time, will check again in {WAIT_TIME}s" + ) + time.sleep(WAIT_TIME) + # Need to sync to refresh status of executions + for lp in launch_plans: + print(f"About to sync execution_id={lp.id.name}") + remote.sync(lp) + + # Report result of each launch plan + for lp in launch_plans: + print(lp) + + # Collect all failing launch plans + non_succeeded_lps = [ + lp + for lp in launch_plans + if lp.closure.phase != WorkflowExecutionPhase.SUCCEEDED + ] + + if len(non_succeeded_lps) == 0: + print("All executions succeeded.") + return True + + print("Failed executions:") + # Report failing cases + for lp in non_succeeded_lps: + print(f" workflow={lp.spec.launch_plan.name}, execution_id={lp.id.name}") + if terminate_workflow_on_failure: + remote.terminate(lp, "aborting execution scheduled in functional test") + return False + + +def valid(workflow_group): + """ + Return True if a workflow group is contained in FLYTESNACKS_WORKFLOW_GROUPS, + False otherwise. + """ + return workflow_group in FLYTESNACKS_WORKFLOW_GROUPS.keys() + + +def run( + flytesnacks_release_tag: str, + priorities: List[str], + config_file_path, + terminate_workflow_on_failure: bool, +) -> List[Dict[str, str]]: + remote = FlyteRemote( + Config.auto(config_file=config_file_path), + default_project="flytesnacks", + default_domain="development", + ) + + # For a given release tag and priority, this function filters the workflow groups from the flytesnacks + # manifest file. For example, for the release tag "v0.2.224" and the priority "P0" it returns [ "core" ]. + manifest_url = "https://raw.githubusercontent.com/flyteorg/flytesnacks/" \ + f"{flytesnacks_release_tag}/cookbook/flyte_tests_manifest.json" + r = requests.get(manifest_url) + parsed_manifest = r.json() + + workflow_groups = [ + group["name"] for group in parsed_manifest if group["priority"] in priorities + ] + results = [] + for workflow_group in workflow_groups: + if not valid(workflow_group): + results.append( + { + "label": workflow_group, + "status": "coming soon", + "color": "grey", + } + ) + continue + + try: + workflows_succeeded = schedule_workflow_group( + flytesnacks_release_tag, + workflow_group, + remote, + terminate_workflow_on_failure, + ) + except Exception: + print(traceback.format_exc()) + workflows_succeeded = False + + if workflows_succeeded: + background_color = "green" + status = "passing" + else: + background_color = "red" + status = "failing" + + # Workflow groups can be only in one of three states: + # 1. passing: this indicates all the workflow executions for that workflow group + # executed successfully + # 2. failing: this state indicates that at least one execution failed in that + # workflow group + # 3. coming soon: this state is used to indicate that the workflow group was not + # implemented yet. + # + # Each state has a corresponding status and color to be used in the badge for that + # workflow group. + result = { + "label": workflow_group, + "status": status, + "color": background_color, + } + results.append(result) + return results + + +@click.command() +@click.option( + "--return_non_zero_on_failure", + default=False, + is_flag=True, + help="Return a non-zero exit status if any workflow fails", +) +@click.option( + "--terminate_workflow_on_failure", + default=False, + is_flag=True, + help="Abort failing workflows upon exit", +) +@click.argument("flytesnacks_release_tag") +@click.argument("priorities") +@click.argument("config_file") +def cli( + flytesnacks_release_tag, + priorities, + config_file, + return_non_zero_on_failure, + terminate_workflow_on_failure, +): + print(f"return_non_zero_on_failure={return_non_zero_on_failure}") + results = run( + flytesnacks_release_tag, priorities, config_file, terminate_workflow_on_failure + ) + + # Write a json object in its own line describing the result of this run to stdout + print(f"Result of run:\n{json.dumps(results)}") + + # Return a non-zero exit code if core fails + if return_non_zero_on_failure: + for result in results: + if result["status"] not in ("passing", "coming soon"): + sys.exit(1) + + +if __name__ == "__main__": + cli() diff --git a/boilerplate/flyte/golang_support_tools/go.mod b/boilerplate/flyte/golang_support_tools/go.mod index 307398c..0486588 100644 --- a/boilerplate/flyte/golang_support_tools/go.mod +++ b/boilerplate/flyte/golang_support_tools/go.mod @@ -4,7 +4,7 @@ go 1.17 require ( github.com/alvaroloes/enumer v1.1.2 - github.com/flyteorg/flytestdlib v0.4.7 + github.com/flyteorg/flytestdlib v0.4.16 github.com/golangci/golangci-lint v1.38.0 github.com/pseudomuto/protoc-gen-doc v1.4.1 github.com/vektra/mockery v0.0.0-20181123154057-e78b021dcbb5 @@ -14,7 +14,10 @@ require ( 4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a // indirect cloud.google.com/go v0.75.0 // indirect cloud.google.com/go/storage v1.12.0 // indirect - github.com/Azure/azure-sdk-for-go v51.0.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go v62.3.0+incompatible // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.17 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.10 // indirect @@ -46,6 +49,7 @@ require ( github.com/esimonov/ifshort v1.0.1 // indirect github.com/fatih/color v1.10.0 // indirect github.com/fatih/structtag v1.2.0 // indirect + github.com/flyteorg/stow v0.3.1 // indirect github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/fzipp/gocyclo v0.3.1 // indirect @@ -62,6 +66,7 @@ require ( github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gofrs/flock v0.8.0 // indirect + github.com/gofrs/uuid v4.2.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.4.3 // indirect @@ -82,7 +87,6 @@ require ( github.com/gostaticanalysis/comment v1.4.1 // indirect github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect - github.com/graymeta/stow v0.2.7 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/huandu/xstrings v1.0.0 // indirect github.com/imdario/mergo v0.3.5 // indirect @@ -135,7 +139,6 @@ require ( github.com/ryancurrah/gomodguard v1.2.0 // indirect github.com/ryanrolds/sqlclosecheck v0.3.0 // indirect github.com/sanposhiho/wastedassign v0.1.3 // indirect - github.com/satori/go.uuid v1.2.0 // indirect github.com/securego/gosec/v2 v2.6.1 // indirect github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c // indirect github.com/sirupsen/logrus v1.8.0 // indirect @@ -163,10 +166,10 @@ require ( golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect golang.org/x/mod v0.4.1 // indirect - golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect golang.org/x/oauth2 v0.0.0-20210126194326-f9ce19ea3013 // indirect - golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect - golang.org/x/text v0.3.5 // indirect + golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect golang.org/x/tools v0.1.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/boilerplate/flyte/golang_support_tools/go.sum b/boilerplate/flyte/golang_support_tools/go.sum index a62010d..92515a8 100644 --- a/boilerplate/flyte/golang_support_tools/go.sum +++ b/boilerplate/flyte/golang_support_tools/go.sum @@ -41,9 +41,14 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v32.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v51.0.0+incompatible h1:p7blnyJSjJqf5jflHbSGhIhEpXIgIFmYZNg5uwqweso= -github.com/Azure/azure-sdk-for-go v51.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v62.3.0+incompatible h1:Ctfsn9UoA/BB4HMYQlbPPgNXdX0tZ4tmb85+KFb2+RE= +github.com/Azure/azure-sdk-for-go v62.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= @@ -177,8 +182,9 @@ github.com/denis-tingajkin/go-header v0.4.2 h1:jEeSF4sdv8/3cT/WY8AgDHUoItNSoEZ7q github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCFFnBUn4RN0nRcs1LJA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -210,10 +216,12 @@ github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/flyteorg/flytestdlib v0.4.7 h1:SMPPXI3j/MjP7D2fqaR+lPQkTrqYS7xZbwsgJI2F8SU= -github.com/flyteorg/flytestdlib v0.4.7/go.mod h1:fv1ar34LJLMTaf0tbfetisLykUlARi7rP+NQTUn6QQs= +github.com/flyteorg/flytestdlib v0.4.16 h1:r4dCPUOqoE9xCAhOw9KDB7O6cBoCxyEtepIWYcj93H0= +github.com/flyteorg/flytestdlib v0.4.16/go.mod h1:WA5Y4hrcgD0ybGOKJVOQ4sP8q7NLRV+S5SWOlH0axgM= github.com/flyteorg/protoc-gen-doc v1.4.2 h1:Otw0F+RHaPQ8XlpzhLLgjsCMcrAIcMO01Zh+ALe3rrE= github.com/flyteorg/protoc-gen-doc v1.4.2/go.mod h1:exDTOVwqpp30eV/EDPFLZy3Pwr2sn6hBC1WIYH/UbIg= +github.com/flyteorg/stow v0.3.1 h1:cBMbWl03Gsy5KoA5mutUYTuYpqtT7Pb8+ANGCLnmFEs= +github.com/flyteorg/stow v0.3.1/go.mod h1:HBld7ud0i4khMHwJjkO8v+NSP7ddKa/ruhf4I8fliaA= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -276,6 +284,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gofrs/flock v0.8.0 h1:MSdYClljsF3PbENUUEx85nkWfJSGfzYI9yEBZOJz6CY= github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -397,8 +407,6 @@ github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5 h github.com/gostaticanalysis/forcetypeassert v0.0.0-20200621232751-01d4955beaa5/go.mod h1:qZEedyP/sY1lTGV1uJ3VhWZ2mqag3IkWsDHVbplHXak= github.com/gostaticanalysis/nilerr v0.1.1 h1:ThE+hJP0fEp4zWLkWHWcRyI2Od0p7DlgYG3Uqrmrcpk= github.com/gostaticanalysis/nilerr v0.1.1/go.mod h1:wZYb6YI5YAxxq0i1+VJbY0s2YONW0HU0GPE3+5PWN4A= -github.com/graymeta/stow v0.2.7 h1:b31cB1Ylw/388sYSZxnmpjT2QxC21AaQ8fRnUtE13b4= -github.com/graymeta/stow v0.2.7/go.mod h1:JAs139Zr29qfsecy7b+h9DRsWXbFbsd7LCrbCDYI84k= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -484,6 +492,8 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -577,7 +587,6 @@ github.com/nbutton23/zxcvbn-go v0.0.0-20201221231540-e56b841a3c88/go.mod h1:KSVJ github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/exhaustive v0.1.0 h1:kVlMw8h2LHPMGUVqUj6230oQjjTMFjwcZrnkhXzFfl8= github.com/nishanths/exhaustive v0.1.0/go.mod h1:S1j9110vxV1ECdCudXRkeMnFQ/DQk9ajLT0Uf2MYZQQ= @@ -698,8 +707,6 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sanposhiho/wastedassign v0.1.3 h1:qIMpTh4NGZYRbFJ+DSpLoVn8F4SLciX2afRvXPefC7w= github.com/sanposhiho/wastedassign v0.1.3/go.mod h1:LGpq5Hsv74QaqM47WtIsRSF/ik9kqk07kchgv66tLVE= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/securego/gosec/v2 v2.6.1 h1:+KCw+uz16FYfFyJ/A5aU6uP7mnrL+j1TbDnk1yN+8R0= github.com/securego/gosec/v2 v2.6.1/go.mod h1:I76p3NTHBXsGhybUW+cEQ692q2Vp+A0Z6ZLzDIZy+Ao= @@ -907,14 +914,16 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -993,8 +1002,8 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1003,8 +1012,9 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1225,8 +1235,9 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/boilerplate/flyte/golang_test_targets/download_tooling.sh b/boilerplate/flyte/golang_test_targets/download_tooling.sh index c0ab06b..02fb004 100755 --- a/boilerplate/flyte/golang_test_targets/download_tooling.sh +++ b/boilerplate/flyte/golang_test_targets/download_tooling.sh @@ -17,8 +17,8 @@ set -e # In the format of ":" or ":" if no cli tools=( "github.com/vektra/mockery/cmd/mockery" - "github.com/flyteorg/flytestdlib/cli/pflags" - "github.com/golangci/golangci-lint/cmd/golangci-lint" + "github.com/flyteorg/flytestdlib/cli/pflags@latest" + "github.com/golangci/golangci-lint/cmd/golangci-lint@latest" "github.com/alvaroloes/enumer" "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc" ) diff --git a/cmd/root.go b/cmd/root.go index babd93c..6a5b4f0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,8 +9,7 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" - "github.com/aws/aws-sdk-go/aws/ec2metadata" + "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" "github.com/flyteorg/flytestdlib/config" @@ -78,30 +77,30 @@ func PollUntilTimeout(ctx context.Context, pollInterval, timeout time.Duration, return wait.PollUntil(pollInterval, condition, childCtx.Done()) } -func checkAWSCreds() error { - cfg, err := session.NewSession(&aws.Config{}) +func checkAWSCreds() (*credentials.Value, error) { + sess, err := session.NewSession(&aws.Config{}) if err != nil { - return err + return nil, err } - role := &ec2rolecreds.EC2RoleProvider{ - Client: ec2metadata.New(cfg), - } - creds, err := role.Retrieve() + // Determine the AWS credentials from the default credential chain + creds, err := sess.Config.Credentials.Get() if err != nil { - return err + return nil, err } if creds.AccessKeyID == "" || creds.SecretAccessKey == "" || creds.SessionToken == "" { - return fmt.Errorf("invalid data in credential fetch") + return nil, fmt.Errorf("invalid data in credential fetch") } - return nil + return &creds, nil } func waitForAWSCreds(ctx context.Context, timeout time.Duration) error { return PollUntilTimeout(ctx, time.Second*5, timeout, func() (bool, error) { - if err := checkAWSCreds(); err != nil { - logger.Errorf(ctx, "Failed to Get credentials.") + if creds, err := checkAWSCreds(); err != nil { + logger.Errorf(ctx, "failed to get AWS credentials: %s", err) return false, nil + } else if creds != nil { + logger.Infof(ctx, "found AWS credentials from provider: %s", creds.ProviderName) } return true, nil })