diff --git a/conda_smithy/anaconda_token_rotation.py b/conda_smithy/anaconda_token_rotation.py new file mode 100644 index 000000000..a29d75233 --- /dev/null +++ b/conda_smithy/anaconda_token_rotation.py @@ -0,0 +1,388 @@ +"""This module updates/rotates anaconda/binstar tokens. + +The correct way to use this module is to call its functions via the command +line utility. The relevant one is + + conda-smithy update-anaconda-token + +Note that if you are using appveyor, you will need to push the changes to the +conda-forge.yml in your feedstock to GitHub. +""" +import os +import sys +from contextlib import redirect_stderr, redirect_stdout + +import requests + +from .utils import update_conda_forge_config + + +def _get_anaconda_token(): + """use this helper to enable easier patching for tests""" + try: + from .ci_register import anaconda_token + + return anaconda_token + except ImportError: + raise RuntimeError( + "You must have the anaconda token defined to do token rotation!" + ) + + +def rotate_anaconda_token( + user, + project, + feedstock_directory, + drone=True, + circle=True, + travis=True, + azure=True, + appveyor=True, + token_name="BINSTAR_TOKEN", +): + """Rotate the anaconda (binstar) token used by the CI providers + + All exceptions are swallowed and stdout/stderr from this function is + redirected to `/dev/null`. Sanitized error messages are + displayed at the end. + + If you need to debug this function, define `DEBUG_ANACONDA_TOKENS` in + your environment before calling this function. + """ + # we are swallong all of the logs below, so we do a test import here + # to generate the proper errors for missing tokens + # note that these imports cover all providers + from .ci_register import travis_endpoint # noqa + from .azure_ci_utils import default_config # noqa + + anaconda_token = _get_anaconda_token() + + # capture stdout, stderr and suppress all exceptions so we don't + # spill tokens + failed = False + err_msg = None + with open(os.devnull, "w") as fp: + if "DEBUG_ANACONDA_TOKENS" in os.environ: + fpo = sys.stdout + fpe = sys.stderr + else: + fpo = fp + fpe = fp + + with redirect_stdout(fpo), redirect_stderr(fpe): + try: + if circle: + try: + rotate_token_in_circle( + user, project, anaconda_token, token_name + ) + except Exception as e: + if "DEBUG_ANACONDA_TOKENS" in os.environ: + raise e + else: + err_msg = ( + "Failed to rotate token for %s/%s" + " on circle!" + ) % (user, project) + failed = True + raise RuntimeError(err_msg) + + if drone: + try: + rotate_token_in_drone( + user, project, anaconda_token, token_name + ) + except Exception as e: + if "DEBUG_ANACONDA_TOKENS" in os.environ: + raise e + else: + err_msg = ( + "Failed to rotate token for %s/%s" " on drone!" + ) % (user, project) + failed = True + raise RuntimeError(err_msg) + + if travis: + try: + rotate_token_in_travis( + user, + project, + feedstock_directory, + anaconda_token, + token_name, + ) + except Exception as e: + if "DEBUG_ANACONDA_TOKENS" in os.environ: + raise e + else: + err_msg = ( + "Failed to rotate token for %s/%s" + " on travis!" + ) % (user, project) + failed = True + raise RuntimeError(err_msg) + + if azure: + try: + rotate_token_in_azure( + user, project, anaconda_token, token_name + ) + except Exception as e: + if "DEBUG_ANACONDA_TOKENS" in os.environ: + raise e + else: + err_msg = ( + "Failed to rotate token for %s/%s" " on azure!" + ) % (user, project) + failed = True + raise RuntimeError(err_msg) + + if appveyor: + try: + rotate_token_in_appveyor( + feedstock_directory, anaconda_token, token_name + ) + except Exception as e: + if "DEBUG_ANACONDA_TOKENS" in os.environ: + raise e + else: + err_msg = ( + "Failed to rotate token for %s/%s" + " on appveyor!" + ) % (user, project) + failed = True + raise RuntimeError(err_msg) + + except Exception as e: + if "DEBUG_ANACONDA_TOKENS" in os.environ: + raise e + failed = True + if failed: + if err_msg: + raise RuntimeError(err_msg) + else: + raise RuntimeError( + ( + "Rotating the feedstock token in proviers for %s/%s failed!" + " Try the command locally with DEBUG_ANACONDA_TOKENS" + " defined in the environment to investigate!" + ) + % (user, project) + ) + + +def rotate_token_in_circle(user, project, binstar_token, token_name): + from .ci_register import circle_token + + url_template = ( + "https://circleci.com/api/v1.1/project/github/{user}/{project}/envvar{extra}?" + "circle-token={token}" + ) + + r = requests.get( + url_template.format( + token=circle_token, user=user, project=project, extra="" + ) + ) + if r.status_code != 200: + r.raise_for_status() + + have_binstar_token = False + for evar in r.json(): + if evar["name"] == token_name: + have_binstar_token = True + + if have_binstar_token: + r = requests.delete( + url_template.format( + token=circle_token, + user=user, + project=project, + extra="/%s" % token_name, + ) + ) + if r.status_code != 200: + r.raise_for_status() + + data = {"name": token_name, "value": binstar_token} + response = requests.post( + url_template.format( + token=circle_token, user=user, project=project, extra="" + ), + data, + ) + if response.status_code != 201: + raise ValueError(response) + + +def rotate_token_in_drone(user, project, binstar_token, token_name): + from .ci_register import drone_session + + session = drone_session() + + r = session.get(f"/api/repos/{user}/{project}/secrets") + r.raise_for_status() + have_binstar_token = False + for secret in r.json(): + if token_name == secret["name"]: + have_binstar_token = True + + if have_binstar_token: + r = session.patch( + f"/api/repos/{user}/{project}/secrets/{token_name}", + json={"data": binstar_token, "pull_request": False}, + ) + r.raise_for_status() + else: + response = session.post( + f"/api/repos/{user}/{project}/secrets", + json={ + "name": token_name, + "data": binstar_token, + "pull_request": False, + }, + ) + if response.status_code != 200: + response.raise_for_status() + + +def rotate_token_in_travis( + user, project, feedstock_directory, binstar_token, token_name +): + """update the binstar token in travis.""" + from .ci_register import ( + travis_endpoint, + travis_headers, + travis_get_repo_info, + ) + + headers = travis_headers() + + repo_info = travis_get_repo_info(user, project) + repo_id = repo_info["id"] + + r = requests.get( + "{}/repo/{repo_id}/env_vars".format(travis_endpoint, repo_id=repo_id), + headers=headers, + ) + if r.status_code != 200: + r.raise_for_status() + + have_binstar_token = False + ev_id = None + for ev in r.json()["env_vars"]: + if ev["name"] == token_name: + have_binstar_token = True + ev_id = ev["id"] + + data = { + "env_var.name": token_name, + "env_var.value": binstar_token, + "env_var.public": "false", + } + + if have_binstar_token: + r = requests.patch( + "{}/repo/{repo_id}/env_var/{ev_id}".format( + travis_endpoint, repo_id=repo_id, ev_id=ev_id, + ), + headers=headers, + json=data, + ) + r.raise_for_status() + else: + r = requests.post( + "{}/repo/{repo_id}/env_vars".format( + travis_endpoint, repo_id=repo_id + ), + headers=headers, + json=data, + ) + if r.status_code != 201: + r.raise_for_status() + + # we remove the token in the conda-forge.yml since on travis the + # encrypted values override any value we put in the API + with update_conda_forge_config(feedstock_directory) as code: + if ( + "travis" in code + and "secure" in code["travis"] + and token_name in code["travis"]["secure"] + ): + del code["travis"]["secure"][token_name] + + if len(code["travis"]["secure"]) == 0: + del code["travis"]["secure"] + + if len(code["travis"]) == 0: + del code["travis"] + + print( + "An old value of the variable %s for travis was found in the " + "conda-forge.yml. You may need to rerender this feedstock to " + "use the new value since encrypted secrets inserted in travis.yml " + "files override those set in the UI/API!" + ) + + +def rotate_token_in_azure(user, project, binstar_token, token_name): + from .azure_ci_utils import build_client, get_default_build_definition + from .azure_ci_utils import default_config as config + from vsts.build.v4_1.models import BuildDefinitionVariable + + bclient = build_client() + + existing_definitions = bclient.get_definitions( + project=config.project_name, name=project + ) + if existing_definitions: + assert len(existing_definitions) == 1 + ed = existing_definitions[0] + else: + raise RuntimeError( + "Cannot add %s to a repo that is not already registerd on azure CI!" + % token_name + ) + + ed = bclient.get_definition(ed.id, project=config.project_name) + + if not hasattr(ed, "variables") or ed.variables is None: + variables = {} + else: + variables = ed.variables + + variables[token_name] = BuildDefinitionVariable( + allow_override=False, is_secret=True, value=binstar_token, + ) + + build_definition = get_default_build_definition( + user, + project, + config=config, + variables=variables, + id=ed.id, + revision=ed.revision, + ) + + bclient.update_definition( + definition=build_definition, + definition_id=ed.id, + project=ed.project.name, + ) + + +def rotate_token_in_appveyor(feedstock_directory, binstar_token, token_name): + from .ci_register import appveyor_token + + headers = {"Authorization": "Bearer {}".format(appveyor_token)} + url = "https://ci.appveyor.com/api/account/encrypt" + response = requests.post( + url, headers=headers, data={"plainValue": binstar_token} + ) + if response.status_code != 200: + raise ValueError(response) + + with update_conda_forge_config(feedstock_directory) as code: + code.setdefault("appveyor", {}).setdefault("secure", {})[ + token_name + ] = response.content.decode("utf-8") diff --git a/conda_smithy/cli.py b/conda_smithy/cli.py index 086b65bfd..903894084 100644 --- a/conda_smithy/cli.py +++ b/conda_smithy/cli.py @@ -52,7 +52,7 @@ class Subcommand(object): def __init__(self, parser, help=None): subcommand_parser = parser.add_parser( - self.subcommand, help=help, aliases=self.aliases + self.subcommand, help=help, description=help, aliases=self.aliases ) subcommand_parser.set_defaults(subcommand_func=self) self.subcommand_parser = subcommand_parser @@ -525,7 +525,8 @@ def __call__(self, args): def main(): parser = argparse.ArgumentParser( - "a tool to help create, administer and manage feedstocks." + prog="conda smithy", + description="a tool to help create, administer and manage feedstocks.", ) subparser = parser.add_subparsers() # TODO: Consider allowing plugins/extensions using entry_points. @@ -606,7 +607,12 @@ def __init__(self, parser): super(RegisterFeedstockToken, self).__init__( parser, "Register the feedstock token w/ the CI services for builds and " - "with the token registry.", + "with the token registry. \n\n" + "All exceptions are swallowed and stdout/stderr from this function is" + "redirected to `/dev/null`. Sanitized error messages are" + "displayed at the end.\n\n" + "If you need to debug this function, define `DEBUG_ANACONDA_TOKENS` in" + "your environment before calling this function.", ) scp = self.subcommand_parser scp.add_argument( @@ -687,5 +693,88 @@ def __call__(self, args): print("Successfully registered the feedstock token!") +class UpdateAnacondaToken(Subcommand): + subcommand = "update-anaconda-token" + aliases = [ + "rotate-anaconda-token", + "update-binstar-token", + "rotate-binstar-token", + ] + + def __init__(self, parser): + super(UpdateAnacondaToken, self).__init__( + parser, + "Update the anaconda/binstar token used for package uploads.\n\n" + "All exceptions are swallowed and stdout/stderr from this function is" + "redirected to `/dev/null`. Sanitized error messages are" + "displayed at the end.\n\n" + "If you need to debug this function, define `DEBUG_ANACONDA_TOKENS` in" + "your environment before calling this function.", + ) + scp = self.subcommand_parser + scp.add_argument( + "--feedstock_directory", + default=os.getcwd(), + help="The directory of the feedstock git repository.", + ) + scp.add_argument( + "--token_name", + default="BINSTAR_TOKEN", + help="The name of the environment variable you'd like to hold the token.", + ) + group = scp.add_mutually_exclusive_group() + group.add_argument("--user", help="github username of the repo") + group.add_argument( + "--organization", + default="conda-forge", + help="github organization of the repo", + ) + for ci in [ + "Azure", + "Travis", + "Circle", + "Drone", + "Appveyor", + ]: + scp.add_argument( + "--without-{}".format(ci.lower()), + dest=ci.lower(), + action="store_false", + help="If set, the token on {} will be not changed.".format(ci), + ) + default = {ci.lower(): True} + scp.set_defaults(**default) + + def __call__(self, args): + from conda_smithy.anaconda_token_rotation import rotate_anaconda_token + + owner = args.user or args.organization + repo = os.path.basename(os.path.abspath(args.feedstock_directory)) + + print( + "Updating the anaconda/binstar token. Can take up to ~30 seconds." + ) + + # do all providers first + rotate_anaconda_token( + owner, + repo, + args.feedstock_directory, + drone=args.drone, + circle=args.circle, + travis=args.travis, + azure=args.azure, + appveyor=args.appveyor, + token_name=args.token_name, + ) + + print("Successfully updated the anaconda/binstar token!") + if args.appveyor: + print( + "Appveyor tokens are stored in the repo so you must commit the " + "local changes and push them before the new token will be used!" + ) + + if __name__ == "__main__": main() diff --git a/conda_smithy/feedstock_tokens.py b/conda_smithy/feedstock_tokens.py index dd92b139e..a1a0d4509 100644 --- a/conda_smithy/feedstock_tokens.py +++ b/conda_smithy/feedstock_tokens.py @@ -618,6 +618,8 @@ def add_feedstock_token_to_azure(user, project, feedstock_token, clobber): "Cannot add FEEDSTOCK_TOKEN to a repo that is not already registerd on azure CI!" ) + ed = bclient.get_definition(ed.id, project=config.project_name) + if not hasattr(ed, "variables") or ed.variables is None: variables = {} else: diff --git a/news/rotate.rst b/news/rotate.rst new file mode 100644 index 000000000..b1cb42145 --- /dev/null +++ b/news/rotate.rst @@ -0,0 +1,23 @@ +**Added:** + +* Added code to rotate anaconda tokens. + +**Changed:** + +* + +**Deprecated:** + +* + +**Removed:** + +* + +**Fixed:** + +* Fixed bug in feedstock token registration that deleted other secrets from azure. + +**Security:** + +* diff --git a/tests/test_anaconda_token_rotation.py b/tests/test_anaconda_token_rotation.py new file mode 100644 index 000000000..42ad7e03c --- /dev/null +++ b/tests/test_anaconda_token_rotation.py @@ -0,0 +1,174 @@ +from unittest import mock + +import pytest + +from conda_smithy.anaconda_token_rotation import rotate_anaconda_token + + +@pytest.mark.parametrize("appveyor", [True, False]) +@pytest.mark.parametrize("drone", [True, False]) +@pytest.mark.parametrize("circle", [True, False]) +@pytest.mark.parametrize("azure", [True, False]) +@pytest.mark.parametrize("travis", [True, False]) +@mock.patch("conda_smithy.anaconda_token_rotation._get_anaconda_token") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_appveyor") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_drone") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_circle") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_travis") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_azure") +def test_rotate_anaconda_token( + azure_mock, + travis_mock, + circle_mock, + drone_mock, + appveyor_mock, + get_ac_token, + appveyor, + drone, + circle, + travis, + azure, +): + user = "foo" + project = "bar" + + anaconda_token = "abc123" + get_ac_token.return_value = anaconda_token + + rotate_anaconda_token( + user, + project, + "abc", + drone=drone, + circle=circle, + travis=travis, + azure=azure, + appveyor=appveyor, + token_name="MY_FANCY_TOKEN", + ) + + if drone: + drone_mock.assert_called_once_with( + user, project, anaconda_token, "MY_FANCY_TOKEN" + ) + else: + drone_mock.assert_not_called() + + if circle: + circle_mock.assert_called_once_with( + user, project, anaconda_token, "MY_FANCY_TOKEN" + ) + else: + circle_mock.assert_not_called() + + if travis: + travis_mock.assert_called_once_with( + user, project, "abc", anaconda_token, "MY_FANCY_TOKEN" + ) + else: + travis_mock.assert_not_called() + + if azure: + azure_mock.assert_called_once_with( + user, project, anaconda_token, "MY_FANCY_TOKEN" + ) + else: + azure_mock.assert_not_called() + + if appveyor: + appveyor_mock.assert_called_once_with( + "abc", anaconda_token, "MY_FANCY_TOKEN" + ) + else: + appveyor_mock.assert_not_called() + + +@pytest.mark.parametrize("appveyor", [True, False]) +@pytest.mark.parametrize("drone", [True, False]) +@pytest.mark.parametrize("circle", [True, False]) +@pytest.mark.parametrize("azure", [True, False]) +@pytest.mark.parametrize("travis", [True, False]) +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_appveyor") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_drone") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_circle") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_travis") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_azure") +def test_rotate_anaconda_token_notoken( + azure_mock, + travis_mock, + circle_mock, + drone_mock, + appveyor_mock, + appveyor, + drone, + circle, + travis, + azure, + monkeypatch, +): + user = "foo" + project = "bar" + + with pytest.raises(RuntimeError) as e: + rotate_anaconda_token( + user, + project, + None, + drone=drone, + circle=circle, + travis=travis, + azure=azure, + appveyor=appveyor, + ) + + assert "anaconda token" in str(e.value) + + drone_mock.assert_not_called() + circle_mock.assert_not_called() + travis_mock.assert_not_called() + azure_mock.assert_not_called() + appveyor_mock.assert_not_called() + + +@pytest.mark.parametrize( + "provider", ["drone", "circle", "travis", "azure", "appveyor"] +) +@mock.patch("conda_smithy.anaconda_token_rotation._get_anaconda_token") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_appveyor") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_drone") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_circle") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_travis") +@mock.patch("conda_smithy.anaconda_token_rotation.rotate_token_in_azure") +def test_rotate_anaconda_token_provider_error( + azure_mock, + travis_mock, + circle_mock, + drone_mock, + appveyor_mock, + get_ac_token, + provider, +): + user = "foo" + project = "bar" + + anaconda_token = "abc123" + get_ac_token.return_value = anaconda_token + + user = "foo" + project = "bar-feedstock" + + if provider == "drone": + drone_mock.side_effect = ValueError("blah") + if provider == "circle": + circle_mock.side_effect = ValueError("blah") + if provider == "travis": + travis_mock.side_effect = ValueError("blah") + if provider == "azure": + azure_mock.side_effect = ValueError("blah") + if provider == "appveyor": + appveyor_mock.side_effect = ValueError("blah") + + with pytest.raises(RuntimeError) as e: + rotate_anaconda_token(user, project, None) + + assert "on %s" % provider in str(e.value)