From 213ac3c87fde2ebf26882e1498650d264962ddf9 Mon Sep 17 00:00:00 2001 From: Becky Sweger Date: Fri, 5 Apr 2024 13:40:13 -0400 Subject: [PATCH 1/4] Add ruff code formatter to the project Add ruff, a corresponding config file, and the formatting changes that it suggested --- .ruff.toml | 6 ++++ pulumi/__init__.py | 0 pulumi/__main__.py | 9 ++--- pulumi/hubs/hub_setup.py | 7 ++-- pulumi/hubs/iam.py | 72 ++++++++++++++++------------------------ pulumi/hubs/s3.py | 45 ++++++++++--------------- pulumi/requirements.txt | 6 +++- 7 files changed, 66 insertions(+), 79 deletions(-) create mode 100644 .ruff.toml create mode 100644 pulumi/__init__.py diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..7539319 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,6 @@ +line-length = 120 +#lint.extend-select = ['I'] + +[format] +# Like Black, use double quotes for strings. +quote-style = "double" \ No newline at end of file diff --git a/pulumi/__init__.py b/pulumi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pulumi/__main__.py b/pulumi/__main__.py index 1ad82b4..93a3dfc 100644 --- a/pulumi/__main__.py +++ b/pulumi/__main__.py @@ -4,13 +4,14 @@ from hubs.hub_setup import set_up_hub + def get_hubs() -> list[dict]: - '''Get the list of cloud-enabled hubs.''' - with open('hubs/hubs.yaml', 'r') as file: - hubs = yaml.safe_load(file).get('hubs') + """Get the list of cloud-enabled hubs.""" + with open("hubs/hubs.yaml", "r") as file: + hubs = yaml.safe_load(file).get("hubs") return hubs + hub_list = get_hubs() for hub in hub_list: set_up_hub(hub) - diff --git a/pulumi/hubs/hub_setup.py b/pulumi/hubs/hub_setup.py index cc8e485..69dcb40 100644 --- a/pulumi/hubs/hub_setup.py +++ b/pulumi/hubs/hub_setup.py @@ -3,13 +3,14 @@ from hubs.s3 import create_s3_infrastructure from hubs.iam import create_iam_infrastructure -def set_up_hub(hub_info:dict): - ''' + +def set_up_hub(hub_info: dict): + """ Create all AWS instrastructure needed for a Hubverse hub. For simplicity, this demo uses the hub name as the bucket name, though the new cloud section of a hub's admin.json config allows a different bucket name. - ''' + """ create_s3_infrastructure(hub_info) create_iam_infrastructure(hub_info) diff --git a/pulumi/hubs/iam.py b/pulumi/hubs/iam.py index 0dbb4d8..82c47bc 100644 --- a/pulumi/hubs/iam.py +++ b/pulumi/hubs/iam.py @@ -5,7 +5,7 @@ def create_trust_policy(org: str, repo: str): """Create the trust policy that will used with the IAM role for Github Actions.""" # retrieve information about the hubverse account's OIDC github provider - oidc_github = aws.iam.get_open_id_connect_provider(url='https://token.actions.githubusercontent.com') + oidc_github = aws.iam.get_open_id_connect_provider(url="https://token.actions.githubusercontent.com") # Create the policy that defines who will be allowed to assume # a role using the OIDC provider we create for GitHub Actions. @@ -16,23 +16,23 @@ def create_trust_policy(org: str, repo: str): statements=[ aws.iam.GetPolicyDocumentStatementArgs( actions=["sts:AssumeRoleWithWebIdentity"], - principals=[aws.iam.GetPolicyDocumentStatementPrincipalArgs( - type="Federated", - #identifiers=[f"arn:aws:iam::{aws.get_caller_identity().account_id}:oidc-provider/token.actions.githubusercontent.com"] - identifiers=[oidc_github.arn] - )], + principals=[ + aws.iam.GetPolicyDocumentStatementPrincipalArgs( + type="Federated", + # identifiers=[f"arn:aws:iam::{aws.get_caller_identity().account_id}:oidc-provider/token.actions.githubusercontent.com"] + identifiers=[oidc_github.arn], + ) + ], conditions=[ aws.iam.GetPolicyDocumentStatementConditionArgs( - test="StringEquals", - variable=f'{oidc_github.url}:aud', - values=['sts.amazonaws.com'] + test="StringEquals", variable=f"{oidc_github.url}:aud", values=["sts.amazonaws.com"] ), aws.iam.GetPolicyDocumentStatementConditionArgs( test="StringEquals", - variable=f'{oidc_github.url}:sub', - values=[f'repo:{org}/{repo}:ref:refs/heads/main'] - ) - ] + variable=f"{oidc_github.url}:sub", + values=[f"repo:{org}/{repo}:ref:refs/heads/main"], + ), + ], ) ] ) @@ -46,16 +46,15 @@ def create_github_role(hub_name: str, policy_document): github_role = aws.iam.Role( name=hub_name, resource_name=hub_name, - description='The role assumed by CI/CD for writing data to S3.', - tags={'hub': hub_name}, - assume_role_policy=policy_document + description="The role assumed by CI/CD for writing data to S3.", + tags={"hub": hub_name}, + assume_role_policy=policy_document, ) return github_role def create_bucket_write_policy(hub_name: str): - # Create a policy that allows put operations to the hub's # S3 bucket. This will then be attached to the IAM role that # GitHub actions assumes. @@ -65,53 +64,40 @@ def create_bucket_write_policy(hub_name: str): actions=[ "s3:ListBucket", ], - resources=[ - f'arn:aws:s3:::{hub_name}' - ], + resources=[f"arn:aws:s3:::{hub_name}"], ), aws.iam.GetPolicyDocumentStatementArgs( - actions=[ - "s3:PutObject", - "s3:PutObjectAcl", - "s3:GetObject", - "s3:GetObjectAcl", - "s3:DeleteObject" - - ], - resources=[ - f'arn:aws:s3:::{hub_name}/*' - ], - ) + actions=["s3:PutObject", "s3:PutObjectAcl", "s3:GetObject", "s3:GetObjectAcl", "s3:DeleteObject"], + resources=[f"arn:aws:s3:::{hub_name}/*"], + ), ] ) - bucket_write_policy_name = f'{hub_name}-write-bucket-policy' + bucket_write_policy_name = f"{hub_name}-write-bucket-policy" bucket_write_policy = aws.iam.Policy( name=bucket_write_policy_name, resource_name=bucket_write_policy_name, - description=f'Policy attached to {hub_name} role. It allows writing to the {hub_name} S3 bucket', + description=f"Policy attached to {hub_name} role. It allows writing to the {hub_name} S3 bucket", policy=s3_write_policy.json, - tags={'hub': hub_name}, + tags={"hub": hub_name}, ) return bucket_write_policy + def attach_bucket_write_policy(hub_name: str, github_role, bucket_write_policy): """Attach the S3 write policy to the role that Github Actions assumes.""" # Update the role we created for Github Actions by attaching the # policy that allows writes to the hub's S3 bucket - aws.iam.RolePolicyAttachment( - resource_name=hub_name, - role=github_role.name, - policy_arn=bucket_write_policy.id - ) + aws.iam.RolePolicyAttachment(resource_name=hub_name, role=github_role.name, policy_arn=bucket_write_policy.id) + def create_iam_infrastructure(hub_info: dict): """Create the IAM infrastructure needed for a hub.""" - org = hub_info['org'] - repo = hub_info['repo'] - hub = hub_info['hub'] + org = hub_info["org"] + repo = hub_info["repo"] + hub = hub_info["hub"] trust_policy = create_trust_policy(org, repo) github_role = create_github_role(hub, trust_policy) s3_write_policy = create_bucket_write_policy(hub) diff --git a/pulumi/hubs/s3.py b/pulumi/hubs/s3.py index b9b6990..ddddb83 100644 --- a/pulumi/hubs/s3.py +++ b/pulumi/hubs/s3.py @@ -1,17 +1,14 @@ import pulumi_aws as aws from pulumi import ResourceOptions + def create_bucket(hub_name: str) -> aws.s3.Bucket: """ Create a new S3 bucket for a hub. (for simplicity, in this demo we're setting the bucket name to the hub name) """ - hub_bucket = aws.s3.Bucket( - hub_name, - bucket=hub_name, - tags={'hub': hub_name} - ) + hub_bucket = aws.s3.Bucket(hub_name, bucket=hub_name, tags={"hub": hub_name}) return hub_bucket @@ -27,14 +24,14 @@ def make_bucket_public(bucket: aws.s3.Bucket, bucket_name: str): # By default, new S3 buckets do not allow public access. Updating # those settings will allow us to create a bucket policy for public access. hub_bucket_public_access_block = aws.s3.BucketPublicAccessBlock( - #resource_name=f'{bucket_name}-read-bucket-policy', - #resource_name=bucket.bucket.apply(lambda bucket: f'{bucket}-public-access-block'), - resource_name=f'{bucket_name}-public-access-block', + # resource_name=f'{bucket_name}-read-bucket-policy', + # resource_name=bucket.bucket.apply(lambda bucket: f'{bucket}-public-access-block'), + resource_name=f"{bucket_name}-public-access-block", bucket=bucket.id, block_public_acls=True, ignore_public_acls=True, block_public_policy=False, - restrict_public_buckets=False + restrict_public_buckets=False, ) # Create an S3 policy that allows public read access. @@ -45,44 +42,36 @@ def make_bucket_public(bucket: aws.s3.Bucket, bucket_name: str): actions=[ "s3:GetObject", ], - principals=[ - aws.iam.GetPolicyDocumentStatementPrincipalArgs(type="*", identifiers=["*"]) - ], - resources=[ - f'arn:aws:s3:::{bucket_name}/*' - ], + principals=[aws.iam.GetPolicyDocumentStatementPrincipalArgs(type="*", identifiers=["*"])], + resources=[f"arn:aws:s3:::{bucket_name}/*"], ), aws.iam.GetPolicyDocumentStatementArgs( sid="PublicListBucket", actions=[ "s3:ListBucket", ], - principals=[ - aws.iam.GetPolicyDocumentStatementPrincipalArgs(type="*", identifiers=["*"]) - ], - resources=[ - f'arn:aws:s3:::{bucket_name}' - ], - ) + principals=[aws.iam.GetPolicyDocumentStatementPrincipalArgs(type="*", identifiers=["*"])], + resources=[f"arn:aws:s3:::{bucket_name}"], + ), ] ) # Apply the public read policy to the bucket. aws.s3.BucketPolicy( - #resource_name=f'{bucket_name}-read-bucket-policy', - #resource_name=Output.concat(bucket.id, '-read-bucket-policy'), - resource_name=f'{bucket_name}-read-bucket-policy', + # resource_name=f'{bucket_name}-read-bucket-policy', + # resource_name=Output.concat(bucket.id, '-read-bucket-policy'), + resource_name=f"{bucket_name}-read-bucket-policy", bucket=bucket.id, policy=s3_policy.json, # The dependency below ensures that the bucket's public access block has - # already been updated to allow public access. Otherwise, trying to + # already been updated to allow public access. Otherwise, trying to # apply the "everyone can read" policy will throw a 403. - opts=ResourceOptions(depends_on=[hub_bucket_public_access_block]) + opts=ResourceOptions(depends_on=[hub_bucket_public_access_block]), ) def create_s3_infrastructure(hub_info: dict) -> aws.s3.Bucket: - hub_name = hub_info['hub'] + hub_name = hub_info["hub"] bucket = create_bucket(hub_name) make_bucket_public(bucket, hub_name) return bucket diff --git a/pulumi/requirements.txt b/pulumi/requirements.txt index 16907c4..46abe46 100644 --- a/pulumi/requirements.txt +++ b/pulumi/requirements.txt @@ -1,3 +1,7 @@ pulumi>=3.0.0,<4.0.0 pulumi-aws>=6.0.2,<7.0.0 -PyYAML>=6.0.1 \ No newline at end of file +PyYAML>=6.0.1 + +# dev requirements +# TODO: separate dev requirements +ruff>=0.3.5 From d0cb3f0480c8683d878d0166fa990882b18a1aa4 Mon Sep 17 00:00:00 2001 From: Becky Sweger Date: Fri, 5 Apr 2024 13:57:35 -0400 Subject: [PATCH 2/4] Add mypy static type checker to the project Note that mypy doesn't play especially well with pulumi. Problematic lines are ignored so the type checker will pass. --- pulumi/hubs/s3.py | 4 ++-- pulumi/requirements.txt | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pulumi/hubs/s3.py b/pulumi/hubs/s3.py index ddddb83..29273f1 100644 --- a/pulumi/hubs/s3.py +++ b/pulumi/hubs/s3.py @@ -1,5 +1,5 @@ import pulumi_aws as aws -from pulumi import ResourceOptions +from pulumi import ResourceOptions # type: ignore def create_bucket(hub_name: str) -> aws.s3.Bucket: @@ -66,7 +66,7 @@ def make_bucket_public(bucket: aws.s3.Bucket, bucket_name: str): # The dependency below ensures that the bucket's public access block has # already been updated to allow public access. Otherwise, trying to # apply the "everyone can read" policy will throw a 403. - opts=ResourceOptions(depends_on=[hub_bucket_public_access_block]), + opts=ResourceOptions(depends_on=[hub_bucket_public_access_block]), # type: ignore ) diff --git a/pulumi/requirements.txt b/pulumi/requirements.txt index 46abe46..10ac576 100644 --- a/pulumi/requirements.txt +++ b/pulumi/requirements.txt @@ -4,4 +4,6 @@ PyYAML>=6.0.1 # dev requirements # TODO: separate dev requirements +mypy>=1.9.0 ruff>=0.3.5 +types-PyYAML>=6.0.12 From 196b65e8867c73172631922c51ade77a1b95cb5c Mon Sep 17 00:00:00 2001 From: Becky Sweger Date: Fri, 5 Apr 2024 14:04:05 -0400 Subject: [PATCH 3/4] Add a workflow to run code checks This is the first iteration of code check workflow for the hubverse-infrastructure repo. It's obviously missing a job step to run tests, which is a future piece of "getting ready for production" work. --- .github/workflows/run-code-checks.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/run-code-checks.yaml diff --git a/.github/workflows/run-code-checks.yaml b/.github/workflows/run-code-checks.yaml new file mode 100644 index 0000000..72bc2da --- /dev/null +++ b/.github/workflows/run-code-checks.yaml @@ -0,0 +1,23 @@ +name: run-code-checks +run-name: ${{ github.action }} triggered by ${{ github.event_name }} from ${{ github.actor }} + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + run-checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + cache: true + - name: Install dependencies + run: pip install -r pulumi/requirements.txt + - name: lint + run: ruff check + - name: type check + run: mypy . --ignore-missing-imports + + From 766c5401b37527703fc463398079478789a313f2 Mon Sep 17 00:00:00 2001 From: Becky Sweger Date: Fri, 5 Apr 2024 14:05:43 -0400 Subject: [PATCH 4/4] Add a pre-commit configuration file Adding a pre-commit config that mirrors the new "code check" GitHub action. This is optional, but can be used by devs who have pre-commit as part of their workflow. https://pre-commit.com/ --- .pre-commit-config.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..68068a6 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: check-yaml + args: [--allow-multiple-documents] + - id: detect-aws-credentials + - id: detect-private-key +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.11 + hooks: + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format +- repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v1.8.0' + hooks: + - id: mypy + args: [--ignore-missing-imports] \ No newline at end of file