Merge pull request #29 from Infectious-Disease-Modeling-Hubs/bsweger/…

…add-code-checks

Add code checks to hubverse-infrastructure

.github/workflows/run-code-checks.yaml

@@ -0,0 +1,23 @@
+name: run-code-checks
+run-name: ${{ github.action }} triggered by ${{ github.event_name }} from ${{ github.actor }}
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  run-checks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+            cache: true
+      - name: Install dependencies
+        run: pip install -r pulumi/requirements.txt
+      - name: lint
+        run: ruff check
+      - name: type check
+        run: mypy . --ignore-missing-imports
+
+

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.5.0
+  hooks:
+  - id: trailing-whitespace
+  - id: check-yaml
+    args: [--allow-multiple-documents]
+  - id: detect-aws-credentials
+  - id: detect-private-key
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.1.11
+  hooks:
+    # Run the linter.
+    - id: ruff
+      args: [ --fix ]
+    # Run the formatter.
+    - id: ruff-format
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: 'v1.8.0'
+  hooks:
+  - id: mypy
+    args: [--ignore-missing-imports]

.ruff.toml

@@ -0,0 +1,6 @@
+line-length = 120
+#lint.extend-select = ['I']
+
+[format]
+# Like Black, use double quotes for strings.
+quote-style = "double"

pulumi/__main__.py

@@ -4,13 +4,14 @@
 
 from hubs.hub_setup import set_up_hub
 
+
 def get_hubs() -> list[dict]:
-    '''Get the list of cloud-enabled hubs.'''
-    with open('hubs/hubs.yaml', 'r') as file:
-        hubs = yaml.safe_load(file).get('hubs')
+    """Get the list of cloud-enabled hubs."""
+    with open("hubs/hubs.yaml", "r") as file:
+        hubs = yaml.safe_load(file).get("hubs")
     return hubs
 
+
 hub_list = get_hubs()
 for hub in hub_list:
     set_up_hub(hub)
-

pulumi/hubs/hub_setup.py

@@ -3,13 +3,14 @@
 from hubs.s3 import create_s3_infrastructure
 from hubs.iam import create_iam_infrastructure
 
-def set_up_hub(hub_info:dict):
-    '''
+
+def set_up_hub(hub_info: dict):
+    """
     Create all AWS instrastructure needed for a Hubverse hub.
     For simplicity, this demo uses the hub name as the bucket name,
     though the new cloud section of a hub's admin.json config allows
     a different bucket name.
-    '''
+    """
 
     create_s3_infrastructure(hub_info)
     create_iam_infrastructure(hub_info)

pulumi/hubs/iam.py

@@ -5,7 +5,7 @@ def create_trust_policy(org: str, repo: str):
     """Create the trust policy that will used with the IAM role for Github Actions."""
 
     # retrieve information about the hubverse account's OIDC github provider
-    oidc_github = aws.iam.get_open_id_connect_provider(url='https://token.actions.githubusercontent.com')
+    oidc_github = aws.iam.get_open_id_connect_provider(url="https://token.actions.githubusercontent.com")
 
     # Create the policy that defines who will be allowed to assume
     # a role using the OIDC provider we create for GitHub Actions.
@@ -16,23 +16,23 @@ def create_trust_policy(org: str, repo: str):
         statements=[
             aws.iam.GetPolicyDocumentStatementArgs(
                 actions=["sts:AssumeRoleWithWebIdentity"],
-                principals=[aws.iam.GetPolicyDocumentStatementPrincipalArgs(
-                    type="Federated",
-                    #identifiers=[f"arn:aws:iam::{aws.get_caller_identity().account_id}:oidc-provider/token.actions.githubusercontent.com"]
-                    identifiers=[oidc_github.arn]
-                )],
+                principals=[
+                    aws.iam.GetPolicyDocumentStatementPrincipalArgs(
+                        type="Federated",
+                        # identifiers=[f"arn:aws:iam::{aws.get_caller_identity().account_id}:oidc-provider/token.actions.githubusercontent.com"]
+                        identifiers=[oidc_github.arn],
+                    )
+                ],
                 conditions=[
                     aws.iam.GetPolicyDocumentStatementConditionArgs(
-                       test="StringEquals",
-                       variable=f'{oidc_github.url}:aud',
-                       values=['sts.amazonaws.com']
+                        test="StringEquals", variable=f"{oidc_github.url}:aud", values=["sts.amazonaws.com"]
                     ),
                     aws.iam.GetPolicyDocumentStatementConditionArgs(
                         test="StringEquals",
-                        variable=f'{oidc_github.url}:sub',
-                        values=[f'repo:{org}/{repo}:ref:refs/heads/main']
-                    ) 
-                ]
+                        variable=f"{oidc_github.url}:sub",
+                        values=[f"repo:{org}/{repo}:ref:refs/heads/main"],
+                    ),
+                ],
             )
         ]
     )
@@ -46,16 +46,15 @@ def create_github_role(hub_name: str, policy_document):
     github_role = aws.iam.Role(
         name=hub_name,
         resource_name=hub_name,
-        description='The role assumed by CI/CD for writing data to S3.',
-        tags={'hub': hub_name},
-        assume_role_policy=policy_document
+        description="The role assumed by CI/CD for writing data to S3.",
+        tags={"hub": hub_name},
+        assume_role_policy=policy_document,
     )
 
     return github_role
 
 
 def create_bucket_write_policy(hub_name: str):
-
     # Create a policy that allows put operations to the hub's
     # S3 bucket. This will then be attached to the IAM role that
     # GitHub actions assumes.
@@ -65,53 +64,40 @@ def create_bucket_write_policy(hub_name: str):
                 actions=[
                     "s3:ListBucket",
                 ],
-                resources=[
-                    f'arn:aws:s3:::{hub_name}'
-                ],
+                resources=[f"arn:aws:s3:::{hub_name}"],
             ),
             aws.iam.GetPolicyDocumentStatementArgs(
-                actions=[
-                    "s3:PutObject",
-                    "s3:PutObjectAcl",
-                    "s3:GetObject",
-                    "s3:GetObjectAcl",
-                    "s3:DeleteObject"
-
-                ],
-                resources=[
-                    f'arn:aws:s3:::{hub_name}/*'
-                ],
-            )
+                actions=["s3:PutObject", "s3:PutObjectAcl", "s3:GetObject", "s3:GetObjectAcl", "s3:DeleteObject"],
+                resources=[f"arn:aws:s3:::{hub_name}/*"],
+            ),
         ]
     )
 
-    bucket_write_policy_name = f'{hub_name}-write-bucket-policy'
+    bucket_write_policy_name = f"{hub_name}-write-bucket-policy"
     bucket_write_policy = aws.iam.Policy(
         name=bucket_write_policy_name,
         resource_name=bucket_write_policy_name,
-        description=f'Policy attached to {hub_name} role. It allows writing to the {hub_name} S3 bucket',
+        description=f"Policy attached to {hub_name} role. It allows writing to the {hub_name} S3 bucket",
         policy=s3_write_policy.json,
-        tags={'hub': hub_name},
+        tags={"hub": hub_name},
     )
 
     return bucket_write_policy
 
+
 def attach_bucket_write_policy(hub_name: str, github_role, bucket_write_policy):
     """Attach the S3 write policy to the role that Github Actions assumes."""
 
     # Update the role we created for Github Actions by attaching the
     # policy that allows writes to the hub's S3 bucket
-    aws.iam.RolePolicyAttachment(
-        resource_name=hub_name,
-        role=github_role.name,
-        policy_arn=bucket_write_policy.id
-    )
+    aws.iam.RolePolicyAttachment(resource_name=hub_name, role=github_role.name, policy_arn=bucket_write_policy.id)
+
 
 def create_iam_infrastructure(hub_info: dict):
     """Create the IAM infrastructure needed for a hub."""
-    org = hub_info['org']
-    repo = hub_info['repo']
-    hub = hub_info['hub']
+    org = hub_info["org"]
+    repo = hub_info["repo"]
+    hub = hub_info["hub"]
     trust_policy = create_trust_policy(org, repo)
     github_role = create_github_role(hub, trust_policy)
     s3_write_policy = create_bucket_write_policy(hub)

pulumi/hubs/s3.py

@@ -1,17 +1,14 @@
 import pulumi_aws as aws
-from pulumi import ResourceOptions
+from pulumi import ResourceOptions  # type: ignore
+
 
 def create_bucket(hub_name: str) -> aws.s3.Bucket:
     """
     Create a new S3 bucket for a hub.
     (for simplicity, in this demo we're setting the bucket name to the hub name)
     """
 
-    hub_bucket = aws.s3.Bucket(
-        hub_name,
-        bucket=hub_name,
-        tags={'hub': hub_name}
-    )
+    hub_bucket = aws.s3.Bucket(hub_name, bucket=hub_name, tags={"hub": hub_name})
 
     return hub_bucket
 
@@ -27,14 +24,14 @@ def make_bucket_public(bucket: aws.s3.Bucket, bucket_name: str):
     # By default, new S3 buckets do not allow public access. Updating
     # those settings will allow us to create a bucket policy for public access.
     hub_bucket_public_access_block = aws.s3.BucketPublicAccessBlock(
-        #resource_name=f'{bucket_name}-read-bucket-policy',
-        #resource_name=bucket.bucket.apply(lambda bucket: f'{bucket}-public-access-block'),
-        resource_name=f'{bucket_name}-public-access-block',
+        # resource_name=f'{bucket_name}-read-bucket-policy',
+        # resource_name=bucket.bucket.apply(lambda bucket: f'{bucket}-public-access-block'),
+        resource_name=f"{bucket_name}-public-access-block",
         bucket=bucket.id,
         block_public_acls=True,
         ignore_public_acls=True,
         block_public_policy=False,
-        restrict_public_buckets=False
+        restrict_public_buckets=False,
     )
 
     # Create an S3 policy that allows public read access.
@@ -45,44 +42,36 @@ def make_bucket_public(bucket: aws.s3.Bucket, bucket_name: str):
                 actions=[
                     "s3:GetObject",
                 ],
-                principals=[
-                    aws.iam.GetPolicyDocumentStatementPrincipalArgs(type="*", identifiers=["*"])
-                ],
-                resources=[
-                    f'arn:aws:s3:::{bucket_name}/*'
-                ],
+                principals=[aws.iam.GetPolicyDocumentStatementPrincipalArgs(type="*", identifiers=["*"])],
+                resources=[f"arn:aws:s3:::{bucket_name}/*"],
             ),
             aws.iam.GetPolicyDocumentStatementArgs(
                 sid="PublicListBucket",
                 actions=[
                     "s3:ListBucket",
                 ],
-                principals=[
-                    aws.iam.GetPolicyDocumentStatementPrincipalArgs(type="*", identifiers=["*"])
-                ],
-                resources=[
-                    f'arn:aws:s3:::{bucket_name}'
-                ],
-            )
+                principals=[aws.iam.GetPolicyDocumentStatementPrincipalArgs(type="*", identifiers=["*"])],
+                resources=[f"arn:aws:s3:::{bucket_name}"],
+            ),
         ]
     )
 
     # Apply the public read policy to the bucket.
     aws.s3.BucketPolicy(
-        #resource_name=f'{bucket_name}-read-bucket-policy',
-        #resource_name=Output.concat(bucket.id, '-read-bucket-policy'),
-        resource_name=f'{bucket_name}-read-bucket-policy',
+        # resource_name=f'{bucket_name}-read-bucket-policy',
+        # resource_name=Output.concat(bucket.id, '-read-bucket-policy'),
+        resource_name=f"{bucket_name}-read-bucket-policy",
         bucket=bucket.id,
         policy=s3_policy.json,
         # The dependency below ensures that the bucket's public access block has
-        # already been updated to allow public access. Otherwise, trying to 
+        # already been updated to allow public access. Otherwise, trying to
         # apply the "everyone can read" policy will throw a 403.
-        opts=ResourceOptions(depends_on=[hub_bucket_public_access_block])
+        opts=ResourceOptions(depends_on=[hub_bucket_public_access_block]),  # type: ignore
     )
 
 
 def create_s3_infrastructure(hub_info: dict) -> aws.s3.Bucket:
-    hub_name =  hub_info['hub']
+    hub_name = hub_info["hub"]
     bucket = create_bucket(hub_name)
     make_bucket_public(bucket, hub_name)
     return bucket

pulumi/requirements.txt

@@ -1,3 +1,9 @@
 pulumi>=3.0.0,<4.0.0
 pulumi-aws>=6.0.2,<7.0.0
-PyYAML>=6.0.1
+PyYAML>=6.0.1
+
+# dev requirements
+# TODO: separate dev requirements
+mypy>=1.9.0
+ruff>=0.3.5
+types-PyYAML>=6.0.12