From b6a2732c292d49e82b304d37b39c10e7a05083f6 Mon Sep 17 00:00:00 2001 From: troyraen Date: Tue, 22 Mar 2022 16:15:40 -0400 Subject: [PATCH 01/17] add definitions for IAM roles and policies --- broker/iam/README.md | 186 ++++++++++++++++++++ broker/iam/policies/production_project.yaml | 8 + broker/iam/policies/testing_project.yaml | 8 + broker/iam/roles/developer_student.yaml | 7 + broker/iam/roles/user_public.yaml | 6 + 5 files changed, 215 insertions(+) create mode 100644 broker/iam/README.md create mode 100644 broker/iam/policies/production_project.yaml create mode 100644 broker/iam/policies/testing_project.yaml create mode 100644 broker/iam/roles/developer_student.yaml create mode 100644 broker/iam/roles/user_public.yaml diff --git a/broker/iam/README.md b/broker/iam/README.md new file mode 100644 index 00000000..5c7e0854 --- /dev/null +++ b/broker/iam/README.md @@ -0,0 +1,186 @@ +# IAM Resources + +Basic idea: + +We need to connect three things: resource(s), permission(s), and user(s). + +1. Create a Role - this is a collection of permissions and is a registered resource in GCP. +1. Create a Policy - this is a local file that defines a collection of bindings (bindings attach roles to members) +1. Attach the Policy to the Project, or to a specific Resource - access will be restricted this restricts access to the resource according to the policy. Policies can be attached to projects, and therefore apply to all resources in the project. + +See: +- [IAM overview](https://cloud.google.com/iam/docs/overview) +- [IAM predefined roles reference](https://cloud.google.com/iam/docs/understanding-roles#predefined) +- [IAM permissions reference](https://cloud.google.com/iam/docs/permissions-reference) + +## Setup + +Set GCP environment variables and authenticate yourself +For reference, our current projects are: + +- production project: `ardent-cycling-243415` +- testing project: `avid-heading-329016` + +```bash +# Set environment variables +# fill these in with your values; they are used throughout +export GOOGLE_CLOUD_PROJECT= +export GOOGLE_APPLICATION_CREDENTIALS= + +# Authenticate to use gcloud tools in this project +gcloud auth activate-service-account \ + --project="${GOOGLE_CLOUD_PROJECT}" \ + --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" +``` + +## Roles + +### Create the roles + +```bash +# To Do: +# get the organization id (not project id) +# for role_yaml in roles: +# read role_id from file, then: +role_id=developerStudent +role_yaml=roles/developer_student.yaml +gcloud iam roles create $role_id --project=$GOOGLE_CLOUD_PROJECT --file=$role_yaml +``` + +### Update a role + +(simpler in python so you don't have to mess with the etag fingerprint) + +```python +# this is an untested example from the following url +# https://cloud.google.com/iam/docs/creating-custom-roles +def edit_role(name, project, title, description, permissions, stage): + """Creates a role.""" + + # pylint: disable=no-member + role = service.projects().roles().patch( + name='projects/' + project + '/roles/' + name, + body={ + 'title': title, + 'description': description, + 'includedPermissions': permissions, + 'stage': stage + }).execute() + + print('Updated role: ' + role['name']) + return role +``` + +## Policies + +### Set the policy on the project + +This binds member-role pairs to every resource in the project. (See below to bind to a specific resource.) + +We must download the current policy, update the file and use it to set a new policy. +Setting a policy overrides the current policy. + +```bash +# To Do: +# for policy_yaml in policies: +policy_yaml="policies/studev.yaml" +gcloud projects set-iam-policy $GOOGLE_CLOUD_PROJECT $policy_yaml +``` + +### Set a policy on a specific resource + +(simpler in python so you don't have to mess with the etag fingerprint) + +```python +# this is an example from +# https://cloud.google.com/pubsub/docs/access-control#python_2 +from google.cloud import pubsub_v1 + +# TODO(developer): Choose an existing subscription. +# project_id = "your-project-id" +# subscription_id = "your-subscription-id" + +client = pubsub_v1.SubscriberClient() +subscription_path = client.subscription_path(project_id, subscription_id) + +policy = client.get_iam_policy(request={"resource": subscription_path}) + +# Add all users as viewers. +policy.bindings.add(role="roles/pubsub.viewer", members=["domain:google.com"]) + +# Add a group as an editor. +policy.bindings.add(role="roles/editor", members=["group:cloud-logs@google.com"]) + +# Set the policy +policy = client.set_iam_policy( + request={"resource": subscription_path, "policy": policy} +) + +print("IAM policy for subscription {} set: {}".format(subscription_id, policy)) + +client.close() +``` + +## Onboard a new developer + +### Developer instructions + +Complete the instructions in the initial setup tutorial +(docs/source/broker/run-a-broker-instance/initial-setup.rst). + +You will need to do this in conjunction with a Pitt-Google project manager so that they can grant you the necessary permissions. + +You will need to provide the manager with both your Google account email address (e.g., Gmail address) and your service account name (which you will choose during setup). + +### Project manager instructions + +Make sure you've authenticated with the GCP project that the developer needs access to (see [Setup](#setup)). + +Setup: + +```bash +# fill in the user's Google account email address (e.g., Gmail address): +user_email= + +# fill in the user's service account name and set the email address +service_account_name= +service_account_email="${service_account_name}@${GOOGLE_CLOUD_PROJECT}.iam.gserviceaccount.com" + +# choose a role_id (one eample is commented out below), and set the role +role_id= +# role_id="developerStudent" +role="projects/${GOOGLE_CLOUD_PROJECT}/roles/${role_id}" +# the above syntax is for a custom role that we've defined in the project +# you can also use a predefined role. see the reference link given above for all options +# role="role/viewer" +``` + +Add two policy bindings; one for the user's Google account (e.g., Gmail address, used for console access, etc.) and one for the user's service account (used to make API calls). +Note this needs to be done in conjunction with the developer's setup. +The developer needs permissions (a role) bound to their Google account in order to create a service account, and the service account must exist before we can bind a role to it. + +```bash +gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="user:${user_email}" \ + --role="${role}" + +# the service account needs to exist before +# we can run this command to bind the policy +gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="serviceAccount:${service_account_email}" \ + --role="${role}" +``` + +### Offboard a developer + +Follow the setup in [Project manager instructions](#project-manager-instructions), then remove both policy bindings: + +```bash +gcloud projects remove-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="user:${user_email}" \ + --role="${role}" + +gcloud projects remove-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="serviceAccount:${service_account_email}" \ + --role="${role}" +``` diff --git a/broker/iam/policies/production_project.yaml b/broker/iam/policies/production_project.yaml new file mode 100644 index 00000000..8fd81c09 --- /dev/null +++ b/broker/iam/policies/production_project.yaml @@ -0,0 +1,8 @@ +# etag must be obtained from the current policy and filled in below +etag: +bindings: +- role: "projects/ardent-cycling-243415/roles/userPublic" + members: + # allUsers is anyone on the internet, no authentication required. + # Note that access to some GCP services does require authentication. + - user:allUsers diff --git a/broker/iam/policies/testing_project.yaml b/broker/iam/policies/testing_project.yaml new file mode 100644 index 00000000..a4c39479 --- /dev/null +++ b/broker/iam/policies/testing_project.yaml @@ -0,0 +1,8 @@ +# etag must be obtained from the current policy and filled in below +etag: +bindings: +- role: "projects/avid-heading-329016/roles/developerStudent" + members: + # add users or service accounts. + # syntax for service accounts: + # - serviceAccount:@.iam.gserviceaccount.com diff --git a/broker/iam/roles/developer_student.yaml b/broker/iam/roles/developer_student.yaml new file mode 100644 index 00000000..51b880be --- /dev/null +++ b/broker/iam/roles/developer_student.yaml @@ -0,0 +1,7 @@ +title: "Student Developer" +# role-id: "developerStudent" +description: "Role granting permissions for student developers" +stage: "ALPHA" +includedPermissions: +- storage.buckets.get +- storage.buckets.list diff --git a/broker/iam/roles/user_public.yaml b/broker/iam/roles/user_public.yaml new file mode 100644 index 00000000..aa32ac49 --- /dev/null +++ b/broker/iam/roles/user_public.yaml @@ -0,0 +1,6 @@ +title: "Public User of Pitt-Google Broker" +role-id: "userPublic" +description: "Role graning permissions necessary for accessing Pitt-Google's public data resources. Intended to be bound to the `allUsers` member in Pitt-Google's production project. (allUsers is anyone on the internet, no authentication required. Note that access to some GCP services does require authentication." +stage: "ALPHA" +includedPermissions: +- pubsub.topics.attachSubscription From ffa8de2e89e68816ec7e6384ee45083a940d6357 Mon Sep 17 00:00:00 2001 From: troyraen Date: Tue, 22 Mar 2022 16:15:54 -0400 Subject: [PATCH 02/17] update initial setup instructions for new pitt-google developers --- .../run-a-broker-instance/initial-setup.rst | 60 ++++++++++++------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/docs/source/broker/run-a-broker-instance/initial-setup.rst b/docs/source/broker/run-a-broker-instance/initial-setup.rst index 94e7024b..65d4f79c 100644 --- a/docs/source/broker/run-a-broker-instance/initial-setup.rst +++ b/docs/source/broker/run-a-broker-instance/initial-setup.rst @@ -17,6 +17,11 @@ project. Create a GCP Project -------------------- +.. note:: + + You do not need to complete this section if you are a new developer for Pitt-Google + broker. You will use our GCP projects; you do not need to create your own. + 1. Create a new Google Cloud Platform (GCP) project. - A. Go to the `Cloud Resource @@ -61,10 +66,11 @@ may be asked to re-authenticate occasionally in the future. .. code:: bash - PROJECT_ID=my-pgb-project # replace with your GCP Project ID + # fill in the ID of the GCP project you created above (or Pitt-Google's project ID) + PROJECT_ID= - gcloud auth login # follow the instructions to login to GCP - gcloud config set project $PROJECT_ID # set your project ID + gcloud auth login # follow the instructions to login to GCP with a Google account + gcloud config set project $PROJECT_ID # set gcloud to use this project by default 2. **Install Python libraries** for `GCP services `__ and @@ -80,8 +86,8 @@ may be asked to re-authenticate occasionally in the future. conda create -n pgb python=3.7 conda activate pgb - # install the requirements. assumes txt file is in current directory - pip3 install -r requirements.txt + # install the pgb-broker-utils library, which also installs several google.cloud libraries + pip3 install pgb-broker-utils **Note**: On an M1 Mac, first use Conda to install Astropy (``conda install astropy=3.2.1``), then comment the related line out of @@ -94,35 +100,39 @@ the requirements file before doing ``pip install``. .. code:: bash - PROJECT_ID=my-pgb-project # replace with your GCP Project ID - NAME=my-service-account # replace with desired account name - KEY_PATH=local/path/GCP_auth_key.json # replace with desired path (ending in .json) + # choose a service account name (e.g., your name) and fill it in + SA_NAME= + # choose a local path to store your authentication file and fill it in (file name must end with .json) + KEY_PATH= - gcloud iam service-accounts create $NAME - gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:$NAME@$PROJECT_ID.iam.gserviceaccount.com" --role="roles/owner" - gcloud iam service-accounts keys create $KEY_PATH --iam-account=$NAME@$PROJECT_ID.iam.gserviceaccount.com + # create the service account + gcloud iam service-accounts create $SA_NAME -4. **Set environment variables** + # If this is a Pitt-Google project, send your service account name (SA_NAME) + # to a project manager to and as them to grant you a "developer" role on the project + # Otherwise, assign a role to your service account. + # This example below assigns a predifined role called "editor" + # gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + # --member="serviceAccount:${SA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com" \ + # --role="role/editor" -.. code:: bash + # download the authentication file + gcloud iam service-accounts keys create $KEY_PATH --iam-account="${SA_NAME}@${PROJECT_ID}.iam.gserviceaccount.com" - PROJECT_ID=my-pgb-project # replace with your GCP Project ID - KEY_PATH=local/path/GCP_auth_key.json # same path as in step 3 +4. **Set environment variables** +.. code:: bash export GOOGLE_CLOUD_PROJECT="$PROJECT_ID" export GOOGLE_APPLICATION_CREDENTIALS="$KEY_PATH" # export CLOUDSDK_COMPUTE_ZONE= -If you are using a Conda environment, you can configure the environment -variables as follows: +If you are using a Conda environment, you can configure it to automatically set these environment +variables when you activate the environment as follows: .. code:: bash - PROJECT_ID=my-pgb-project # replace with your GCP Project ID - KEY_PATH=local/path/for/key/file.json # same path as in step 3 - - # log into the environment and create de/activate files + # log into the environment and create activate and deactivate files conda activate pgb cd $CONDA_PREFIX mkdir -p ./etc/conda/activate.d @@ -130,14 +140,18 @@ variables as follows: touch ./etc/conda/activate.d/env_vars.sh touch ./etc/conda/deactivate.d/env_vars.sh - # add environment variables + # add commands to automatically set these variables when the environment is activated echo "export GOOGLE_CLOUD_PROJECT='$PROJECT_ID'" >> ./etc/conda/activate.d/env_vars.sh echo "export GOOGLE_APPLICATION_CREDENTIALS='$KEY_PATH'" >> ./etc/conda/activate.d/env_vars.sh + + # add commands to automatically unset these variables when the environment is deactivated echo 'unset GOOGLE_CLOUD_PROJECT' >> ./etc/conda/deactivate.d/env_vars.sh echo 'unset GOOGLE_APPLICATION_CREDENTIALS' >> ./etc/conda/deactivate.d/env_vars.sh 5. **Check that your authentication works** by making an API request. - Here we request a list of Cloud Storage buckets (in Python): + The example below requests a list of Cloud Storage buckets (in Python): + +(This will not work until your service account is assigned to a role, per instructions in step 3) .. code:: python From efc862aa250af1f10eec3987a2852ff126080e6a Mon Sep 17 00:00:00 2001 From: troyraen Date: Tue, 22 Mar 2022 16:32:25 -0400 Subject: [PATCH 03/17] add toc and run mdformat --- broker/iam/README.md | 42 ++++++++++++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/broker/iam/README.md b/broker/iam/README.md index 5c7e0854..52b60cfd 100644 --- a/broker/iam/README.md +++ b/broker/iam/README.md @@ -1,4 +1,21 @@ -# IAM Resources +# IAM Resources + + + +- [IAM Resources](#iam-resources) + - [Setup](#setup) + - [Roles](#roles) + - [Create the roles](#create-the-roles) + - [Update a role](#update-a-role) + - [Policies](#policies) + - [Set the policy on the project](#set-the-policy-on-the-project) + - [Set a policy on a specific resource](#set-a-policy-on-a-specific-resource) + - [Onboard a new developer](#onboard-a-new-developer) + - [Developer instructions](#developer-instructions) + - [Project manager instructions](#project-manager-instructions) + - [Offboard a developer](#offboard-a-developer) + + Basic idea: @@ -9,11 +26,12 @@ We need to connect three things: resource(s), permission(s), and user(s). 1. Attach the Policy to the Project, or to a specific Resource - access will be restricted this restricts access to the resource according to the policy. Policies can be attached to projects, and therefore apply to all resources in the project. See: + - [IAM overview](https://cloud.google.com/iam/docs/overview) - [IAM predefined roles reference](https://cloud.google.com/iam/docs/understanding-roles#predefined) - [IAM permissions reference](https://cloud.google.com/iam/docs/permissions-reference) -## Setup +## Setup Set GCP environment variables and authenticate yourself For reference, our current projects are: @@ -33,9 +51,9 @@ gcloud auth activate-service-account \ --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" ``` -## Roles +## Roles -### Create the roles +### Create the roles ```bash # To Do: @@ -47,7 +65,7 @@ role_yaml=roles/developer_student.yaml gcloud iam roles create $role_id --project=$GOOGLE_CLOUD_PROJECT --file=$role_yaml ``` -### Update a role +### Update a role (simpler in python so you don't have to mess with the etag fingerprint) @@ -71,9 +89,9 @@ def edit_role(name, project, title, description, permissions, stage): return role ``` -## Policies +## Policies -### Set the policy on the project +### Set the policy on the project This binds member-role pairs to every resource in the project. (See below to bind to a specific resource.) @@ -87,7 +105,7 @@ policy_yaml="policies/studev.yaml" gcloud projects set-iam-policy $GOOGLE_CLOUD_PROJECT $policy_yaml ``` -### Set a policy on a specific resource +### Set a policy on a specific resource (simpler in python so you don't have to mess with the etag fingerprint) @@ -121,9 +139,9 @@ print("IAM policy for subscription {} set: {}".format(subscription_id, policy)) client.close() ``` -## Onboard a new developer +## Onboard a new developer -### Developer instructions +### Developer instructions Complete the instructions in the initial setup tutorial (docs/source/broker/run-a-broker-instance/initial-setup.rst). @@ -132,7 +150,7 @@ You will need to do this in conjunction with a Pitt-Google project manager so th You will need to provide the manager with both your Google account email address (e.g., Gmail address) and your service account name (which you will choose during setup). -### Project manager instructions +### Project manager instructions Make sure you've authenticated with the GCP project that the developer needs access to (see [Setup](#setup)). @@ -171,7 +189,7 @@ gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ --role="${role}" ``` -### Offboard a developer +### Offboard a developer Follow the setup in [Project manager instructions](#project-manager-instructions), then remove both policy bindings: From 829f36fe4f1871f6b9641b285dace30f7fa585c8 Mon Sep 17 00:00:00 2001 From: troyraen Date: Tue, 22 Mar 2022 16:32:51 -0400 Subject: [PATCH 04/17] rename links section --- broker/iam/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/iam/README.md b/broker/iam/README.md index 52b60cfd..31fd6650 100644 --- a/broker/iam/README.md +++ b/broker/iam/README.md @@ -25,7 +25,7 @@ We need to connect three things: resource(s), permission(s), and user(s). 1. Create a Policy - this is a local file that defines a collection of bindings (bindings attach roles to members) 1. Attach the Policy to the Project, or to a specific Resource - access will be restricted this restricts access to the resource according to the policy. Policies can be attached to projects, and therefore apply to all resources in the project. -See: +Helpful links: - [IAM overview](https://cloud.google.com/iam/docs/overview) - [IAM predefined roles reference](https://cloud.google.com/iam/docs/understanding-roles#predefined) From 730c01224558f5f8f78a574b1345fef9f858328e Mon Sep 17 00:00:00 2001 From: troyraen Date: Tue, 22 Mar 2022 16:34:43 -0400 Subject: [PATCH 05/17] move instruction clarification --- broker/iam/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/broker/iam/README.md b/broker/iam/README.md index 31fd6650..a9c63450 100644 --- a/broker/iam/README.md +++ b/broker/iam/README.md @@ -154,6 +154,9 @@ You will need to provide the manager with both your Google account email address Make sure you've authenticated with the GCP project that the developer needs access to (see [Setup](#setup)). +Note this needs to be done in conjunction with the developer's setup. +The developer needs permissions (a role) bound to their Google account in order to create a service account, and the service account must exist before we can bind a role to it. + Setup: ```bash @@ -174,8 +177,6 @@ role="projects/${GOOGLE_CLOUD_PROJECT}/roles/${role_id}" ``` Add two policy bindings; one for the user's Google account (e.g., Gmail address, used for console access, etc.) and one for the user's service account (used to make API calls). -Note this needs to be done in conjunction with the developer's setup. -The developer needs permissions (a role) bound to their Google account in order to create a service account, and the service account must exist before we can bind a role to it. ```bash gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ From 0f6510f7b320cd376c22db5886c7b860126e5ee4 Mon Sep 17 00:00:00 2001 From: troyraen Date: Tue, 22 Mar 2022 17:02:08 -0400 Subject: [PATCH 06/17] move docs initial setup instructions --- broker/iam/README.md | 2 +- docs/source/broker/initial-setup.rst | 8 ++++++++ .../initial-setup.rst | 2 ++ docs/source/broker/run-a-broker-instance.rst | 1 - docs/source/broker/run-a-broker-instance/setup-broker.rst | 2 +- .../broker/run-a-broker-instance/test-an-instance.rst | 2 +- docs/source/index.rst | 1 + 7 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 docs/source/broker/initial-setup.rst rename docs/source/broker/{run-a-broker-instance => initial-setup}/initial-setup.rst (98%) diff --git a/broker/iam/README.md b/broker/iam/README.md index a9c63450..e3982cd6 100644 --- a/broker/iam/README.md +++ b/broker/iam/README.md @@ -144,7 +144,7 @@ client.close() ### Developer instructions Complete the instructions in the initial setup tutorial -(docs/source/broker/run-a-broker-instance/initial-setup.rst). +(docs/source/broker/initial-setup/initial-setup.rst). You will need to do this in conjunction with a Pitt-Google project manager so that they can grant you the necessary permissions. diff --git a/docs/source/broker/initial-setup.rst b/docs/source/broker/initial-setup.rst new file mode 100644 index 00000000..c9c676a7 --- /dev/null +++ b/docs/source/broker/initial-setup.rst @@ -0,0 +1,8 @@ +Run a Broker Instance +======================== + +.. toctree:: + :maxdepth: 1 + + initial-setup/initial-setup + initial-setup/manager-instructions diff --git a/docs/source/broker/run-a-broker-instance/initial-setup.rst b/docs/source/broker/initial-setup/initial-setup.rst similarity index 98% rename from docs/source/broker/run-a-broker-instance/initial-setup.rst rename to docs/source/broker/initial-setup/initial-setup.rst index 65d4f79c..213aca8e 100644 --- a/docs/source/broker/run-a-broker-instance/initial-setup.rst +++ b/docs/source/broker/initial-setup/initial-setup.rst @@ -21,6 +21,8 @@ Create a GCP Project You do not need to complete this section if you are a new developer for Pitt-Google broker. You will use our GCP projects; you do not need to create your own. + You will need to complete this in conjunction with a Pitt-Google manager so they + can give you appropriate permissions. 1. Create a new Google Cloud Platform (GCP) project. diff --git a/docs/source/broker/run-a-broker-instance.rst b/docs/source/broker/run-a-broker-instance.rst index 4c6c6f69..10451234 100644 --- a/docs/source/broker/run-a-broker-instance.rst +++ b/docs/source/broker/run-a-broker-instance.rst @@ -5,7 +5,6 @@ Run a Broker Instance :maxdepth: 1 run-a-broker-instance/test-an-instance - run-a-broker-instance/initial-setup run-a-broker-instance/setup-broker run-a-broker-instance/run-broker run-a-broker-instance/delete-broker diff --git a/docs/source/broker/run-a-broker-instance/setup-broker.rst b/docs/source/broker/run-a-broker-instance/setup-broker.rst index 3f3e6159..41f7c82f 100644 --- a/docs/source/broker/run-a-broker-instance/setup-broker.rst +++ b/docs/source/broker/run-a-broker-instance/setup-broker.rst @@ -13,7 +13,7 @@ Setup the Broker Prerequisites ------------- -First, complete the :doc:`initial-setup` to setup your +First, complete the :ref:`docs/source/broker/initial-setup/initial-setup` to setup your Google Cloud project and install the SDKs. Make sure your environment variables are set: diff --git a/docs/source/broker/run-a-broker-instance/test-an-instance.rst b/docs/source/broker/run-a-broker-instance/test-an-instance.rst index 3d0a65da..620732fa 100644 --- a/docs/source/broker/run-a-broker-instance/test-an-instance.rst +++ b/docs/source/broker/run-a-broker-instance/test-an-instance.rst @@ -5,7 +5,7 @@ Run, develop, and test a broker instance. **Prerequisites:** -1. Complete the :doc:`initial-setup` for GCP and your local environment. +1. Complete the :ref:`docs/source/broker/initial-setup/initial-setup` for GCP and your local environment. 2. Create a broker instance by following :doc:`setup-broker`. -------------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 98368cab..af58bdb5 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -34,6 +34,7 @@ The Pitt-Google Broker runs on the `Google Cloud Platform Date: Tue, 22 Mar 2022 17:06:18 -0400 Subject: [PATCH 07/17] add manager instructions --- broker/iam/README.md | 2 + .../initial-setup/manager-instructions.rst | 91 +++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 docs/source/broker/initial-setup/manager-instructions.rst diff --git a/broker/iam/README.md b/broker/iam/README.md index e3982cd6..a1bbc1ce 100644 --- a/broker/iam/README.md +++ b/broker/iam/README.md @@ -141,6 +141,8 @@ client.close() ## Onboard a new developer +The following instructions are left for reference, but these are in the docs (directory: docs/source/broker/initial-setup/) and should be kept up-to-date there. + ### Developer instructions Complete the instructions in the initial setup tutorial diff --git a/docs/source/broker/initial-setup/manager-instructions.rst b/docs/source/broker/initial-setup/manager-instructions.rst new file mode 100644 index 00000000..d9d9c862 --- /dev/null +++ b/docs/source/broker/initial-setup/manager-instructions.rst @@ -0,0 +1,91 @@ +Onboard a new developer +======================= + +.. Developer instructions +.. ---------------------- +.. +.. Complete the instructions in the initial setup tutorial +.. (ref:`docs/source/broker/initial-setup/initial-setup`). +.. +.. You will need to do this in conjunction with a Pitt-Google project manager so that they can grant you the necessary permissions. +.. +.. You will need to provide the manager with both your Google account email address (e.g., Gmail address) and your service account name (which you will choose during setup). + +Project manager instructions +---------------------------- + +Note this needs to be done in conjunction with the developer's setup. +The developer needs permissions (a role) bound to their Google account in order to create a service account, and the service account must exist before we can bind a role to it. + +Setup +^^^^^ + +Set GCP environment variables and authenticate yourself to the GCP project that the developer needs access to. +For reference, our current projects are: + + +* production project: ``ardent-cycling-243415`` +* testing project: ``avid-heading-329016`` + +.. code-block:: bash + + # Set environment variables + # fill these in with your values; they are used throughout + export GOOGLE_CLOUD_PROJECT= + export GOOGLE_APPLICATION_CREDENTIALS= + + # Authenticate to use gcloud tools in this project + gcloud auth activate-service-account \ + --project="${GOOGLE_CLOUD_PROJECT}" \ + --key-file="${GOOGLE_APPLICATION_CREDENTIALS}" + +Set some variables defining the user account(s) and role. + +.. code-block:: bash + + # fill in the user's Google account email address (e.g., Gmail address): + user_email= + + # fill in the user's service account name and set the email address + service_account_name= + service_account_email="${service_account_name}@${GOOGLE_CLOUD_PROJECT}.iam.gserviceaccount.com" + + # choose a role_id (one eample is commented out below), and set the role + role_id= + # role_id="developerStudent" + role="projects/${GOOGLE_CLOUD_PROJECT}/roles/${role_id}" + # the above syntax is for a custom role that we've defined in the project + # you can also use a predefined role. see the reference link given above for all options + # role="role/viewer" + +Onboard a developer +^^^^^^^^^^^^^^^^^^^ + +Add two policy bindings; one for the user's Google account (e.g., Gmail address, used for console access, etc.) and one for the user's service account (used to make API calls). + +.. code-block:: bash + + gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="user:${user_email}" \ + --role="${role}" + + # the service account needs to exist before + # we can run this command to bind the policy + gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="serviceAccount:${service_account_email}" \ + --role="${role}" + +Offboard a developer +^^^^^^^^^^^^^^^^^^^^ + +Follow the setup instructions above to set variables, then remove both policy bindings: + +.. code-block:: bash + + gcloud projects remove-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="user:${user_email}" \ + --role="${role}" + + gcloud projects remove-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="serviceAccount:${service_account_email}" \ + --role="${role}" From 124a5472f7562297f7e5a22e8f59cf30020d5776 Mon Sep 17 00:00:00 2001 From: troyraen Date: Tue, 22 Mar 2022 17:20:09 -0400 Subject: [PATCH 08/17] clarify instructions --- broker/iam/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/broker/iam/README.md b/broker/iam/README.md index a1bbc1ce..78f6e5fc 100644 --- a/broker/iam/README.md +++ b/broker/iam/README.md @@ -91,6 +91,9 @@ def edit_role(name, project, title, description, permissions, stage): ## Policies +We should set the project policy when the project is created. +Afterwards, add or remove bindings to the policy individually (see onboarding section below for an example) rather than using the instructions in this section (which set the policy as a whole). + ### Set the policy on the project This binds member-role pairs to every resource in the project. (See below to bind to a specific resource.) @@ -99,9 +102,9 @@ We must download the current policy, update the file and use it to set a new pol Setting a policy overrides the current policy. ```bash -# To Do: -# for policy_yaml in policies: -policy_yaml="policies/studev.yaml" +policy_file="current_policy.yaml" +gcloud projects get-iam-policy $GOOGLE_CLOUD_PROJECT >> $policy_yaml +# update the bindings in the policy_file as needed, then set a new policy gcloud projects set-iam-policy $GOOGLE_CLOUD_PROJECT $policy_yaml ``` From 6b9d0f6bf044918d0c2cbce2e0ba3879bf40d347 Mon Sep 17 00:00:00 2001 From: troyraen Date: Tue, 22 Mar 2022 17:25:24 -0400 Subject: [PATCH 09/17] clarify docs --- docs/source/broker/initial-setup.rst | 2 +- docs/source/broker/initial-setup/initial-setup.rst | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/source/broker/initial-setup.rst b/docs/source/broker/initial-setup.rst index c9c676a7..88716c48 100644 --- a/docs/source/broker/initial-setup.rst +++ b/docs/source/broker/initial-setup.rst @@ -1,4 +1,4 @@ -Run a Broker Instance +Initial Setup ======================== .. toctree:: diff --git a/docs/source/broker/initial-setup/initial-setup.rst b/docs/source/broker/initial-setup/initial-setup.rst index 213aca8e..ce5167b6 100644 --- a/docs/source/broker/initial-setup/initial-setup.rst +++ b/docs/source/broker/initial-setup/initial-setup.rst @@ -19,10 +19,7 @@ Create a GCP Project .. note:: - You do not need to complete this section if you are a new developer for Pitt-Google - broker. You will use our GCP projects; you do not need to create your own. - You will need to complete this in conjunction with a Pitt-Google manager so they - can give you appropriate permissions. + You do not need to complete this section if you are a new developer for Pitt-Google broker. You will use our GCP projects; you do not need to create your own. Skip to the Setup Local Environment section. 1. Create a new Google Cloud Platform (GCP) project. @@ -51,8 +48,13 @@ Setup Local Environment Broker instances *run* 100% in the Google Cloud, as determined by the code in the ``broker`` package. You can *develop* the code and/or -*deploy* an instance to the Cloud from your local machine. Setup your -environment as follows: +*deploy* an instance to the Cloud from your local machine. + +.. note:: + + If you are a new developer for Pitt-Google, You will need to complete this in conjunction with a Pitt-Google manager so they can give you appropriate permissions. + +Setup your environment as follows: 1. **Install Google Cloud SDK command-line tools** using one of the following options. Included tools: gcloud, gsutil, and From e7c9ca7503dea4de52624530f1163a7ba0503e94 Mon Sep 17 00:00:00 2001 From: troyraen Date: Tue, 22 Mar 2022 17:29:38 -0400 Subject: [PATCH 10/17] add policies readme --- broker/iam/policies/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 broker/iam/policies/README.md diff --git a/broker/iam/policies/README.md b/broker/iam/policies/README.md new file mode 100644 index 00000000..1e15120c --- /dev/null +++ b/broker/iam/policies/README.md @@ -0,0 +1,3 @@ +# Policies + +The files in this directory are example policies. They are not currently set on our projects. From c75e70525c9aca7bcf499eaebca09e0fbad5783a Mon Sep 17 00:00:00 2001 From: troyraen Date: Tue, 22 Mar 2022 17:47:19 -0400 Subject: [PATCH 11/17] udpate comment --- broker/iam/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/iam/README.md b/broker/iam/README.md index 78f6e5fc..bc739740 100644 --- a/broker/iam/README.md +++ b/broker/iam/README.md @@ -23,7 +23,7 @@ We need to connect three things: resource(s), permission(s), and user(s). 1. Create a Role - this is a collection of permissions and is a registered resource in GCP. 1. Create a Policy - this is a local file that defines a collection of bindings (bindings attach roles to members) -1. Attach the Policy to the Project, or to a specific Resource - access will be restricted this restricts access to the resource according to the policy. Policies can be attached to projects, and therefore apply to all resources in the project. +1. Attach the Policy to the Project, or to a specific Resource Helpful links: From d1881132e8c11e16584a4d639af375baaabcf018 Mon Sep 17 00:00:00 2001 From: troyraen Date: Wed, 23 Mar 2022 01:54:59 -0400 Subject: [PATCH 12/17] clarify instructions --- docs/source/broker/initial-setup/manager-instructions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/broker/initial-setup/manager-instructions.rst b/docs/source/broker/initial-setup/manager-instructions.rst index d9d9c862..8ffe2ce6 100644 --- a/docs/source/broker/initial-setup/manager-instructions.rst +++ b/docs/source/broker/initial-setup/manager-instructions.rst @@ -20,7 +20,7 @@ The developer needs permissions (a role) bound to their Google account in order Setup ^^^^^ -Set GCP environment variables and authenticate yourself to the GCP project that the developer needs access to. +Set GCP environment variables and authenticate yourself to the GCP project that the developer needs access to (probably the testing project). For reference, our current projects are: From 088b973425bfd9ac235bda40fe60a0e907be0483 Mon Sep 17 00:00:00 2001 From: troyraen Date: Thu, 24 Mar 2022 19:11:29 -0400 Subject: [PATCH 13/17] add developer instructions --- .../initial-setup/manager-instructions.rst | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/source/broker/initial-setup/manager-instructions.rst b/docs/source/broker/initial-setup/manager-instructions.rst index 8ffe2ce6..d5bc788f 100644 --- a/docs/source/broker/initial-setup/manager-instructions.rst +++ b/docs/source/broker/initial-setup/manager-instructions.rst @@ -1,15 +1,18 @@ Onboard a new developer ======================= -.. Developer instructions -.. ---------------------- -.. -.. Complete the instructions in the initial setup tutorial -.. (ref:`docs/source/broker/initial-setup/initial-setup`). -.. -.. You will need to do this in conjunction with a Pitt-Google project manager so that they can grant you the necessary permissions. -.. -.. You will need to provide the manager with both your Google account email address (e.g., Gmail address) and your service account name (which you will choose during setup). +..contents:: title + :depth: 2 + +Developer instructions +---------------------- + +Complete the instructions in the initial setup tutorial +(ref:`docs/source/broker/initial-setup/initial-setup`). + +You will need to do this in conjunction with a Pitt-Google project manager so that they can grant you the necessary permissions. + +You will need to provide the manager with both your Google account email address (e.g., Gmail address) and your service account name (which you will choose during setup). Project manager instructions ---------------------------- From e804120f762095b2f6f62d5a018a77ad21b7144c Mon Sep 17 00:00:00 2001 From: troyraen Date: Sat, 26 Mar 2022 22:16:58 -0400 Subject: [PATCH 14/17] rename directory policies -> policy_examples --- broker/iam/{policies => policy_examples}/README.md | 0 broker/iam/{policies => policy_examples}/production_project.yaml | 0 broker/iam/{policies => policy_examples}/testing_project.yaml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename broker/iam/{policies => policy_examples}/README.md (100%) rename broker/iam/{policies => policy_examples}/production_project.yaml (100%) rename broker/iam/{policies => policy_examples}/testing_project.yaml (100%) diff --git a/broker/iam/policies/README.md b/broker/iam/policy_examples/README.md similarity index 100% rename from broker/iam/policies/README.md rename to broker/iam/policy_examples/README.md diff --git a/broker/iam/policies/production_project.yaml b/broker/iam/policy_examples/production_project.yaml similarity index 100% rename from broker/iam/policies/production_project.yaml rename to broker/iam/policy_examples/production_project.yaml diff --git a/broker/iam/policies/testing_project.yaml b/broker/iam/policy_examples/testing_project.yaml similarity index 100% rename from broker/iam/policies/testing_project.yaml rename to broker/iam/policy_examples/testing_project.yaml From 5cb55e50438c4238dd547e57d227e341cc0478f6 Mon Sep 17 00:00:00 2001 From: troyraen Date: Sun, 27 Mar 2022 19:59:53 -0400 Subject: [PATCH 15/17] change studentDeveloper to developer --- broker/iam/README.md | 21 +++++++++++---------- broker/iam/roles/developer.yaml | 9 +++++++++ broker/iam/roles/developer_student.yaml | 7 ------- 3 files changed, 20 insertions(+), 17 deletions(-) create mode 100644 broker/iam/roles/developer.yaml delete mode 100644 broker/iam/roles/developer_student.yaml diff --git a/broker/iam/README.md b/broker/iam/README.md index bc739740..58e4e49e 100644 --- a/broker/iam/README.md +++ b/broker/iam/README.md @@ -55,13 +55,11 @@ gcloud auth activate-service-account \ ### Create the roles +Here is an example for creating the developer role. + ```bash -# To Do: -# get the organization id (not project id) -# for role_yaml in roles: -# read role_id from file, then: -role_id=developerStudent -role_yaml=roles/developer_student.yaml +role_id=developer +role_yaml=roles/developer.yaml gcloud iam roles create $role_id --project=$GOOGLE_CLOUD_PROJECT --file=$role_yaml ``` @@ -144,7 +142,7 @@ client.close() ## Onboard a new developer -The following instructions are left for reference, but these are in the docs (directory: docs/source/broker/initial-setup/) and should be kept up-to-date there. +NOTE: The following instructions are left for reference, but these are in the docs (currently at: docs/source/broker/initial-setup/manager-instructions.rst) and should be kept up-to-date there. ### Developer instructions @@ -169,12 +167,12 @@ Setup: user_email= # fill in the user's service account name and set the email address -service_account_name= +# service_account_name= service_account_email="${service_account_name}@${GOOGLE_CLOUD_PROJECT}.iam.gserviceaccount.com" -# choose a role_id (one eample is commented out below), and set the role +# choose a role_id (one example is commented out below), and set the role role_id= -# role_id="developerStudent" +# role_id="developer" role="projects/${GOOGLE_CLOUD_PROJECT}/roles/${role_id}" # the above syntax is for a custom role that we've defined in the project # you can also use a predefined role. see the reference link given above for all options @@ -193,6 +191,9 @@ gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ --member="serviceAccount:${service_account_email}" \ --role="${role}" + +# required to deploy Cloud Functions. only needs to be granted to the user account. +gcloud iam service-accounts add-iam-policy-binding avid-heading-329016@appspot.gserviceaccount.com --member=user:amp322@pitt.edu --role=roles/iam.serviceAccountUser ``` ### Offboard a developer diff --git a/broker/iam/roles/developer.yaml b/broker/iam/roles/developer.yaml new file mode 100644 index 00000000..7b686d57 --- /dev/null +++ b/broker/iam/roles/developer.yaml @@ -0,0 +1,9 @@ +title: "Pitt-Google Developer" +# role-id: "developer" +description: "Role granting permissions for Pitt-Google developers that are not already bundled in a suitable, predefined role." +stage: "ALPHA" +includedPermissions: +- storage.buckets.get +- storage.buckets.list +- storage.objects.list +- storage.objects.get diff --git a/broker/iam/roles/developer_student.yaml b/broker/iam/roles/developer_student.yaml deleted file mode 100644 index 51b880be..00000000 --- a/broker/iam/roles/developer_student.yaml +++ /dev/null @@ -1,7 +0,0 @@ -title: "Student Developer" -# role-id: "developerStudent" -description: "Role granting permissions for student developers" -stage: "ALPHA" -includedPermissions: -- storage.buckets.get -- storage.buckets.list From 75f50798441fc96a6909dac6d9649fd51e8a0ecb Mon Sep 17 00:00:00 2001 From: troyraen Date: Sun, 27 Mar 2022 20:03:30 -0400 Subject: [PATCH 16/17] comment non-keyword. leave for reference --- broker/iam/roles/user_public.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/broker/iam/roles/user_public.yaml b/broker/iam/roles/user_public.yaml index aa32ac49..976dfd72 100644 --- a/broker/iam/roles/user_public.yaml +++ b/broker/iam/roles/user_public.yaml @@ -1,5 +1,6 @@ title: "Public User of Pitt-Google Broker" -role-id: "userPublic" +# intended role-id is (set this at deployment): +# "userPublic" description: "Role graning permissions necessary for accessing Pitt-Google's public data resources. Intended to be bound to the `allUsers` member in Pitt-Google's production project. (allUsers is anyone on the internet, no authentication required. Note that access to some GCP services does require authentication." stage: "ALPHA" includedPermissions: From fcaa2b040fcd4f17cd6de18d59d0ae697b6219bb Mon Sep 17 00:00:00 2001 From: troyraen Date: Sun, 27 Mar 2022 20:31:37 -0400 Subject: [PATCH 17/17] leave comment --- broker/iam/roles/developer.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/broker/iam/roles/developer.yaml b/broker/iam/roles/developer.yaml index 7b686d57..ca84e0e4 100644 --- a/broker/iam/roles/developer.yaml +++ b/broker/iam/roles/developer.yaml @@ -1,5 +1,6 @@ title: "Pitt-Google Developer" -# role-id: "developer" +# intended role-id is (set this at deployment): +# "developer" description: "Role granting permissions for Pitt-Google developers that are not already bundled in a suitable, predefined role." stage: "ALPHA" includedPermissions: