-
Notifications
You must be signed in to change notification settings - Fork 0
/
setup.sh
300 lines (241 loc) · 10.9 KB
/
setup.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
#!/usr/bin/env bash
#####################################################################
# REFERENCES
# - https://cloud.google.com/docs/terraform/best-practices-for-terraform
# - https://registry.terraform.io/modules/terraform-google-modules/kubernetes-engine/google/latest/submodules/safer-cluster
# - https://cloud.google.com/kubernetes-engine/docs/how-to/hardening-your-cluster#restrict_self_modify
# - https://ashwin9798.medium.com/nginx-with-docker-and-node-js-a-beginners-guide-434fe1216b6b
# - https://docs.docker.com/develop/develop-images/dockerfile_best-practices/
# - https://webbylab.com/blog/minimal_size_docker_image_for_your_nodejs_app/
# - https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md
# - https://cloud.google.com/nodejs/docs/reference/secret-manager/latest
# - https://cloud.google.com/secret-manager/docs/manage-access-to-secrets#secretmanager-create-secret-gcloud
# - https://cloud.google.com/run/docs/securing/service-identity
# - https://aandhsolutions.com/blog/run-nginx-as-unprivileged-user-in-docker-container-on-kubernetes/
# - https://forums.docker.com/t/running-nginx-official-image-as-non-root/135759
# - https://kubernetes.io/blog/2021/11/09/non-root-containers-and-devices/
# - https://cloud.google.com/kubernetes-engine/docs/concepts/gateway-security
# - https://cloud.google.com/kubernetes-engine/docs/how-to/secure-gateway#secure-using-secret
# - https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
#####################################################################
export PROJECT_ID=$(gcloud config get-value project)
export PROJECT_USER=$(gcloud config get-value core/account) # set current user
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)")
export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain
export GCP_REGION="us-central1" # CHANGEME (OPT)
export GCP_ZONE="us-central1-a" # CHANGEME (OPT)
export NETWORK_NAME="safer-cluster-network-dev"
# enable apis
gcloud services enable compute.googleapis.com \
oslogin.googleapis.com \
container.googleapis.com \
containersecurity.googleapis.com \
secretmanager.googleapis.com \
artifactregistry.googleapis.com \
cloudbuild.googleapis.com \
storage.googleapis.com \
run.googleapis.com \
iap.googleapis.com
# configure gcloud sdk
gcloud config set compute/region $GCP_REGION
gcloud config set compute/zone $GCP_ZONE
############################################################
# initialize project for Terraform
# using short-lived token (1hr) instead of permanent key
############################################################
# create bucket for shared state
export BUCKET_NAME="$PROJECT_ID-tfstate"
gcloud storage buckets create gs://$BUCKET_NAME
# set tf env for Apple silicon env
export TFENV_ARCH=amd64
export TFENV_CONFIG_DIR=${XDG_CACHE_HOME:-$HOME/.cache}/tfenv/${TFENV_ARCH}
# authenticate current user with short-lived token for tf
export GOOGLE_OAUTH_ACCESS_TOKEN=$(gcloud auth print-access-token)
# authenticate current user for local SDK testing
gcloud auth application-default login
############################################################
# create secret versions in secret manager
# *** assuming secret created by TF in cloud-config/environments/dev ***
############################################################
export SECRET_ID="foo"
export SECRET_VALUE="Super_Secret" # just for demo but would never do this
export VERSION_ID="latest"
# add secret version
echo -n $SECRET_VALUE | \
gcloud secrets versions add $SECRET_ID --data-file=-
# verify
gcloud secrets versions access $VERSION_ID --secret=$SECRET_ID
############################################################
# create service account for test app and grant secret access
# *** assuming secret created by TF in cloud-config/environments/dev ***
############################################################
export SA_NAME="app-sa"
export SA_EMAIL="$SA_NAME@$PROJECT_ID.iam.gserviceaccount.com"
export KNS="app-ns" # kubernetes namespace
export KSA="app-sa" # kubernetes service account
# create service account
gcloud iam service-accounts create $SA_NAME \
--description="$SA_NAME" \
--display-name="$DISPLAY_NAME"
# grant myself user role
gcloud iam service-accounts add-iam-policy-binding $SA_EMAIL \
--member="user:$PROJECT_USER" \
--role="roles/iam.serviceAccountUser"
gcloud secrets add-iam-policy-binding $SECRET_ID \
--member="user:$PROJECT_USER" \
--role="roles/secretmanager.secretAccessor"
# grant service account secret accessor
gcloud secrets add-iam-policy-binding $SECRET_ID \
--member="serviceAccount:$SA_EMAIL" \
--role="roles/secretmanager.secretAccessor"
# grant workload identity to sa for k8s binding
gcloud iam service-accounts add-iam-policy-binding $SA_EMAIL \
--role roles/iam.workloadIdentityUser \
--member "serviceAccount:$PROJECT_ID.svc.id.goog[$KNS/$KSA]"
############################################################
# push test images to the repo
# *** assuming repo created by TF in cloud-config/environments/01_shared ***
############################################################
export REPO_NAME="mike-test-repo"
export IMAGE1_NAME="app"
export TAG1_NAME="v1"
export IMAGE1_URL=${GCP_REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO_NAME}/${IMAGE1_NAME}:${TAG1_NAME}
# optional (or use public image in manifest)
export IMAGE2_NAME="proxy"
export TAG2_NAME="v1"
export IMAGE2_URL=${GCP_REGION}-docker.pkg.dev/${PROJECT_ID}/${REPO_NAME}/${IMAGE2_NAME}:${TAG2_NAME}
export PROJECT_DIR=$(pwd)
# configure auth
gcloud auth configure-docker ${GCP_REGION}-docker.pkg.dev
# run this in app dir
cd $PROJECT_DIR/app
gcloud builds submit --tag $IMAGE1_URL
# (optional) run this in app-proxy dir
cd $PROJECT_DIR/app-proxy
gcloud builds submit --tag $IMAGE2_URL
cd $PROJECT_DIR
############################################################
# Test app runs and accesses secret manager in temp Cloud Run service
# - note: PORT env var automatically set with Cloud Run
############################################################
export SERVICE_NAME="app-test"
export SERVICE_PORT="3000" # override default 8080 for this nodejs app test
gcloud run deploy $SERVICE_NAME \
--platform managed \
--region $GCP_REGION \
--service-account $SA_EMAIL \
--allow-unauthenticated \
--image $IMAGE1_URL \
--port $SERVICE_PORT \
--set-env-vars "PROJECT_ID=$PROJECT_ID" \
--set-env-vars "SECRET_ID=$SECRET_ID" \
--set-env-vars "SECRET_VERSION_ID=$SECRET_VERSION_ID"
# confirm service is running
gcloud run services list \
--platform managed \
--region $GCP_REGION
# get service url and test
export SVC_URL=$(gcloud run services describe $SERVICE_NAME --platform managed --region $GCP_REGION --format="value(status.url)")
curl -X GET $SVC_URL
############################################################
# TLS self-signed test for Gateway
############################################################
export PRIVATE_KEY_FILE="app.key"
export CSR_CONFIG_FILE="app-csr.conf"
export CSR_FILE="app.csr"
export CERTIFICATE_FILE="app.crt"
export CERTIFICATE_TERM="730" # 2 yrs
export CERTIFICATE_NAME="app-internal-cert"
openssl genrsa -out $PRIVATE_KEY_FILE 2048
openssl ecparam -name prime256v1 -genkey -noout -out $PRIVATE_KEY_FILE
# create CSR config
cat <<EOF > $CSR_CONFIG_FILE
[req]
default_bits = 2048
req_extensions = extension_requirements
distinguished_name = dn_requirements
prompt = no
[extension_requirements]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @sans_list
[dn_requirements]
countryName = US
stateOrProvinceName = MT
localityName = Missoula
0.organizationName = DoiT International
organizationalUnitName = engineering
commonName = app.example.com
emailAddress = [email protected]
[sans_list]
DNS.1 = app.example.com
EOF
# create CSR
openssl req -new -key $PRIVATE_KEY_FILE \
-out $CSR_FILE \
-config $CSR_CONFIG_FILE
# create certificate
openssl x509 -req \
-signkey $PRIVATE_KEY_FILE \
-in $CSR_FILE \
-out $CERTIFICATE_FILE \
-extfile $CSR_CONFIG_FILE \
-extensions extension_requirements \
-days $CERTIFICATE_TERM
# create regional SSL cert
gcloud compute ssl-certificates create $CERTIFICATE_NAME \
--certificate=$CERTIFICATE_FILE \
--private-key=$PRIVATE_KEY_FILE \
--region=$GCP_REGION
# verify
gcloud compute ssl-certificates describe $CERTIFICATE_NAME \
--region=$GCP_REGION
# create kubernetes TLS secret (when cluster avail)
kubectl create secret tls app-example-com \
--namespace=$KNS \
--cert=$CERTIFICATE_FILE \
--key=$PRIVATE_KEY_FILE
######### NAT GATEWAY FOR TESTING BASTION ############
export NAT_GW_IP="nat-gw-ip"
export CLOUD_ROUTER_NAME="router-1"
export CLOUD_ROUTER_ASN="64523"
export NAT_GW_NAME="nat-gateway-1"
# create IP address
gcloud compute addresses create $NAT_GW_IP --region $GCP_REGION
# create cloud router and nat gateway
gcloud compute routers create $CLOUD_ROUTER_NAME \
--network $NETWORK_NAME \
--asn $CLOUD_ROUTER_ASN \
--region $GCP_REGION
gcloud compute routers nats create $NAT_GW_NAME \
--router=$CLOUD_ROUTER_NAME \
--region=$GCP_REGION \
--auto-allocate-nat-external-ips \
--nat-all-subnet-ip-ranges \
--enable-logging
# change to static IP (test)
gcloud compute routers nats update $NAT_GW_NAME \
--router=$CLOUD_ROUTER_NAME \
--nat-external-ip-pool=$NAT_GW_IP
##########################################################
# K8S / GKE cluster resources
# *** assuming cluster created by TF in cloud-config/environments/dev ***
##########################################################
# apply pod-level security policies
# - https://cloud.google.com/kubernetes-engine/docs/how-to/podsecurityadmission
# - https://kubernetes.io/docs/tasks/configure-pod-container/enforce-standards-namespace-labels/#applying-to-all-namespaces
# another option
# - https://cloud.google.com/kubernetes-engine/docs/how-to/podsecurityadmission#alternatives
# assume secrets created [foo, bar, buzz, baz] from TF so assign values
# recommend using workload identity and accessing from your code (best practice)
# - https://cloud.google.com/kubernetes-engine/docs/tutorials/workload-identity-secrets
# other options
# - https://cloud.google.com/kubernetes-engine/docs/how-to/encrypting-secrets
# - https://github.com/GoogleCloudPlatform/secrets-store-csi-driver-provider-gcp
# - https://cloud.google.com/secret-manager/docs/access-secret-version
# access bastion and apply k8s configs
export BASTION_NAME="bastion-vm"
export BASTION_ZONE="us-central1-c"
gcloud compute ssh $BASTION_NAME --zone $BASTION_ZONE
# from bastion, test gateway internal IP and TLS works with curl
curl -k https://app.example.com --resolve 'app.example.com:443:10.0.0.9'