-
-
Save mikesparr/090a1f94bf24286a953d89a37874110f to your computer and use it in GitHub Desktop.
BILLING="YOUR-BILLING-ACCT" | |
ORGANIZATION="<ORG-ID-NUMBER>" | |
FOLDER="<FOLDER-ID-NUMBER>" | |
CUSTOMER="<CUSTOMER-ID>" | |
# user groups | |
export ORG_ADMIN_GROUP="[email protected]" | |
export BILLING_ADMIN_GROUP="[email protected]" | |
export SECURITY_ADMIN_GROUP="[email protected]" | |
export NETWORK_ADMIN_GROUP="[email protected]" | |
export DEVELOPER_GROUP="[email protected]" | |
export DEVOPS_GROUP="[email protected]" | |
# shared projects | |
BILLING_PROJECT_ID="billing" | |
SECURITY_PROJECT_ID="security" | |
MONITORING_PROJECT_ID="monitoring" | |
DEVOPS_PROJECT_ID="devops" | |
# service projects | |
SANDBOX_PROJECT_ID="myco-sandbox" | |
DATA_SCIENCE_PROJECT_ID="myco-data-science" | |
BACKEND_STAGE_PROJECT_ID="myco-backend-stage" | |
BACKEND_PROD_PROJECT_ID="myco-backend-prod" | |
FRONTEND_STAGE_PROJECT_ID="myco-frontend-stage" | |
FRONTEND_PROD_PROJECT_ID="myco-frontend-prod" |
#!/bin/bash | |
# --- Configuration --- | |
source .env | |
# --- Helper Function (optional, for cleaner output) --- | |
apply_policy() { | |
local constraint_name="$1" | |
local policy_file="$2" | |
local description="$3" | |
echo "-----------------------------------------------------" | |
echo "Applying Policy: $description ($constraint_name)" | |
echo "Policy File Content:" | |
cat "$policy_file" | |
echo "" | |
gcloud resource-manager org-policies set-policy "$policy_file" --organization="$ORGANIZATION" | |
if [[ $? -eq 0 ]]; then | |
echo "SUCCESS: Applied policy $constraint_name." | |
else | |
echo "ERROR: Failed to apply policy $constraint_name. Check permissions and configuration." | |
# Consider adding 'exit 1' here if you want the script to stop on failure | |
fi | |
rm "$policy_file" # Clean up temporary file | |
echo "-----------------------------------------------------" | |
echo "" | |
} | |
# --- Check if ORGANIZATION and CUSTOMER are set --- | |
if [[ "$ORGANIZATION" == "<ORG-ID-NUMBER>" || "$CUSTOMER" == "<CUSTOMER-ID>" ]]; then | |
echo "ERROR: Please replace the placeholder values for ORGANIZATION and CUSTOMER in .env file." | |
exit 1 | |
fi | |
# --- 1. Restrict VM External IP Access --- | |
POLICY_FILE_VM_IP="policy_vm_external_ip.yaml" | |
cat << EOF > "$POLICY_FILE_VM_IP" | |
constraint: constraints/compute.vmExternalIpAccess | |
listPolicy: | |
allValues: DENY | |
EOF | |
apply_policy "constraints/compute.vmExternalIpAccess" "$POLICY_FILE_VM_IP" "Restrict VM External IP Access" | |
# --- 2. Enforce Public Access Prevention for Cloud Storage --- | |
POLICY_FILE_GCS_PUBLIC="policy_gcs_public.yaml" | |
cat << EOF > "$POLICY_FILE_GCS_PUBLIC" | |
constraint: constraints/storage.publicAccessPrevention | |
booleanPolicy: | |
enforced: true | |
EOF | |
apply_policy "constraints/storage.publicAccessPrevention" "$POLICY_FILE_GCS_PUBLIC" "Enforce Storage Public Access Prevention" | |
# --- 3. Enforce Uniform Bucket-level Access for Cloud Storage --- | |
POLICY_FILE_GCS_PUBLIC="policy_gcs_uniform.yaml" | |
cat << EOF > "$POLICY_FILE_GCS_PUBLIC" | |
constraint: constraints/storage.uniformBucketLevelAccess | |
booleanPolicy: | |
enforced: true | |
EOF | |
apply_policy "constraints/storage.uniformBucketLevelAccess" "$POLICY_FILE_GCS_PUBLIC" "Require Uniform Bucket-level Access" | |
# --- 4. Disable Service Account Key Creation --- | |
POLICY_FILE_SA_KEYS="policy_sa_keys.yaml" | |
cat << EOF > "$POLICY_FILE_SA_KEYS" | |
constraint: constraints/iam.disableServiceAccountKeyCreation | |
booleanPolicy: | |
enforced: true | |
EOF | |
apply_policy "constraints/iam.disableServiceAccountKeyCreation" "$POLICY_FILE_SA_KEYS" "Disable Service Account Key Creation" | |
# POLICY_FILE_SA_KEYS_MNG="policy_sa_keys_managed.yaml" | |
# cat << EOF > "$POLICY_FILE_SA_KEYS_MNG" | |
# constraint: constraints/iam.managed.disableServiceAccountKeyCreation | |
# booleanPolicy: | |
# enforced: true | |
# EOF | |
# apply_policy "constraints/iam.managed.disableServiceAccountKeyCreation" "$POLICY_FILE_SA_KEYS_MNG" "Disable Service Account Key Creation (Managed)" | |
# --- 5. Disable Service Account Key Upload --- | |
POLICY_FILE_SA_KEYS_UP="policy_sa_keys_up.yaml" | |
cat << EOF > "$POLICY_FILE_SA_KEYS_UP" | |
constraint: constraints/iam.disableServiceAccountKeyUpload | |
booleanPolicy: | |
enforced: true | |
EOF | |
apply_policy "constraints/iam.disableServiceAccountKeyUpload" "$POLICY_FILE_SA_KEYS_UP" "Disable Service Account Key Upload" | |
# POLICY_FILE_SA_KEYS_UP_MNG="policy_sa_keys_up_managed.yaml" | |
# cat << EOF > "$POLICY_FILE_SA_KEYS_UP_MNG" | |
# constraint: constraints/iam.managed.disableServiceAccountKeyUpload | |
# booleanPolicy: | |
# enforced: true | |
# EOF | |
# apply_policy "constraints/iam.managed.disableServiceAccountKeyUpload" "$POLICY_FILE_SA_KEYS_UP_MNG" "Disable Service Account Key Upload (Managed)" | |
# --- 6. Restrict Allowed Domains for IAM Policies --- | |
POLICY_FILE_IAM_DOMAINS="policy_iam_domains.yaml" | |
cat << EOF > "$POLICY_FILE_IAM_DOMAINS" | |
constraint: constraints/iam.allowedPolicyMemberDomains | |
listPolicy: | |
allowedValues: | |
- $CUSTOMER | |
EOF | |
apply_policy "constraints/iam.allowedPolicyMemberDomains" "$POLICY_FILE_IAM_DOMAINS" "Restrict Allowed IAM Member Domains" | |
# --- 7. Disable Serial Port Access --- | |
POLICY_FILE_SERIAL="policy_serial_port.yaml" | |
cat << EOF > "$POLICY_FILE_SERIAL" | |
constraint: constraints/compute.disableSerialPortAccess | |
booleanPolicy: | |
enforced: true | |
EOF | |
apply_policy "constraints/compute.disableSerialPortAccess" "$POLICY_FILE_SERIAL" "Disable Serial Port Access" | |
# --- 8. Skip Default Network Creation --- | |
POLICY_FILE_DEFAULT_NET="policy_default_network.yaml" | |
cat << EOF > "$POLICY_FILE_DEFAULT_NET" | |
constraint: constraints/compute.skipDefaultNetworkCreation | |
booleanPolicy: | |
enforced: true | |
EOF | |
apply_policy "constraints/compute.skipDefaultNetworkCreation" "$POLICY_FILE_DEFAULT_NET" "Skip Default Network Creation" | |
echo "All foundational organization policies applied." | |
echo "Verification: Use 'gcloud resource-manager org-policies describe CONSTRAINT_NAME --organization=$ORGANIZATION' for each constraint." | |
echo "Example: gcloud resource-manager org-policies describe constraints/compute.vmExternalIpAccess --organization=$ORGANIZATION" |
#!/usr/bin/env bash | |
##################################################################### | |
# REFERENCES | |
# - https://cloud.google.com/resource-manager/docs/organization-policy/org-policy-constraints | |
# - https://cloud.google.com/logging/docs/central-log-storage | |
# - https://cloud.google.com/logging/docs/export/configure_export_v2 | |
# - https://cloud.google.com/billing/docs/how-to/export-data-bigquery-setup | |
# - https://cloud.google.com/artifact-registry/docs/docker/store-docker-container-images | |
# - https://cloud.google.com/monitoring/settings/manage-api | |
# - https://cloud.google.com/sdk/gcloud/reference/apphub | |
##################################################################### | |
export PROJECT_ID=$(gcloud config get-value project) | |
export PROJECT_USER=$(gcloud config get-value core/account) # set current user | |
export PROJECT_NUMBER=$(gcloud projects describe $PROJECT_ID --format="value(projectNumber)") | |
export IDNS=${PROJECT_ID}.svc.id.goog # workflow identity domain | |
export GCP_REGION="us-central1" # CHANGEME (OPT) | |
export GCP_ZONE="us-central1-a" # CHANGEME (OPT) | |
export NETWORK_NAME="default" | |
# configure gcloud sdk | |
gcloud config set compute/region $GCP_REGION | |
gcloud config set compute/zone $GCP_ZONE | |
# projects | |
export BILLING_PROJECT_ID="billing" # CHANGEME (IDENTIFY YOUR ACTUAL PROJECT ID) | |
export SECURITY_PROJECT_ID="security" # CHANGEME (IDENTIFY YOUR ACTUAL PROJECT ID) | |
export MONITORING_PROJECT_ID="monitoring" # CHANGEME (IDENTIFY YOUR ACTUAL PROJECT ID) | |
export DEVOPS_PROJECT_ID="devops" # CHANGEME (IDENTIFY YOUR ACTUAL PROJECT ID) | |
# env (billing, organization, folder, customer, project IDs) | |
source .env | |
######################################################## | |
# ORGANIZATION | |
######################################################## | |
export DOMAIN="example.com" | |
# apply org policies | |
echo "Applying foundational org policies..." | |
./policies.sh | |
echo "Applying foundational org policies... DONE!" | |
# remove default global roles (project and billing account) | |
gcloud organizations remove-iam-policy-binding $ORGANIZATION \ | |
--member="domain:$DOMAIN" \ | |
--role="roles/resourcemanager.projectCreator" \ | |
--condition=None | |
gcloud organizations remove-iam-policy-binding $ORGANIZATION \ | |
--member="domain:$DOMAIN" \ | |
--role="roles/billing.user" \ | |
--condition=None | |
######################################################## | |
# SECURITY | |
######################################################## | |
export SECURITY_BUCKET_NAME="myco-audit-logs" | |
export SECURITY_BUCKET_LOCATION="US" | |
export LOG_SINK_NAME="audit-logs-sink" | |
export LOG_SINK_BUCKET_PATH="storage.googleapis.com/$SECURITY_BUCKET_NAME" | |
# enable services apis | |
gcloud services enable compute.googleapis.com \ | |
storage.googleapis.com \ | |
--project $SECURITY_PROJECT_ID | |
# create bucket | |
gcloud storage buckets create gs://$SECURITY_BUCKET_NAME \ | |
--public-access-prevention \ | |
--uniform-bucket-level-access \ | |
--location $SECURITY_BUCKET_LOCATION \ | |
--project $SECURITY_PROJECT_ID | |
# update bucket to clear soft delete | |
gcloud storage buckets update gs://$SECURITY_BUCKET_NAME \ | |
--clear-soft-delete \ | |
--project $SECURITY_PROJECT_ID | |
# create aggregated audit log sink | |
gcloud logging sinks create $LOG_SINK_NAME $LOG_SINK_BUCKET_PATH \ | |
--log-filter='logName:cloudaudit.googleapis.com' \ | |
--description="Audit logs from my organization" \ | |
--organization=$ORGANIZATION \ | |
--include-children | |
# fetch sink writer identity | |
LOG_WRITER_IDENTITY=$(gcloud logging sinks describe $LOG_SINK_NAME --organization $ORGANIZATION --format="value(writerIdentity)") | |
echo $LOG_WRITER_IDENTITY | |
# authorize writer identity on destination bucket | |
gcloud storage buckets add-iam-policy-binding gs://$SECURITY_BUCKET_NAME \ | |
--member=$LOG_WRITER_IDENTITY \ | |
--role="roles/storage.objectCreator" | |
######################################################## | |
# BILLING | |
######################################################## | |
export BILLING_DATASET_NAME="billing_export" | |
# enable services apis | |
gcloud services enable compute.googleapis.com \ | |
bigquery.googleapis.com \ | |
bigquerydatatransfer.googleapis.com \ | |
--project $BILLING_PROJECT_ID | |
# create dataset | |
bq --location=$GCP_REGION mk -d \ | |
--description "Billing data export." \ | |
--project_id $BILLING_PROJECT_ID \ | |
$BILLING_DATASET_NAME | |
# Billing export (enabled in console and selected configured bucket) | |
######################################################## | |
# MONITORING (TBD) | |
######################################################## | |
# add projects to monitoring metrics scope | |
gcloud beta monitoring metrics-scopes create "projects/$DEVOPS_PROJECT_ID" \ | |
--project $MONITORING_PROJECT_ID | |
# repeat for engineering projects (less the AppHub mgmt project) | |
gcloud beta monitoring metrics-scopes create "projects/$SANDBOX_PROJECT_ID" \ | |
--project $MONITORING_PROJECT_ID | |
gcloud beta monitoring metrics-scopes create "projects/$DATA_SCIENCE_PROJECT_ID" \ | |
--project $MONITORING_PROJECT_ID | |
gcloud beta monitoring metrics-scopes create "projects/$BACKEND_STAGE_PROJECT_ID" \ | |
--project $MONITORING_PROJECT_ID | |
gcloud beta monitoring metrics-scopes create "projects/$BACKEND_PROD_PROJECT_ID" \ | |
--project $MONITORING_PROJECT_ID | |
gcloud beta monitoring metrics-scopes create "projects/$FRONTEND_STAGE_PROJECT_ID" \ | |
--project $MONITORING_PROJECT_ID | |
gcloud beta monitoring metrics-scopes create "projects/$FRONTEND_PROD_PROJECT_ID" \ | |
--project $MONITORING_PROJECT_ID | |
######################################################## | |
# DEVOPS | |
######################################################## | |
export TF_STATE_BUCKET_NAME="myco-tf-state" | |
export TF_STATE_BUCKET_LOCATION="US" | |
export TF_SERVICE_ACCOUNT_NAME="tf-admin" | |
export REPO_NAME="myco-docker-repo" | |
export DEVOPS_PROJECT_NUMBER=$(gcloud projects describe $DEVOPS_PROJECT_ID --format="value(projectNumber)") | |
export CLOUD_BUILD_SERVICE_AGENT="service-$DEVOPS_PROJECT_NUMBER@gcp-sa-cloudbuild.iam.gserviceaccount.com" | |
export CONNECTION_NAME="myco-source-repo" | |
# enable services apis | |
gcloud services enable compute.googleapis.com \ | |
storage.googleapis.com \ | |
artifactregistry.googleapis.com \ | |
cloudbuild.googleapis.com \ | |
secretmanager.googleapis.com \ | |
cloudresourcemanager.googleapis.com \ | |
--project $DEVOPS_PROJECT_ID | |
# create bucket | |
gcloud storage buckets create gs://$TF_STATE_BUCKET_NAME \ | |
--public-access-prevention \ | |
--uniform-bucket-level-access \ | |
--location $TF_STATE_BUCKET_LOCATION \ | |
--project $DEVOPS_PROJECT_ID | |
# update bucket (removed soft delete) | |
gcloud storage buckets update gs://$TF_STATE_BUCKET_NAME \ | |
--clear-soft-delete \ | |
--project $DEVOPS_PROJECT_ID | |
# create TF service account | |
gcloud iam service-accounts create $TF_SERVICE_ACCOUNT_NAME \ | |
--description="Terraform Admin Service Account" \ | |
--display-name="$TF_SERVICE_ACCOUNT_NAME" \ | |
--project $DEVOPS_PROJECT_ID | |
# grant roles TF should be able to perform with (permissive) | |
gcloud projects add-iam-policy-binding $DEVOPS_PROJECT_ID \ | |
--member="serviceAccount:$TF_SERVICE_ACCOUNT_NAME@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com" \ | |
--role="roles/storage.objectCreator" \ | |
--role="roles/iam.serviceAccountTokenCreator" \ | |
--role="roles/cloudbuild.builds.editor" \ | |
--role="roles/cloudbuild.connectionAdmin" \ | |
--role="roles/artifactregistry.admin" | |
gcloud resource-manager folders add-iam-policy-binding $ENGINEERING_FOLDER_ID \ | |
--member="serviceAccount:$TF_SERVICE_ACCOUNT_NAME@$DEVOPS_PROJECT_ID.iam.gserviceaccount.com" \ | |
--role="roles/apigateway.admin" \ | |
--role="roles/iam.serviceAccountTokenCreator" \ | |
--role="roles/iam.serviceAccountAdmin" \ | |
--role="roles/iam.securityAdmin" \ | |
--role="roles/servicenetworking.networksAdmin" \ | |
--role="roles/cloudfunctions.admin" \ | |
--role="roles/run.builder" \ | |
--role="roles/compute.instanceAdmin" \ | |
--role="roles/compute.loadBalancerAdmin" \ | |
--role="roles/compute.publicIpAdmin" \ | |
--role="roles/compute.storageAdmin" \ | |
--role="roles/compute.networkAdmin" \ | |
--role="roles/compute.securityAdmin" \ | |
--role="roles/monitoring.admin" \ | |
--role="roles/storage.admin" \ | |
--role="roles/pubsub.admin" \ | |
--role="roles/pubsublite.admin" \ | |
--role="roles/logging.admin" \ | |
--role="roles/secretmanager.admin" \ | |
--role="roles/dns.admin" \ | |
--role="roles/cloudkms.admin" \ | |
--role="roles/cloudtrace.admin" \ | |
--role="roles/cloudsql.admin" | |
# grant devops group ability to impersonate TF SA | |
gcloud projects add-iam-policy-binding $DEVOPS_PROJECT_ID \ | |
--member="group:$DEVOPS_GROUP" \ | |
--role="roles/iam.serviceAccountTokenCreator" \ | |
--condition=None | |
# grant cloud build service agent secret access | |
gcloud projects add-iam-policy-binding $DEVOPS_PROJECT_ID \ | |
--member="serviceAccount:$CLOUD_BUILD_SERVICE_AGENT" \ | |
--role="roles/secretmanager.admin" \ | |
--condition=None | |
# create github connection | |
gcloud builds connections create github $CONNECTION_NAME \ | |
--region=$GCP_REGION \ | |
--project $DEVOPS_PROJECT_ID | |
# link manually then confirm | |
gcloud builds connections describe $CONNECTION_NAME \ | |
--region $GCP_REGION \ | |
--project $DEVOPS_PROJECT_ID | |
# artifact registry | |
gcloud artifacts repositories create $REPO_NAME \ | |
--repository-format=docker \ | |
--location=$GCP_REGION \ | |
--description="Docker repository" \ | |
--project $DEVOPS_PROJECT_ID | |
# authorize local docker to push | |
gcloud auth configure-docker $GCP_REGION-docker.pkg.dev |
Organization Setup
This Gist is simply what I personally used to bootstrap the Google Cloud organization of my latest startup, PetPublish, prior to a friend stepping in to help IaC-ify our infra and help build our CI/CD pipelines, monitoring, tracing, etc.
At my prior company I authored many comprehensive and opinionated Google Cloud best practice guides / checklists, and other resources. You're welcome to check some of them out, which will hint towards what this Gist scaffolds.
What does this solve?
It makes your cloud posture more secure right from the start. Google Cloud obfuscates a lot of the cloud hosting complexity from users with some intelligent default settings so they can get started right away. Unfortunately, however, as you scale or need to enhance your security posture for compliance (or remediation), there are some best practices that should be followed. This applies those practices to the organization from the start, reducing the rework and pain later.
Key enhancements
- remove default org IAM roles (Project Creator and Billing Account User) that allow anyone in your domain to spin up resources
- often orgs later find all these projects from tutorials and training littered throughout their organization and have to clean up and this prevents that
- apply foundational org-wide security policies
- no external IP for VMs, etc. (unless explicitly removing public IPs, they are attached automatically)
- skip default networks (upon project creation default networks are generated w/ subnets in every region, using up IP ranges or risk collissions)
- disable keys (no long-lived keys created/downloaded by default)
- restrict IAM to organization (IAM users or groups must belong to org [no external])
- no public bucket access (restrict storage buckets from being exposed publicly)
- disable serial port access (prevent privilege escalation and bypassing security)
- ship audit logs to separate project
- in the event of a breach a bad actor may try to cover their tracks, but separately stored audit logs with separate permissions prevent laterally erasing actions taken up to the breach
- also often a requirement for compliance for fed, iso, and soc certifications
- ship billing data to separate project
- if you work with partners or 3rd-party products, switching sometimes can result in historical data loss
- by storing your historical billing data from the start, you can empower billing or FinOps teams, and reduce switching costs to remain provider agnostic
- centralize monitoring and alerts
- centralize devops (tf state, artifacts [optional])
Example org structure
This is how I organized my latest startup leveraging folders (like AWS OU) and projects (like AWS account). The shared-services folder includes standard projects I always include for accommodating the above. You can add them at any time but it's best to start and there's no cost to have projects, only the eventual costs for running resources, which are minimal (cheap storage mostly).
Centralized monitoring
I'm still researching the new Preview feature called App Hub and App-enabled Folders but we did allow app-enabled folders for the engineering
folder in the diagram above. Normally I centralize monitoring so future production support staff could gain access to troubleshoot without necessarily being granted access to production (another compliance thing).
Billing export to BigQuery
There is the possibly for more verbose pricing export as well, but I just chose the first 2 as illustrated to ship to our billing
project and BigQuery dataset billing_export
.
Additional suggestions / considerations
Cost control
AI security