Created
February 1, 2024 20:37
-
-
Save timvw/52f742553f691c92e69c33fc3102c2a6 to your computer and use it in GitHub Desktop.
Build infra for creating iceberg tables in snowflake with the terraform provider
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
resource "snowflake_unsafe_execute" "volume" { | |
execute = <<EOT | |
CREATE OR REPLACE EXTERNAL VOLUME ${var.external_volume_name} | |
STORAGE_LOCATIONS = | |
( | |
( | |
NAME = '${var.external_volume_bucket_name}' | |
STORAGE_PROVIDER = 'S3' | |
STORAGE_BASE_URL = 's3://${var.external_volume_bucket_name}' | |
STORAGE_AWS_ROLE_ARN = '${var.snowflake_iam_role_arn}' | |
ENCRYPTION=(TYPE='AWS_SSE_KMS') | |
) | |
) | |
EOT | |
revert = <<EOT | |
DROP EXTERNAL VOLUME IF EXISTS ${var.external_volume_name} | |
EOT | |
query = <<EOT | |
DESC EXTERNAL VOLUME ${var.external_volume_name} | |
EOT | |
} | |
resource "snowflake_unsafe_execute" "catalog" { | |
execute = <<EOT | |
CREATE or replace CATALOG INTEGRATION ${var.catalog_name} | |
CATALOG_SOURCE=GLUE | |
CATALOG_NAMESPACE='${var.catalog_namespace}' | |
TABLE_FORMAT=ICEBERG | |
GLUE_AWS_ROLE_ARN='${var.snowflake_iam_role_arn}' | |
GLUE_CATALOG_ID='${var.catalog_id}' | |
ENABLED=TRUE | |
EOT | |
revert = <<EOT | |
DROP CATALOG INTEGRATION IF EXISTS ${var.catalog_name} | |
EOT | |
query = <<EOT | |
DESCRIBE CATALOG INTEGRATION ${var.catalog_name} | |
EOT | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
output "volume_aws_iam_user_arn" { | |
value = jsondecode(snowflake_unsafe_execute.volume.query_results[index(snowflake_unsafe_execute.volume.query_results.*.property, "STORAGE_LOCATION_1")].property_value).STORAGE_AWS_IAM_USER_ARN | |
} | |
output "volume_external_id" { | |
value = jsondecode(snowflake_unsafe_execute.volume.query_results[index(snowflake_unsafe_execute.volume.query_results.*.property, "STORAGE_LOCATION_1")].property_value).STORAGE_AWS_EXTERNAL_ID | |
} | |
output "catalog_aws_iam_user_arn" { | |
value = snowflake_unsafe_execute.catalog.query_results[index(snowflake_unsafe_execute.catalog.query_results.*.property, "GLUE_AWS_IAM_USER_ARN")].property_value | |
} | |
output "catalog_external_id" { | |
value = snowflake_unsafe_execute.catalog.query_results[index(snowflake_unsafe_execute.catalog.query_results.*.property, "GLUE_AWS_EXTERNAL_ID")].property_value | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
variable "external_volume_name" { | |
type = string | |
description = "Name of the external volume" | |
} | |
variable "external_volume_bucket_name" { | |
type = string | |
description = "Name of an S3 bucket that stores your data files" | |
} | |
variable "snowflake_iam_role_arn" { | |
type = string | |
description = "Specifies the Amazon Resource Name (ARN) of the AWS identity and access management (IAM) role that grants privileges on the S3 bucket containing your data files" | |
} | |
variable "catalog_name" { | |
type = string | |
description = "String that specifies the identifier (name) for the catalog integration" | |
} | |
variable "catalog_id" { | |
type = string | |
description = "Specifies the ID of your AWS account" | |
} | |
variable "catalog_namespace" { | |
type = string | |
description = "Specifies your AWS Glue Data Catalog namespace (for example, my_glue_database). This is the default namespace for all Iceberg tables that you associate with this catalog integration. You can override this value by specifying the namespace at the table level when you create a table." | |
default = "default" | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment