diff --git a/infra/dcp/main.tf b/infra/dcp/main.tf index c8365a7..257c4ff 100644 --- a/infra/dcp/main.tf +++ b/infra/dcp/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.0" + version = ">= 5.11.0" } null = { source = "hashicorp/null" @@ -137,6 +137,7 @@ module "cdc" { use_spanner = var.enable_dcp spanner_instance_id = var.enable_dcp ? module.dcp[0].spanner_instance_id : "" spanner_database_id = var.enable_dcp ? module.dcp[0].spanner_database_id : "" + workflow_name = var.enable_dcp && var.dcp_deploy_data_ingestion_workflow ? module.dcp[0].ingestion_orchestrator_name : "" deletion_protection = var.deletion_protection depends_on = [google_project_service.apis] diff --git a/infra/dcp/modules/cdc/main.tf b/infra/dcp/modules/cdc/main.tf index 4aa4823..edee1de 100644 --- a/infra/dcp/modules/cdc/main.tf +++ b/infra/dcp/modules/cdc/main.tf @@ -183,6 +183,26 @@ resource "google_cloud_run_v2_job" "dc_data_job" { name = "INPUT_DIR" value = "gs://${google_storage_bucket.data_bucket.name}/${var.gcs_data_bucket_input_folder}" } + env { + name = "WORKFLOW_NAME" + value = var.workflow_name + } + env { + name = "PROJECT_ID" + value = var.project_id + } + env { + name = "WORKFLOW_LOCATION" + value = var.region + } + env { + name = "TEMP_LOCATION" + value = "gs://${google_storage_bucket.data_bucket.name}/temp" + } + env { + name = "REGION" + value = var.region + } } vpc_access { connector = google_vpc_access_connector.connector.id diff --git a/infra/dcp/modules/cdc/service_account.tf b/infra/dcp/modules/cdc/service_account.tf index 2a6b44a..e2669b5 100644 --- a/infra/dcp/modules/cdc/service_account.tf +++ b/infra/dcp/modules/cdc/service_account.tf @@ -14,7 +14,8 @@ resource "google_project_iam_member" "datacommons_service_account_roles" { "roles/vpcaccess.user", "roles/iam.serviceAccountUser", "roles/secretmanager.secretAccessor", - "roles/spanner.databaseUser" + "roles/spanner.databaseUser", + "roles/workflows.invoker" ]), var.use_spanner ? [] : ["roles/spanner.databaseUser"]) project = var.project_id member = "serviceAccount:${google_service_account.datacommons_service_account.email}" @@ -46,3 +47,17 @@ resource "google_secret_manager_secret_version" "maps_api_key_version" { secret = google_secret_manager_secret.maps_api_key[0].id secret_data = local.maps_api_key } + +resource "google_storage_bucket_iam_member" "cdc_data_bucket_access" { + bucket = google_storage_bucket.data_bucket.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${google_service_account.datacommons_service_account.email}" +} + +resource "google_storage_bucket_iam_member" "dataflow_bucket_access" { + bucket = google_storage_bucket.data_bucket.name + role = "roles/storage.objectAdmin" + member = "serviceAccount:${local.name_prefix}dcp-ingestion-sa@${var.project_id}.iam.gserviceaccount.com" +} + + diff --git a/infra/dcp/modules/dcp/ingestion_helper.tf b/infra/dcp/modules/dcp/ingestion_helper.tf index f6db013..8c42a5d 100644 --- a/infra/dcp/modules/dcp/ingestion_helper.tf +++ b/infra/dcp/modules/dcp/ingestion_helper.tf @@ -25,7 +25,7 @@ resource "google_cloud_run_v2_service" "ingestion_helper" { } env { name = "SPANNER_INSTANCE_ID" - value = var.spanner_instance_id + value = var.create_spanner_instance ? google_spanner_instance.main[0].name : var.spanner_instance_id } env { name = "SPANNER_DATABASE_ID" diff --git a/infra/dcp/modules/dcp/outputs.tf b/infra/dcp/modules/dcp/outputs.tf index 4128477..fd50e51 100644 --- a/infra/dcp/modules/dcp/outputs.tf +++ b/infra/dcp/modules/dcp/outputs.tf @@ -22,3 +22,9 @@ output "data_ingestion_bucket_url" { description = "GCS path to the dynamically provisioned bucket for customer custom MCF datasets" value = var.deploy_data_ingestion_workflow && var.create_ingestion_bucket ? google_storage_bucket.data_ingestion_bucket[0].url : null } + +output "ingestion_orchestrator_name" { + description = "Short name of the Cloud Workflows ingestion orchestrator" + value = var.deploy_data_ingestion_workflow ? google_workflows_workflow.ingestion_orchestrator[0].name : null +} + diff --git a/infra/dcp/modules/dcp/spanner.tf b/infra/dcp/modules/dcp/spanner.tf index 127795b..15ba2f9 100644 --- a/infra/dcp/modules/dcp/spanner.tf +++ b/infra/dcp/modules/dcp/spanner.tf @@ -5,6 +5,7 @@ resource "google_spanner_instance" "main" { display_name = var.create_spanner_instance ? (var.spanner_instance_id != "" ? "${local.name_prefix}${var.spanner_instance_id}" : "${local.name_prefix}dcp-instance") : var.spanner_instance_id processing_units = var.spanner_processing_units force_destroy = !var.deletion_protection + edition = "ENTERPRISE" } diff --git a/infra/dcp/variables.tf b/infra/dcp/variables.tf index c39c5b1..4066735 100644 --- a/infra/dcp/variables.tf +++ b/infra/dcp/variables.tf @@ -93,7 +93,7 @@ variable "dcp_spanner_database_id" { variable "dcp_spanner_processing_units" { description = "Spanner units for DCP" type = number - default = 100 + default = 1000 } variable "dcp_service_cpu" {