Add support for Clickhouse (#10)

Clickhouse can now be optionally enabled by setting
`enable_clickhouse=true`. Similar to our Cloudformation stack, it is a
single EC2 instance.

**I no longer create an NLB or Autoscaling group for this like in
Cloudformation.**. Instead the lambdas will point directly at the EC2
instance. If we ever change or replace the EC2 host it will
automatically update all references to use the new IP. The NLB in this
case wasn't adding anything of value.

I believe I have a way to preserve the Clickhouse metadata EBS volume
between replacements too. This will allow us to change things like the
AMI or other params and recreate a new Clickhouse instance with a short
downtime but without losing any data.

The variables `use_external_clickhouse_address` in the root module,
`external_clickhouse_s3_bucket_name`, and `clickhouse_instance_count` in
the submodule are for a specific use case for one customer and shouldn't
be used by anyone else.

Other Changes:
* Allow passing `custom_domain` and `custom_certificate_arn` for
Cloudfront in the root module
* Randomize the RDS final snapshot suffix. (This is to support
create/destroy/create/destroy CI workflows)

Author: mdeeks

.gitignore

@@ -10,8 +10,8 @@ crash.log
 crash.*.log
 
 # Exclude all .tfvars files, which are likely to contain sensitive data, such as
-# password, private keys, and other secrets. These should not be part of version 
-# control as they are data points which are potentially sensitive and subject 
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
 # to change depending on the environment.
 *.tfvars
 *.tfvars.json
@@ -36,13 +36,15 @@ override.tf.json
 .terraformrc
 terraform.rc
 
-# Do not commit the terraform.tf and provider.tf files. This module should not have concrete 
+# Do not commit the terraform.tf and provider.tf files. This module should not have concrete
 # backend or provider configs. Users will set that up themselves.
-terraform.tf
-provider.tf
+/terraform.tf
+/provider.tf
+/modules/*/terraform.tf
+/modules/*/provider.tf
 
 # Ignore the braintrust-sandbox directory as it is used for internal testing
 braintrust-sandbox/*
 
 # Ignore IntelliJ IDEA project files
-.idea
+.idea

README.md

@@ -2,8 +2,6 @@
 
 This module is used to create the VPC, Databases, Lambdas, and associated resources for the self-hosted Braintrust data plane.
 
-**NOTE: This module is not yet ready for production use. It is still under development.**
-
 ## How to use this module
 
 To use this module, **copy the [`examples/braintrust-data-plane`](examples/braintrust-data-plane) directory to a new Terraform directory in your own repository**. Follow the instructions in the [`README.md`](examples/braintrust-data-plane/README.md) file in that directory to configure the module for your environment.

examples/braintrust-data-plane/main.tf

@@ -52,6 +52,12 @@ module "braintrust-data-plane" {
   # List of origins to whitelist for CORS
   # whitelisted_origins                   = []
 
+  # Custom domain name for the CloudFront distribution
+  # custom_domain                       = null
+
+  # ARN of the ACM certificate for the custom domain
+  # custom_certificate_arn              = null
+
   # The maximum number of requests per user allowed in the time frame specified by outbound_rate_limit_window_minutes. Setting to 0 will disable rate limits
   # outbound_rate_limit_max_requests      = 0
 

main.tf

@@ -89,17 +89,18 @@ module "services" {
   postgres_host     = module.database.postgres_database_address
   redis_host        = module.redis.redis_endpoint
   redis_port        = module.redis.redis_port
-  # TODO: Brainstore
-  # brainstore_hostname       = var.brainstore_hostname
-  # brainstore_port           = var.brainstore_port
-  # brainstore_s3_bucket_name = var.brainstore_s3_bucket_name
+
+  clickhouse_host      = try(module.clickhouse[0].clickhouse_instance_private_ip, null)
+  clickhouse_secret_id = try(module.clickhouse[0].clickhouse_secret_id, null)
 
   # Service configuration
   braintrust_org_name                 = var.braintrust_org_name
   api_handler_provisioned_concurrency = var.api_handler_provisioned_concurrency
   whitelisted_origins                 = var.whitelisted_origins
   outbound_rate_limit_window_minutes  = var.outbound_rate_limit_window_minutes
   outbound_rate_limit_max_requests    = var.outbound_rate_limit_max_requests
+  custom_domain                       = var.custom_domain
+  custom_certificate_arn              = var.custom_certificate_arn
 
   # Networking
   service_security_group_ids = [module.main_vpc.default_security_group_id]
@@ -121,3 +122,17 @@ module "services" {
 
   kms_key_arn = local.kms_key_arn
 }
+
+module "clickhouse" {
+  source = "./modules/clickhouse"
+  count  = var.enable_clickhouse ? 1 : 0
+
+  deployment_name                  = var.deployment_name
+  clickhouse_instance_count        = var.use_external_clickhouse_address ? 0 : 1
+  clickhouse_instance_type         = var.clickhouse_instance_type
+  clickhouse_metadata_storage_size = var.clickhouse_metadata_storage_size
+  clickhouse_subnet_id             = module.main_vpc.private_subnet_1_id
+  clickhouse_security_group_ids    = [module.main_vpc.default_security_group_id]
+
+  kms_key_arn = local.kms_key_arn
+}

modules/clickhouse/iam.tf

@@ -0,0 +1,50 @@
+resource "aws_iam_instance_profile" "clickhouse" {
+  name = "${var.deployment_name}-ClickhouseInstanceProfile"
+  role = aws_iam_role.clickhouse.name
+}
+
+resource "aws_iam_role" "clickhouse" {
+  name = "${var.deployment_name}-ClickhouseRole"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect = "Allow"
+      Principal = {
+        Service = "ec2.amazonaws.com"
+      }
+      Action = "sts:AssumeRole"
+    }]
+  })
+}
+
+resource "aws_iam_role_policy" "clickhouse_secret_access" {
+  name = "AccessSecret"
+  role = aws_iam_role.clickhouse.id
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect   = "Allow"
+      Action   = "secretsmanager:GetSecretValue"
+      Resource = aws_secretsmanager_secret.clickhouse_secret.arn
+    }]
+  })
+}
+
+resource "aws_iam_role_policy" "clickhouse_s3_access" {
+  name = "AccessS3Bucket"
+  role = aws_iam_role.clickhouse.id
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect = "Allow"
+      Action = "s3:*"
+      Resource = [
+        "arn:aws:s3:::${local.clickhouse_bucket_name}",
+        "arn:aws:s3:::${local.clickhouse_bucket_name}/*"
+      ]
+    }]
+  })
+}
+
+

modules/clickhouse/main.tf

@@ -0,0 +1,118 @@
+locals {
+  clickhouse_bucket_name = var.external_clickhouse_s3_bucket_name == null ? aws_s3_bucket.clickhouse_s3_bucket[0].id : var.external_clickhouse_s3_bucket_name
+}
+
+data "aws_region" "current" {}
+
+data "aws_ami" "amazon_linux_2" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["amzn2-ami-hvm-*-x86_64-gp2"]
+  }
+}
+
+data "aws_subnet" "clickhouse_subnet" {
+  id = var.clickhouse_subnet_id
+}
+
+resource "aws_launch_template" "clickhouse" {
+  count         = var.clickhouse_instance_count
+  name          = "${var.deployment_name}-clickhouse"
+  image_id      = data.aws_ami.amazon_linux_2.id
+  instance_type = var.clickhouse_instance_type
+
+  iam_instance_profile {
+    name = aws_iam_instance_profile.clickhouse.name
+  }
+
+  metadata_options {
+    http_endpoint               = "enabled"
+    http_tokens                 = "required"
+    http_put_response_hop_limit = 1
+  }
+
+  network_interfaces {
+    delete_on_termination = true
+    security_groups       = var.clickhouse_security_group_ids
+    subnet_id             = var.clickhouse_subnet_id
+  }
+
+  block_device_mappings {
+    device_name = "/dev/xvda"
+    ebs {
+      volume_size           = 128
+      volume_type           = "gp3"
+      encrypted             = true
+      kms_key_id            = var.kms_key_arn
+      delete_on_termination = true
+    }
+  }
+
+  key_name = var.clickhouse_instance_key_pair_name
+
+  user_data = base64encode(templatefile("${path.module}/user_data.sh", {
+    aws_region                   = data.aws_region.current.name
+    s3_bucket_name               = local.clickhouse_bucket_name
+    clickhouse_secret_id         = aws_secretsmanager_secret.clickhouse_secret.arn
+    clickhouse_secret_version_id = aws_secretsmanager_secret_version.clickhouse_secret.version_id
+  }))
+
+  tag_specifications {
+    resource_type = "instance"
+    tags = {
+      Name = "${var.deployment_name}-clickhouse"
+    }
+  }
+
+  tag_specifications {
+    resource_type = "volume"
+    tags = {
+      Name = "${var.deployment_name}-clickhouse"
+    }
+  }
+
+  tag_specifications {
+    resource_type = "network-interface"
+    tags = {
+      Name = "${var.deployment_name}-clickhouse"
+    }
+  }
+}
+
+resource "aws_instance" "clickhouse" {
+  count = var.clickhouse_instance_count
+  launch_template {
+    id      = aws_launch_template.clickhouse[0].id
+    version = "$Latest"
+  }
+}
+
+# EBS volume for Clickhouse metadata. This needs to be preserved across instances.
+resource "aws_ebs_volume" "clickhouse_metadata" {
+  count             = var.clickhouse_instance_count
+  availability_zone = data.aws_subnet.clickhouse_subnet.availability_zone
+  size              = var.clickhouse_metadata_storage_size
+  type              = "gp3"
+  encrypted         = true
+  kms_key_id        = var.kms_key_arn
+
+  tags = {
+    Name = "${var.deployment_name}-clickhouse-metadata"
+  }
+  lifecycle {
+    prevent_destroy = true
+  }
+}
+
+resource "aws_volume_attachment" "clickhouse_metadata" {
+  count       = var.clickhouse_instance_count
+  device_name = "/dev/sdf"
+  volume_id   = aws_ebs_volume.clickhouse_metadata[0].id
+  instance_id = aws_instance.clickhouse[0].id
+
+  # This is a workaround to ensure the volume is attached after the instance is created.
+  depends_on = [aws_instance.clickhouse]
+}

modules/clickhouse/outputs.tf

@@ -0,0 +1,23 @@
+output "clickhouse_instance_id" {
+  value = try(aws_instance.clickhouse[0].id, null)
+}
+
+output "clickhouse_instance_private_ip" {
+  value = try(aws_instance.clickhouse[0].private_ip, null)
+}
+
+output "clickhouse_secret_arn" {
+  value = aws_secretsmanager_secret.clickhouse_secret.arn
+}
+
+output "clickhouse_s3_bucket_name" {
+  value = try(aws_s3_bucket.clickhouse_s3_bucket[0].id, var.external_clickhouse_s3_bucket_name)
+}
+
+output "clickhouse_secret_version_id" {
+  # This is unfortunately needed because the AWSCURRENT version stage appears to be eventually consistent.
+  # If you try get the secret right after creating it, you will get an error saying AWSCURRENT doesn't exist.
+  # So instead you must point directly at the version_id
+  description = "The ID of the secret version"
+  value       = aws_secretsmanager_secret_version.clickhouse_secret.version_id
+}

modules/clickhouse/s3.tf

@@ -0,0 +1,22 @@
+resource "aws_s3_bucket" "clickhouse_s3_bucket" {
+  count         = var.external_clickhouse_s3_bucket_name == null ? 1 : 0
+  bucket_prefix = "${var.deployment_name}-clickhouse"
+
+  lifecycle {
+    # S3 does not support renaming buckets
+    ignore_changes = [bucket_prefix]
+  }
+}
+
+resource "aws_s3_bucket_server_side_encryption_configuration" "clickhouse_s3_bucket" {
+  count  = var.external_clickhouse_s3_bucket_name == null ? 1 : 0
+  bucket = aws_s3_bucket.clickhouse_s3_bucket[0].id
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm     = var.kms_key_arn != null ? "aws:kms" : "AES256"
+      kms_master_key_id = var.kms_key_arn
+    }
+    bucket_key_enabled = var.kms_key_arn != null
+  }
+}

modules/clickhouse/secrets.tf

@@ -0,0 +1,19 @@
+resource "aws_secretsmanager_secret" "clickhouse_secret" {
+  name_prefix = "${var.deployment_name}/ClickhouseSecret-"
+  kms_key_id  = var.kms_key_arn
+}
+
+data "aws_secretsmanager_random_password" "clickhouse_secret" {
+  exclude_characters  = "\"'@/\\"
+  exclude_punctuation = true
+  password_length     = 32
+}
+
+resource "aws_secretsmanager_secret_version" "clickhouse_secret" {
+  secret_id     = aws_secretsmanager_secret.clickhouse_secret.id
+  secret_string = data.aws_secretsmanager_random_password.clickhouse_secret.random_password
+
+  lifecycle {
+    ignore_changes = [secret_string]
+  }
+}