Terraform

Getting started with Terraform

You can use this Terraform file to set up the IAM role, IAM policies, an SNS topic, and an S3 bucket to integrate with Scanner.

You can get the values for scanner_aws_account_id and scanner_external_id from Scanner under Settings > AWS Accounts. You can view these values during the Link New Account flow, or by viewing the details of a specific AWS account.

It will also create a new SNS topic and update your S3 bucket to send s3:ObjectCreated notifications to this topic. The SNS topic will then relay those notifications to the SQS queue in your Scanner instance.

Providing Variable Values

Create a terraform.tfvars file in the same directory as your Terraform configuration to provide values for the variables:

region                   = "us-east-1"
scanner_aws_account_id   = "123456789012"
scanner_external_id      = "your-external-id"

s3_collect_rule_destination_buckets = ["my-logs-bucket-1", "my-logs-bucket-2"]
s3_index_rule_source_buckets        = ["my-logs-bucket-3", "my-logs-bucket-4"]

# Only needed if using customer-managed KMS keys
s3_collect_rule_destination_buckets_kms_key_arns = []
s3_index_rule_source_buckets_kms_key_arns        = []

Then run terraform apply and Terraform will automatically load values from terraform.tfvars.

Terraform Configuration

variable "region" {
  description = "The AWS region where resources will be created."
  type        = string
}

variable "scanner_aws_account_id" {
  description = "Scanner provides its AWS Account ID to use here."
  type        = string
}

variable "scanner_external_id" {
  description = "Scanner provides an External ID to use here."
  type        = string
}

variable "s3_collect_rule_destination_buckets" {
  description = "Names of S3 buckets used as destinations for Collect Rules (i.e. Scanner will write your raw logs into)."
  type        = list(string)
  default     = []
}

variable "s3_index_rule_source_buckets" {
  description = "Names of S3 buckets used as the source of Index Rules (i.e. from which the raw logs will be ingested to Scanner indexes and become queryable). Include the bucket in both if it is also a Collect Rule destination (i.e. the logs will go all the way from the source -> S3 -> a Scanner index)."
  type        = list(string)
  default     = []
}

variable "s3_collect_rule_destination_buckets_kms_key_arns" {
  description = "Enter the ARNs of the KMS keys used to encrypt any of the linked S3 buckets. Not needed if the buckets do not use KMS keys."
  type        = list(string)
  default     = []
}

variable "s3_index_rule_source_buckets_kms_key_arns" {
  description = "Enter the ARNs of the KMS keys used to encrypt any of the linked S3 buckets. Not needed if the buckets do not use KMS keys."
  type        = list(string)
  default     = []
}

provider "aws" {
  region = var.region
}

# The IAM role used by Scanner to access the S3 files
resource "aws_iam_role" "scnr_scanner_role" {
  name = "scnr-ScannerRole"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect = "Allow"
        Principal = {
          AWS = var.scanner_aws_account_id
        }
        Action = "sts:AssumeRole"
        Condition = {
          StringEquals = {
            "sts:ExternalId" = var.scanner_external_id
          }
        }
      }
    ]
  })
}


# The S3 bucket where Scanner will store all index files
resource "aws_s3_bucket" "scnr_index_files_bucket" {
  bucket = "scnr-index-files-${var.scanner_external_id}"
  // NOTE: With this flag, Terraform will disallow the deletion of the entire
  // stack. Unfortunately Terraform does not yet provide a way to protect only
  // some resources while deleting the rest.
  lifecycle {
    prevent_destroy = true
  }
}

resource "aws_s3_bucket_public_access_block" "scnr_index_files_bucket_public_access_block" {
  bucket = aws_s3_bucket.scnr_index_files_bucket.id

  block_public_acls       = true
  block_public_policy     = true
  ignore_public_acls      = true
  restrict_public_buckets = true
}

resource "aws_s3_bucket_server_side_encryption_configuration" "scnr_index_files_bucket_encryption_config" {
  bucket = aws_s3_bucket.scnr_index_files_bucket.id

  rule {
    bucket_key_enabled = true
    apply_server_side_encryption_by_default {
      sse_algorithm = "aws:kms"
    }
  }
}

resource "aws_s3_bucket_lifecycle_configuration" "scnr_index_files_bucket_lifecycle_configuration" {
  bucket = aws_s3_bucket.scnr_index_files_bucket.id

  rule {
    id     = "ExpireTagging"
    status = "Enabled"
    filter {
      tag {
        key   = "Scnr-Lifecycle"
        value = "expire"
      }
    }
    expiration {
      days = 1
    }
  }

  rule {
    id     = "AbortIncompleteMultiPartUploads"
    status = "Enabled"
    filter {}
    abort_incomplete_multipart_upload {
      days_after_initiation = 1
    }
  }
}

# The SNS topic to notify Scanner when the log files buckets have new files
resource "aws_sns_topic" "scnr_logs_bucket_event_notification_topic" {
  name = "scnr-LogsBucketEventNotificationTopic"

  policy = data.aws_iam_policy_document.scnr_logs_bucket_event_notification_topic_policy.json
}

data "aws_iam_policy_document" "scnr_logs_bucket_event_notification_topic_policy" {
  statement {
    principals {
      type        = "Service"
      identifiers = ["s3.amazonaws.com"]
    }

    effect    = "Allow"
    actions   = ["SNS:Publish"]
    resources = ["*"]
  }
}

resource "aws_sns_topic_subscription" "scnr_logs_bucket_event_notification_topic_subscription" {
  topic_arn            = aws_sns_topic.scnr_logs_bucket_event_notification_topic.arn
  protocol             = "sqs"
  endpoint             = "arn:aws:sqs:${data.aws_region.current.name}:${var.scanner_aws_account_id}:scnr-S3ObjectCreatedNotificationsQueue"
  raw_message_delivery = true
}

# Create one notification configuration for each bucket
resource "aws_s3_bucket_notification" "logs_bucket_event_notification" {
  for_each = toset(var.s3_index_rule_source_buckets)
  bucket = each.value

  topic {
    topic_arn = aws_sns_topic.scnr_logs_bucket_event_notification_topic.arn
    events    = ["s3:ObjectCreated:*"]
  }
}

# The IAM policy allowing the Scanner IAM role to access the S3 buckets
resource "aws_iam_role_policy" "scnr_scanner_policy" {
  name   = "scnr-ScannerRolePolicy"
  role   = aws_iam_role.scnr_scanner_role.name
  policy = data.aws_iam_policy_document.scnr_scanner_policy_document.json
}

data "aws_iam_policy_document" "scnr_scanner_policy_document" {
  # Basic S3 permissions for all buckets
  statement {
    actions = [
      "s3:GetBucketLocation",
      "s3:GetBucketTagging",
      "s3:ListAllMyBuckets",
    ]
    resources = ["*"]
  }

  # Scanner will write the processed index files to the Index Files Bucket.
  # Delete permission is needed as Scanner compacts and expires files in this
  # bucket periodically.
  statement {
    actions = [
      "s3:DeleteObject",
      "s3:DeleteObjectTagging",
      "s3:DeleteObjectVersion",
      "s3:DeleteObjectVersionTagging",
      "s3:GetEncryptionConfiguration",
      "s3:GetLifecycleConfiguration",
      "s3:GetObject",
      "s3:GetObjectTagging",
      "s3:ListBucket",
      "s3:PutObject",
      "s3:PutObjectTagging",
    ]
    resources = [
      "arn:aws:s3:::scnr-index-files-${var.scanner_external_id}",
      "arn:aws:s3:::scnr-index-files-${var.scanner_external_id}/*"
    ]
  }

  # As destinations for Collect Rules, Scanner will write raw log files into
  # these buckets.
  dynamic "statement" {
    for_each = length(var.s3_collect_rule_destination_buckets) > 0 ? [1] : []
    content {
      actions = [
        "s3:GetBucketNotification",
        "s3:GetEncryptionConfiguration",
        "s3:GetLifecycleConfiguration",
        "s3:GetObject",
        "s3:GetObjectTagging",
        "s3:ListBucket",
        "s3:PutObject",
        "s3:PutObjectTagging",
      ]
      # Generate ARNs for each bucket in the list
      resources = flatten([
        for bucket in var.s3_collect_rule_destination_buckets : [
          "arn:aws:s3:::${bucket}",
          "arn:aws:s3:::${bucket}/*"
        ]
      ])
    }
  }

  # Scanner will test the integration by writing/deleting test files into/from
  # Collect Rule destination buckets.
  dynamic "statement" {
    for_each = length(var.s3_collect_rule_destination_buckets) > 0 ? [1] : []
    content {
      actions = [
        "s3:DeleteObject",
        "s3:DeleteObjectTagging",
        "s3:DeleteObjectVersion",
        "s3:DeleteObjectVersionTagging",
      ]
      # Generate ARNs for test files in each bucket
      resources = flatten([
        for bucket in var.s3_collect_rule_destination_buckets : [
          "arn:aws:s3:::${bucket}/*ScannerTestFile"
        ]
      ])
    }
  }

  # As sources for Index Rules, Scanner will read the raw log files from these
  # buckets.
  dynamic "statement" {
    for_each = length(var.s3_index_rule_source_buckets) > 0 ? [1] : []
    content {
      actions = [
        "s3:GetBucketNotification",
        "s3:GetEncryptionConfiguration",
        "s3:GetObject",
        "s3:GetObjectTagging",
        "s3:ListBucket",
      ]
      # Generate ARNs for each bucket in the list
      resources = flatten([
        for bucket in var.s3_index_rule_source_buckets : [
          "arn:aws:s3:::${bucket}",
          "arn:aws:s3:::${bucket}/*"
        ]
      ])
    }
  }

  # If a Collect Rule destination bucket is encrypted by a Customer Managed KMS
  # key, the Scanner Role will need permission to encrypt with the key when
  # writing raw log files into the bucket.
  dynamic "statement" {
    for_each = length(var.s3_collect_rule_destination_buckets_kms_key_arns) > 0 ? [1] : []
    content {
      actions   = [
        "kms:DescribeKey",
        "kms:GenerateDataKey",
      ]
      resources = var.s3_collect_rule_destination_buckets_kms_key_arns
    }
  }

  # If an Index Rule source bucket is encrypted by a Customer Managed KMS key,
  # the Scanner Role will need permission to decrypt with the key when
  # reading/ingesting the raw log files from the bucket.
  dynamic "statement" {
    for_each = length(var.s3_index_rule_source_buckets_kms_key_arns) > 0 ? [1] : []
    content {
      actions   = [
        "kms:Decrypt",
        "kms:DescribeKey",
      ]
      resources = var.s3_index_rule_source_buckets_kms_key_arns
    }
  }
}

output "scanner_role_arn" {
  description = "The ARN of the new Scanner IAM Role"
  value       = aws_iam_role.scnr_scanner_role.arn
}

data "aws_caller_identity" "current" {}

If your S3 buckets are in multiple regions

If the S3 buckets that you want to index are in multiple regions, edit this Terraform file to do the following:

• Create one aws_sns_topic per region.

• Create an aws_sns_topic_subscription for each SNS topic, all pointing to the same SQS queue in your Scanner instance.

• Create an aws_s3_bucket_notification for each S3 bucket, and point it to the SNS topic that is in the same region as the S3 bucket.

Note that cross-region ingestion incurs a small data transfer cost on your S3 bucket. More information here.

(Optional) If you want to use KMS encryption for your SNS notification topic

• It must be a customer-managed key.

• The key policy must allow usage by S3 as described here.