Pulumi

Getting started with Pulumi

You can use this Pulumi Typescript function to set up the IAM role, IAM policies, SNS topic, and S3 bucket to integrate with Scanner.

You provide the value of dataLakeS3Bucket to indicate which S3 bucket you want Scanner to index. Scanner will provide the values for:

• scannerInstanceStsExternalId

• scannerInstanceAwsAccountId

• scannerInstanceSqsIndexQueueArn

import * as aws from "@pulumi/aws";
import * as pulumi from "@pulumi/pulumi";

export function setUpScannerInfra(
  dataLakeS3Bucket: aws.s3.Bucket,
  scannerInstanceStsExternalId: string,
  scannerInstanceAwsAccountId: string,
  scannerInstanceSqsIndexQueueArn: string,
): { scannerRole: aws.iam.Role } {
  const scannerRole = new aws.iam.Role("ScannerRole", {
    name: "ScannerRole",
    assumeRolePolicy: aws.iam.getPolicyDocumentOutput({
      statements: [
        {
          actions: ["sts:AssumeRole"],
          effect: "Allow",
          principals: [{
            type: "AWS",
            identifiers: [scannerInstanceAwsAccountId],
          }],
          conditions: [{
            test: "StringEquals",
            variable: "sts:ExternalId",
            values: [scannerInstanceStsExternalId],
          }],
        },
      ],
    }).json,
  });

  const logsBucketsNotificationTopic = new aws.sns.Topic(
    "LogsBucketsNotificationTopic",
    {
      policy: aws.iam.getPolicyDocumentOutput(
        {
          statements: [
            {
              effect: "Allow",
              principals: [
                {
                  type: "Service",
                  identifiers: ["s3.amazonaws.com"],
                },
              ],
              actions: ["SNS:Publish"],
              resources: ["*"],
            },
          ],
        },
        { provider },
      ).json,
    },
  );

  new aws.sns.TopicSubscription(
    "ScannerLogsBucketsNotificationTopicSubscription",
    {
      topic: logsBucketsNotificationTopic.arn,
      protocol: "sqs",
      endpoint: scannerInstanceSqsIndexQueueArn,
      rawMessageDelivery: true,
    },
  );

  new aws.s3.BucketNotification("DataLakeBucketNotification", {
    bucket: dataLakeS3Bucket.id,
    topics: [{
      topicArn: logsBucketsNotificationTopic.arn,
      events: ["s3:ObjectCreated:*"],
    }],
  });

  const scannerIndexFilesS3BucketName = `scanner-index-files-${scannerInstanceStsExternalId}`;
  const scannerIndexFilesS3Bucket = new aws.s3.Bucket("ScannerIndexFilesBucket", {
    bucket: scannerIndexFilesS3BucketName,
    acl: "private",
    lifecycleRules: [
      {
        id: "ExpireTagging",
        enabled: true,
        tags: {
          'Scnr-Lifecycle': 'expire',
        },
        expiration: {
          days: 1,
        }
      },
      {
        id: "AbortIncompleteMultiPartUploads",
        enabled: true,
        abortIncompleteMultipartUploadDays: 1,
      },
    ],
    serverSideEncryptionConfiguration: {
      rule: {
        applyServerSideEncryptionByDefault: {
          sseAlgorithm: "aws:kms",
        },
        bucketKeyEnabled: true,
      }
    },
  }, {});

  new aws.s3.BucketPublicAccessBlock("ScannerIndexFilesBucketPublicAccessBlock", {
    bucket: scannerIndexFilesS3Bucket.id,
    blockPublicAcls: true,
    blockPublicPolicy: true,
    ignorePublicAcls: true,
    restrictPublicBuckets: true,
  });

  const scannerPolicy = new aws.iam.Policy("ScannerPolicy", {
    description: "Allow ScannerRole to interact with data lake S3 bucket",
    path: "/",
    policy: aws.iam.getPolicyDocumentOutput({
      statements: [
        {
          actions: [
            "s3:ListAllMyBuckets",
            "s3:GetBucketLocation",
            "s3:GetBucketTagging",
          ],
          effect: "Allow",
          resources: ["*"],
        },
        {
          actions: ["s3:ListBucket"],
          effect: "Allow",
          resources: [
            dataLakeS3Bucket.arn,
          ],
        },
        {
          actions: ["s3:GetObject"],
          effect: "Allow",
          resources: [
            pulumi.interpolate `${dataLakeS3Bucket.arn}/*`,
          ],
        },
        {
          actions: ["s3:*"],
          effect: "Allow",
          resources: [
            scannerIndexFilesS3Bucket.arn,
            pulumi.interpolate `${scannerIndexFilesS3Bucket.arn}/*`
          ],
        },
      ],
    }).json,
  });

  new aws.iam.RolePolicyAttachment("ScannerRpa", {
    role: scannerRole.name,
    policyArn: scannerPolicy.arn,
  });

  return {
    scannerRole,
  };
}

You can include this function in your Pulumi codebase or use it as a starting point as you update your infrastructure code to support what Scanner needs.

If your S3 buckets are in multiple regions

If the S3 buckets that you want to index are in multiple regions, edit this Pulumi file to do the following:

• Create one aws.sns.Topic per region.

• Create an aws.sns.TopicSubscription for each SNS topic, all pointing to the same SQS queue in your Scanner instance.

• Create an aws.s3.BucketNotification for each S3 bucket, and point it to the SNS topic that is in the same region as the S3 bucket.