provider/aws: Add support for EMR clusters to aws_appautoscaling_target (#13368)
``` ```
This commit is contained in:
Paul Stack
GitHub
parent
5be2a50b05
commit
3574052c8a
|
|
@ -20,33 +20,33 @@ func resourceAwsAppautoscalingTarget() *schema.Resource {
|
|||
Delete: resourceAwsAppautoscalingTargetDelete,
|
||||
|
||||
Schema: map[string]*schema.Schema{
|
||||
"max_capacity": &schema.Schema{
|
||||
"max_capacity": {
|
||||
Type: schema.TypeInt,
|
||||
Required: true,
|
||||
ForceNew: true,
|
||||
},
|
||||
"min_capacity": &schema.Schema{
|
||||
"min_capacity": {
|
||||
Type: schema.TypeInt,
|
||||
Required: true,
|
||||
ForceNew: true,
|
||||
},
|
||||
"resource_id": &schema.Schema{
|
||||
"resource_id": {
|
||||
Type: schema.TypeString,
|
||||
Required: true,
|
||||
ForceNew: true,
|
||||
},
|
||||
"role_arn": &schema.Schema{
|
||||
"role_arn": {
|
||||
Type: schema.TypeString,
|
||||
Required: true,
|
||||
ForceNew: true,
|
||||
},
|
||||
"scalable_dimension": &schema.Schema{
|
||||
"scalable_dimension": {
|
||||
Type: schema.TypeString,
|
||||
Required: true,
|
||||
ForceNew: true,
|
||||
ValidateFunc: validateAppautoscalingScalableDimension,
|
||||
},
|
||||
"service_namespace": &schema.Schema{
|
||||
"service_namespace": {
|
||||
Type: schema.TypeString,
|
||||
Required: true,
|
||||
ForceNew: true,
|
||||
|
|
@ -116,12 +116,6 @@ func resourceAwsAppautoscalingTargetRead(d *schema.ResourceData, meta interface{
|
|||
return nil
|
||||
}
|
||||
|
||||
// Updating Target is not supported
|
||||
// func getAwsAppautoscalingTargetUpdate(d *schema.ResourceData, meta interface{}) error {
|
||||
// conn := meta.(*AWSClient).appautoscalingconn
|
||||
|
||||
// }
|
||||
|
||||
func resourceAwsAppautoscalingTargetDelete(d *schema.ResourceData, meta interface{}) error {
|
||||
conn := meta.(*AWSClient).appautoscalingconn
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ func TestAccAWSAppautoScalingTarget_basic(t *testing.T) {
|
|||
Providers: testAccProviders,
|
||||
CheckDestroy: testAccCheckAWSAppautoscalingTargetDestroy,
|
||||
Steps: []resource.TestStep{
|
||||
resource.TestStep{
|
||||
{
|
||||
Config: testAccAWSAppautoscalingTargetConfig(randClusterName),
|
||||
Check: resource.ComposeTestCheckFunc(
|
||||
testAccCheckAWSAppautoscalingTargetExists("aws_appautoscaling_target.bar", &target),
|
||||
|
|
@ -34,7 +34,7 @@ func TestAccAWSAppautoScalingTarget_basic(t *testing.T) {
|
|||
),
|
||||
},
|
||||
|
||||
resource.TestStep{
|
||||
{
|
||||
Config: testAccAWSAppautoscalingTargetConfigUpdate(randClusterName),
|
||||
Check: resource.ComposeTestCheckFunc(
|
||||
testAccCheckAWSAppautoscalingTargetExists("aws_appautoscaling_target.bar", &target),
|
||||
|
|
@ -55,7 +55,7 @@ func TestAccAWSAppautoScalingTarget_spotFleetRequest(t *testing.T) {
|
|||
Providers: testAccProviders,
|
||||
CheckDestroy: testAccCheckAWSAppautoscalingTargetDestroy,
|
||||
Steps: []resource.TestStep{
|
||||
resource.TestStep{
|
||||
{
|
||||
Config: testAccAWSAppautoscalingTargetSpotFleetRequestConfig,
|
||||
Check: resource.ComposeTestCheckFunc(
|
||||
testAccCheckAWSAppautoscalingTargetExists("aws_appautoscaling_target.test", &target),
|
||||
|
|
@ -67,6 +67,27 @@ func TestAccAWSAppautoScalingTarget_spotFleetRequest(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestAccAWSAppautoScalingTarget_emrCluster(t *testing.T) {
|
||||
var target applicationautoscaling.ScalableTarget
|
||||
rInt := acctest.RandInt()
|
||||
|
||||
resource.Test(t, resource.TestCase{
|
||||
PreCheck: func() { testAccPreCheck(t) },
|
||||
Providers: testAccProviders,
|
||||
CheckDestroy: testAccCheckAWSAppautoscalingTargetDestroy,
|
||||
Steps: []resource.TestStep{
|
||||
{
|
||||
Config: testAccAWSAppautoscalingTargetEmrClusterConfig(rInt),
|
||||
Check: resource.ComposeTestCheckFunc(
|
||||
testAccCheckAWSAppautoscalingTargetExists("aws_appautoscaling_target.bar", &target),
|
||||
resource.TestCheckResourceAttr("aws_appautoscaling_target.bar", "service_namespace", "elasticmapreduce"),
|
||||
resource.TestCheckResourceAttr("aws_appautoscaling_target.bar", "scalable_dimension", "elasticmapreduce:instancegroup:InstanceCount"),
|
||||
),
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func testAccCheckAWSAppautoscalingTargetDestroy(s *terraform.State) error {
|
||||
conn := testAccProvider.Meta().(*AWSClient).appautoscalingconn
|
||||
|
||||
|
|
@ -327,6 +348,317 @@ resource "aws_appautoscaling_target" "bar" {
|
|||
`, randClusterName, randClusterName, randClusterName)
|
||||
}
|
||||
|
||||
func testAccAWSAppautoscalingTargetEmrClusterConfig(rInt int) string {
|
||||
return fmt.Sprintf(`
|
||||
resource "aws_emr_cluster" "tf-test-cluster" {
|
||||
name = "emr-test-%d"
|
||||
release_label = "emr-4.6.0"
|
||||
applications = ["Spark"]
|
||||
|
||||
ec2_attributes {
|
||||
subnet_id = "${aws_subnet.main.id}"
|
||||
emr_managed_master_security_group = "${aws_security_group.allow_all.id}"
|
||||
emr_managed_slave_security_group = "${aws_security_group.allow_all.id}"
|
||||
instance_profile = "${aws_iam_instance_profile.emr_profile.arn}"
|
||||
}
|
||||
|
||||
master_instance_type = "m3.xlarge"
|
||||
core_instance_type = "m3.xlarge"
|
||||
core_instance_count = 2
|
||||
|
||||
tags {
|
||||
role = "rolename"
|
||||
dns_zone = "env_zone"
|
||||
env = "env"
|
||||
name = "name-env"
|
||||
}
|
||||
|
||||
keep_job_flow_alive_when_no_steps = true
|
||||
|
||||
bootstrap_action {
|
||||
path = "s3://elasticmapreduce/bootstrap-actions/run-if"
|
||||
name = "runif"
|
||||
args = ["instance.isMaster=true", "echo running on master node"]
|
||||
}
|
||||
|
||||
configurations = "test-fixtures/emr_configurations.json"
|
||||
|
||||
depends_on = ["aws_main_route_table_association.a"]
|
||||
|
||||
service_role = "${aws_iam_role.iam_emr_default_role.arn}"
|
||||
autoscaling_role = "${aws_iam_role.emr-autoscaling-role.arn}"
|
||||
}
|
||||
|
||||
resource "aws_emr_instance_group" "task" {
|
||||
cluster_id = "${aws_emr_cluster.tf-test-cluster.id}"
|
||||
instance_count = 1
|
||||
instance_type = "m3.xlarge"
|
||||
}
|
||||
|
||||
resource "aws_security_group" "allow_all" {
|
||||
name = "allow_all_%d"
|
||||
description = "Allow all inbound traffic"
|
||||
vpc_id = "${aws_vpc.main.id}"
|
||||
|
||||
ingress {
|
||||
from_port = 0
|
||||
to_port = 0
|
||||
protocol = "-1"
|
||||
cidr_blocks = ["0.0.0.0/0"]
|
||||
}
|
||||
|
||||
egress {
|
||||
from_port = 0
|
||||
to_port = 0
|
||||
protocol = "-1"
|
||||
cidr_blocks = ["0.0.0.0/0"]
|
||||
}
|
||||
|
||||
depends_on = ["aws_subnet.main"]
|
||||
|
||||
lifecycle {
|
||||
ignore_changes = ["ingress", "egress"]
|
||||
}
|
||||
|
||||
tags {
|
||||
name = "emr_test"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_vpc" "main" {
|
||||
cidr_block = "168.31.0.0/16"
|
||||
enable_dns_hostnames = true
|
||||
|
||||
tags {
|
||||
name = "emr_test_%d"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_subnet" "main" {
|
||||
vpc_id = "${aws_vpc.main.id}"
|
||||
cidr_block = "168.31.0.0/20"
|
||||
|
||||
tags {
|
||||
name = "emr_test_%d"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_internet_gateway" "gw" {
|
||||
vpc_id = "${aws_vpc.main.id}"
|
||||
}
|
||||
|
||||
resource "aws_route_table" "r" {
|
||||
vpc_id = "${aws_vpc.main.id}"
|
||||
|
||||
route {
|
||||
cidr_block = "0.0.0.0/0"
|
||||
gateway_id = "${aws_internet_gateway.gw.id}"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_main_route_table_association" "a" {
|
||||
vpc_id = "${aws_vpc.main.id}"
|
||||
route_table_id = "${aws_route_table.r.id}"
|
||||
}
|
||||
|
||||
resource "aws_iam_role" "iam_emr_default_role" {
|
||||
name = "iam_emr_default_role_%d"
|
||||
|
||||
assume_role_policy = <<EOT
|
||||
{
|
||||
"Version": "2008-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"Service": "elasticmapreduce.amazonaws.com"
|
||||
},
|
||||
"Action": "sts:AssumeRole"
|
||||
}
|
||||
]
|
||||
}
|
||||
EOT
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "service-attach" {
|
||||
role = "${aws_iam_role.iam_emr_default_role.id}"
|
||||
policy_arn = "${aws_iam_policy.iam_emr_default_policy.arn}"
|
||||
}
|
||||
|
||||
resource "aws_iam_policy" "iam_emr_default_policy" {
|
||||
name = "iam_emr_default_policy_%d"
|
||||
|
||||
policy = <<EOT
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Resource": "*",
|
||||
"Action": [
|
||||
"ec2:AuthorizeSecurityGroupEgress",
|
||||
"ec2:AuthorizeSecurityGroupIngress",
|
||||
"ec2:CancelSpotInstanceRequests",
|
||||
"ec2:CreateNetworkInterface",
|
||||
"ec2:CreateSecurityGroup",
|
||||
"ec2:CreateTags",
|
||||
"ec2:DeleteNetworkInterface",
|
||||
"ec2:DeleteSecurityGroup",
|
||||
"ec2:DeleteTags",
|
||||
"ec2:DescribeAvailabilityZones",
|
||||
"ec2:DescribeAccountAttributes",
|
||||
"ec2:DescribeDhcpOptions",
|
||||
"ec2:DescribeInstanceStatus",
|
||||
"ec2:DescribeInstances",
|
||||
"ec2:DescribeKeyPairs",
|
||||
"ec2:DescribeNetworkAcls",
|
||||
"ec2:DescribeNetworkInterfaces",
|
||||
"ec2:DescribePrefixLists",
|
||||
"ec2:DescribeRouteTables",
|
||||
"ec2:DescribeSecurityGroups",
|
||||
"ec2:DescribeSpotInstanceRequests",
|
||||
"ec2:DescribeSpotPriceHistory",
|
||||
"ec2:DescribeSubnets",
|
||||
"ec2:DescribeVpcAttribute",
|
||||
"ec2:DescribeVpcEndpoints",
|
||||
"ec2:DescribeVpcEndpointServices",
|
||||
"ec2:DescribeVpcs",
|
||||
"ec2:DetachNetworkInterface",
|
||||
"ec2:ModifyImageAttribute",
|
||||
"ec2:ModifyInstanceAttribute",
|
||||
"ec2:RequestSpotInstances",
|
||||
"ec2:RevokeSecurityGroupEgress",
|
||||
"ec2:RunInstances",
|
||||
"ec2:TerminateInstances",
|
||||
"ec2:DeleteVolume",
|
||||
"ec2:DescribeVolumeStatus",
|
||||
"ec2:DescribeVolumes",
|
||||
"ec2:DetachVolume",
|
||||
"iam:GetRole",
|
||||
"iam:GetRolePolicy",
|
||||
"iam:ListInstanceProfiles",
|
||||
"iam:ListRolePolicies",
|
||||
"iam:PassRole",
|
||||
"s3:CreateBucket",
|
||||
"s3:Get*",
|
||||
"s3:List*",
|
||||
"sdb:BatchPutAttributes",
|
||||
"sdb:Select",
|
||||
"sqs:CreateQueue",
|
||||
"sqs:Delete*",
|
||||
"sqs:GetQueue*",
|
||||
"sqs:PurgeQueue",
|
||||
"sqs:ReceiveMessage"
|
||||
]
|
||||
}]
|
||||
}
|
||||
EOT
|
||||
}
|
||||
|
||||
# IAM Role for EC2 Instance Profile
|
||||
resource "aws_iam_role" "iam_emr_profile_role" {
|
||||
name = "iam_emr_profile_role_%d"
|
||||
|
||||
assume_role_policy = <<EOT
|
||||
{
|
||||
"Version": "2008-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "",
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"Service": "ec2.amazonaws.com"
|
||||
},
|
||||
"Action": "sts:AssumeRole"
|
||||
}
|
||||
]
|
||||
}
|
||||
EOT
|
||||
}
|
||||
|
||||
resource "aws_iam_instance_profile" "emr_profile" {
|
||||
name = "emr_profile_%d"
|
||||
roles = ["${aws_iam_role.iam_emr_profile_role.name}"]
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "profile-attach" {
|
||||
role = "${aws_iam_role.iam_emr_profile_role.id}"
|
||||
policy_arn = "${aws_iam_policy.iam_emr_profile_policy.arn}"
|
||||
}
|
||||
|
||||
resource "aws_iam_policy" "iam_emr_profile_policy" {
|
||||
name = "iam_emr_profile_policy_%d"
|
||||
|
||||
policy = <<EOT
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Resource": "*",
|
||||
"Action": [
|
||||
"cloudwatch:*",
|
||||
"dynamodb:*",
|
||||
"ec2:Describe*",
|
||||
"elasticmapreduce:Describe*",
|
||||
"elasticmapreduce:ListBootstrapActions",
|
||||
"elasticmapreduce:ListClusters",
|
||||
"elasticmapreduce:ListInstanceGroups",
|
||||
"elasticmapreduce:ListInstances",
|
||||
"elasticmapreduce:ListSteps",
|
||||
"kinesis:CreateStream",
|
||||
"kinesis:DeleteStream",
|
||||
"kinesis:DescribeStream",
|
||||
"kinesis:GetRecords",
|
||||
"kinesis:GetShardIterator",
|
||||
"kinesis:MergeShards",
|
||||
"kinesis:PutRecord",
|
||||
"kinesis:SplitShard",
|
||||
"rds:Describe*",
|
||||
"s3:*",
|
||||
"sdb:*",
|
||||
"sns:*",
|
||||
"sqs:*"
|
||||
]
|
||||
}]
|
||||
}
|
||||
EOT
|
||||
}
|
||||
|
||||
# IAM Role for autoscaling
|
||||
resource "aws_iam_role" "emr-autoscaling-role" {
|
||||
name = "EMR_AutoScaling_DefaultRole_%d"
|
||||
assume_role_policy = "${data.aws_iam_policy_document.emr-autoscaling-role-policy.json}"
|
||||
}
|
||||
|
||||
data "aws_iam_policy_document" "emr-autoscaling-role-policy" {
|
||||
statement {
|
||||
effect = "Allow"
|
||||
actions = ["sts:AssumeRole"]
|
||||
|
||||
principals = {
|
||||
type = "Service"
|
||||
identifiers = ["elasticmapreduce.amazonaws.com","application-autoscaling.amazonaws.com"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_iam_role_policy_attachment" "emr-autoscaling-role" {
|
||||
role = "${aws_iam_role.emr-autoscaling-role.name}"
|
||||
policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforAutoScalingRole"
|
||||
}
|
||||
|
||||
resource "aws_appautoscaling_target" "bar" {
|
||||
service_namespace = "elasticmapreduce"
|
||||
resource_id = "instancegroup/${aws_emr_cluster.tf-test-cluster.id}/${aws_emr_instance_group.task.id}"
|
||||
scalable_dimension = "elasticmapreduce:instancegroup:InstanceCount"
|
||||
role_arn = "${aws_iam_role.emr-autoscaling-role.arn}"
|
||||
min_capacity = 1
|
||||
max_capacity = 8
|
||||
}
|
||||
|
||||
`, rInt, rInt, rInt, rInt, rInt, rInt, rInt, rInt, rInt, rInt)
|
||||
}
|
||||
|
||||
var testAccAWSAppautoscalingTargetSpotFleetRequestConfig = fmt.Sprintf(`
|
||||
resource "aws_iam_role" "fleet_role" {
|
||||
assume_role_policy = <<EOF
|
||||
|
|
|
|||
|
|
@ -927,8 +927,9 @@ func validateDmsReplicationTaskId(v interface{}, k string) (ws []string, es []er
|
|||
func validateAppautoscalingScalableDimension(v interface{}, k string) (ws []string, errors []error) {
|
||||
value := v.(string)
|
||||
dimensions := map[string]bool{
|
||||
"ecs:service:DesiredCount": true,
|
||||
"ec2:spot-fleet-request:TargetCapacity": true,
|
||||
"ecs:service:DesiredCount": true,
|
||||
"ec2:spot-fleet-request:TargetCapacity": true,
|
||||
"elasticmapreduce:instancegroup:InstanceCount": true,
|
||||
}
|
||||
|
||||
if !dimensions[value] {
|
||||
|
|
@ -940,8 +941,9 @@ func validateAppautoscalingScalableDimension(v interface{}, k string) (ws []stri
|
|||
func validateAppautoscalingServiceNamespace(v interface{}, k string) (ws []string, errors []error) {
|
||||
value := v.(string)
|
||||
namespaces := map[string]bool{
|
||||
"ecs": true,
|
||||
"ec2": true,
|
||||
"ecs": true,
|
||||
"ec2": true,
|
||||
"elasticmapreduce": true,
|
||||
}
|
||||
|
||||
if !namespaces[value] {
|
||||
|
|
|
|||
|
|
@ -28,7 +28,15 @@ The following arguments are supported:
|
|||
|
||||
* `max_capacity` - (Required) The max capacity of the scalable target.
|
||||
* `min_capacity` - (Required) The min capacity of the scalable target.
|
||||
* `resource_id` - (Required) The resource type and unique identifier string for the resource associated with the scalable target. For Amazon ECS services, this value is the resource type, followed by the cluster name and service name, such as `service/default/sample-webapp`. For Amazon EC2 Spot fleet requests, the resource type is `spot-fleet-request`, and the identifier is the Spot fleet request ID; for example, `spot-fleet-request/sfr-73fbd2ce-aa30-494c-8788-1cee4EXAMPLE`.
|
||||
* `role_arn` - (Required) The ARN of the IAM role that allows Application AutoScaling to modify your scalable target on your behalf.
|
||||
* `scalable_dimension` - (Required) The scalable dimension of the scalable target. The scalable dimension contains the service namespace, resource type, and scaling property, such as `ecs:service:DesiredCount` for the desired task count of an Amazon ECS service, or `ec2:spot-fleet-request:TargetCapacity` for the target capacity of an Amazon EC2 Spot fleet request.
|
||||
* `service_namespace` - (Required) The AWS service namespace of the scalable target. Valid values are `ecs` for Amazon ECS services and `ec2` Amazon EC2 Spot fleet requests.
|
||||
* `resource_id` - (Required) The resource type and unique identifier string for the resource associated with the scalable target.
|
||||
For Amazon ECS services, this value is the resource type, followed by the cluster name and service name, for example, `service/default/sample-webapp`.
|
||||
For Amazon EC2 Spot fleet requests, the resource type is `spot-fleet-request`, and the identifier is the Spot fleet request ID; for example, `spot-fleet-request/sfr-73fbd2ce-aa30-494c-8788-1cee4EXAMPLE`.
|
||||
For Amazon EMR Cluster Instance Groups, the resource type is `instancegroup`, the identifier is the Cluster Id and the Instance Group Id; for example, `instancegroup/j-2EEZNYKUA1NTV/ig-1791Y4E1L8YI0`.
|
||||
* `role_arn` - (Required) The ARN of the IAM role that allows Application
|
||||
AutoScaling to modify your scalable target on your behalf.
|
||||
* `scalable_dimension` - (Required) The scalable dimension of the scalable target. The scalable dimension contains the service namespace, resource
|
||||
type, and scaling property, such as `ecs:service:DesiredCount` for the desired task count of an Amazon ECS service,
|
||||
`ec2:spot-fleet-request:TargetCapacity` for the target capacity of an Amazon EC2 Spot fleet request or
|
||||
`elasticmapreduce:instancegroup:InstanceCount` for the Instance count of an EMR Cluster Instance Group.
|
||||
* `service_namespace` - (Required) The AWS service namespace of the scalable target.
|
||||
Valid values are `ecs` for Amazon ECS services, `ec2` Amazon EC2 Spot fleet requests and `elasticmapreduce` for Amazon EMR Clusters.
|
||||
|
|
|
|||
Loading…
Reference in New Issue