resilien
/
terraform
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

provider/aws: Fix EMR Bootstrap Action Ordering (#13580) 

* provider/aws: Add failing test for EMR Bootstrap Actions

* aws_emr_cluster: Fix bootstrap action parameter ordering

* provider/aws: Fix EMR Bootstrap arguments

* provider/aws: Args needs to be ForceNew, because we can't update them
This commit is contained in:
Clint 2017-04-12 14:19:38 -05:00 committed by GitHub
parent 491cc0b725
commit 9ef9501e65
3 changed files with 430 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
builtin/providers/aws/resource_aws_emr_cluster.go
View File

@ -138,10 +138,10 @@ func resourceAwsEMRCluster() *schema.Resource {
Required: true,
},
"args": {
Type: schema.TypeSet,
Type: schema.TypeList,
Optional: true,
ForceNew: true,
Elem: &schema.Schema{Type: schema.TypeString},
Set: schema.HashString,
},
},
},
@ -381,6 +381,18 @@ func resourceAwsEMRClusterRead(d *schema.ResourceData, meta interface{}) error {
if err := d.Set("ec2_attributes", flattenEc2Attributes(cluster.Ec2InstanceAttributes)); err != nil {
log.Printf("[ERR] Error setting EMR Ec2 Attributes: %s", err)
}
respBootstraps, err := emrconn.ListBootstrapActions(&emr.ListBootstrapActionsInput{
ClusterId: cluster.Id,
})
if err != nil {
log.Printf("[WARN] Error listing bootstrap actions: %s", err)
}
if err := d.Set("bootstrap_action", flattenBootstrapArguments(respBootstraps.BootstrapActions)); err != nil {
log.Printf("[WARN] Error setting Bootstrap Actions: %s", err)
}
return nil
}
@ -589,6 +601,20 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{
return result
}
func flattenBootstrapArguments(actions []*emr.Command) []map[string]interface{} {
result := make([]map[string]interface{}, 0)
for _, b := range actions {
attrs := make(map[string]interface{})
attrs["name"] = *b.Name
attrs["path"] = *b.ScriptPath
attrs["args"] = flattenStringList(b.Args)
result = append(result, attrs)
}
return result
}
func loadGroups(d *schema.ResourceData, meta interface{}) ([]*emr.InstanceGroup, error) {
emrconn := meta.(*AWSClient).emrconn
reqGrps := &emr.ListInstanceGroupsInput{
@ -699,7 +725,7 @@ func expandBootstrapActions(bootstrapActions []interface{}) []*emr.BootstrapActi
actionAttributes := raw.(map[string]interface{})
actionName := actionAttributes["name"].(string)
actionPath := actionAttributes["path"].(string)
actionArgs := actionAttributes["args"].(*schema.Set).List()
actionArgs := actionAttributes["args"].([]interface{})
action := &emr.BootstrapActionConfig{
Name: aws.String(actionName),

407
builtin/providers/aws/resource_aws_emr_cluster_test.go
View File

@ -3,6 +3,7 @@ package aws
import (
"fmt"
"log"
"reflect"
"testing"
"github.com/aws/aws-sdk-go/aws"
@ -14,7 +15,7 @@ import (
)
func TestAccAWSEMRCluster_basic(t *testing.T) {
var jobFlow emr.RunJobFlowOutput
var cluster emr.Cluster
r := acctest.RandInt()
resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
@ -23,14 +24,51 @@ func TestAccAWSEMRCluster_basic(t *testing.T) {
Steps: []resource.TestStep{
{
Config: testAccAWSEmrClusterConfig(r),
Check: testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
Check: testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
},
},
})
}
func TestAccAWSEMRCluster_bootstrap_ordering(t *testing.T) {
var cluster emr.Cluster
rName := acctest.RandomWithPrefix("tf-emr-bootstrap")
argsInts := []string{
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
}
argsStrings := []string{
"instance.isMaster=true",
"echo running on master node",
}
resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckAWSEmrDestroy,
Steps: []resource.TestStep{
{
Config: testAccAWSEmrClusterConfig_bootstrap(rName),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSEmrClusterExists("aws_emr_cluster.test", &cluster),
testAccCheck_bootstrap_order(&cluster, argsInts, argsStrings),
),
},
},
})
}
func TestAccAWSEMRCluster_terminationProtected(t *testing.T) {
var jobFlow emr.RunJobFlowOutput
var cluster emr.Cluster
r := acctest.RandInt()
resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
@ -40,7 +78,7 @@ func TestAccAWSEMRCluster_terminationProtected(t *testing.T) {
{
Config: testAccAWSEmrClusterConfig(r),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
resource.TestCheckResourceAttr(
"aws_emr_cluster.tf-test-cluster", "termination_protection", "false"),
),
@ -48,7 +86,7 @@ func TestAccAWSEMRCluster_terminationProtected(t *testing.T) {
{
Config: testAccAWSEmrClusterConfigTerminationPolicyUpdated(r),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
resource.TestCheckResourceAttr(
"aws_emr_cluster.tf-test-cluster", "termination_protection", "true"),
),
@ -57,7 +95,7 @@ func TestAccAWSEMRCluster_terminationProtected(t *testing.T) {
//Need to turn off termination_protection to allow the job to be deleted
Config: testAccAWSEmrClusterConfig(r),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
),
},
},
@ -65,7 +103,7 @@ func TestAccAWSEMRCluster_terminationProtected(t *testing.T) {
}
func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) {
var jobFlow emr.RunJobFlowOutput
var cluster emr.Cluster
r := acctest.RandInt()
resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
@ -75,7 +113,7 @@ func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) {
{
Config: testAccAWSEmrClusterConfig(r),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
resource.TestCheckResourceAttr(
"aws_emr_cluster.tf-test-cluster", "visible_to_all_users", "true"),
),
@ -83,7 +121,7 @@ func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) {
{
Config: testAccAWSEmrClusterConfigVisibleToAllUsersUpdated(r),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
resource.TestCheckResourceAttr(
"aws_emr_cluster.tf-test-cluster", "visible_to_all_users", "false"),
),
@ -93,7 +131,7 @@ func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) {
}
func TestAccAWSEMRCluster_tags(t *testing.T) {
var jobFlow emr.RunJobFlowOutput
var cluster emr.Cluster
r := acctest.RandInt()
resource.Test(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
@ -103,7 +141,7 @@ func TestAccAWSEMRCluster_tags(t *testing.T) {
{
Config: testAccAWSEmrClusterConfig(r),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
resource.TestCheckResourceAttr("aws_emr_cluster.tf-test-cluster", "tags.%", "4"),
resource.TestCheckResourceAttr(
"aws_emr_cluster.tf-test-cluster", "tags.role", "rolename"),
@ -117,7 +155,7 @@ func TestAccAWSEMRCluster_tags(t *testing.T) {
{
Config: testAccAWSEmrClusterConfigUpdatedTags(r),
Check: resource.ComposeTestCheckFunc(
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
resource.TestCheckResourceAttr("aws_emr_cluster.tf-test-cluster", "tags.%", "3"),
resource.TestCheckResourceAttr(
"aws_emr_cluster.tf-test-cluster", "tags.dns_zone", "new_zone"),
@ -131,6 +169,45 @@ func TestAccAWSEMRCluster_tags(t *testing.T) {
})
}
func testAccCheck_bootstrap_order(cluster *emr.Cluster, argsInts, argsStrings []string) resource.TestCheckFunc {
return func(s *terraform.State) error {
emrconn := testAccProvider.Meta().(*AWSClient).emrconn
req := emr.ListBootstrapActionsInput{
ClusterId: cluster.Id,
}
resp, err := emrconn.ListBootstrapActions(&req)
if err != nil {
return fmt.Errorf("[ERR] Error listing boostrap actions in test: %s", err)
}
// make sure we actually checked something
var ran bool
for _, ba := range resp.BootstrapActions {
// assume name matches the config
rArgs := aws.StringValueSlice(ba.Args)
if *ba.Name == "test" {
ran = true
if !reflect.DeepEqual(argsInts, rArgs) {
return fmt.Errorf("Error matching Bootstrap args:\n\texpected: %#v\n\tgot: %#v", argsInts, rArgs)
}
} else if *ba.Name == "runif" {
ran = true
if !reflect.DeepEqual(argsStrings, rArgs) {
return fmt.Errorf("Error matching Bootstrap args:\n\texpected: %#v\n\tgot: %#v", argsStrings, rArgs)
}
}
}
if !ran {
return fmt.Errorf("Expected to compare bootstrap actions, but no checks were ran")
}
return nil
}
}
func testAccCheckAWSEmrDestroy(s *terraform.State) error {
conn := testAccProvider.Meta().(*AWSClient).emrconn
@ -163,7 +240,7 @@ func testAccCheckAWSEmrDestroy(s *terraform.State) error {
return nil
}
func testAccCheckAWSEmrClusterExists(n string, v *emr.RunJobFlowOutput) resource.TestCheckFunc {
func testAccCheckAWSEmrClusterExists(n string, v *emr.Cluster) resource.TestCheckFunc {
return func(s *terraform.State) error {
rs, ok := s.RootModule().Resources[n]
if !ok {
@ -185,6 +262,8 @@ func testAccCheckAWSEmrClusterExists(n string, v *emr.RunJobFlowOutput) resource
return fmt.Errorf("EMR cluser not found")
}
*v = *describe.Cluster
if describe.Cluster != nil &&
*describe.Cluster.Status.State != "WAITING" {
return fmt.Errorf("EMR cluser is not up yet")
@ -194,6 +273,308 @@ func testAccCheckAWSEmrClusterExists(n string, v *emr.RunJobFlowOutput) resource
}
}
func testAccAWSEmrClusterConfig_bootstrap(r string) string {
return fmt.Sprintf(`
resource "aws_emr_cluster" "test" {
count = 1
name = "%s"
release_label = "emr-5.0.0"
applications = ["Hadoop", "Hive"]
log_uri = "s3n://terraform/testlog/"
master_instance_type = "m4.large"
core_instance_type = "m1.small"
core_instance_count = 1
service_role = "${aws_iam_role.iam_emr_default_role.arn}"
depends_on = ["aws_main_route_table_association.a"]
ec2_attributes {
subnet_id = "${aws_subnet.main.id}"
emr_managed_master_security_group = "${aws_security_group.allow_all.id}"
emr_managed_slave_security_group = "${aws_security_group.allow_all.id}"
instance_profile = "${aws_iam_instance_profile.emr_profile.arn}"
}
bootstrap_action {
path = "s3://elasticmapreduce/bootstrap-actions/run-if"
name = "runif"
args = ["instance.isMaster=true", "echo running on master node"]
}
bootstrap_action = [
{
path = "s3://${aws_s3_bucket.tester.bucket}/testscript.sh"
name = "test"
args = ["1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
]
},
]
}
resource "aws_iam_instance_profile" "emr_profile" {
name = "%s_profile"
role = "${aws_iam_role.iam_emr_profile_role.name}"
}
resource "aws_iam_role" "iam_emr_default_role" {
name = "%s_default_role"
assume_role_policy = <<EOT
{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "elasticmapreduce.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
EOT
}
resource "aws_iam_role" "iam_emr_profile_role" {
name = "%s_profile_role"
assume_role_policy = <<EOT
{
"Version": "2008-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"Service": "ec2.amazonaws.com"
},
"Action": "sts:AssumeRole"
}
]
}
EOT
}
resource "aws_iam_role_policy_attachment" "profile-attach" {
role = "${aws_iam_role.iam_emr_profile_role.id}"
policy_arn = "${aws_iam_policy.iam_emr_profile_policy.arn}"
}
resource "aws_iam_role_policy_attachment" "service-attach" {
role = "${aws_iam_role.iam_emr_default_role.id}"
policy_arn = "${aws_iam_policy.iam_emr_default_policy.arn}"
}
resource "aws_iam_policy" "iam_emr_default_policy" {
name = "%s_emr"
policy = <<EOT
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"ec2:AuthorizeSecurityGroupEgress",
"ec2:AuthorizeSecurityGroupIngress",
"ec2:CancelSpotInstanceRequests",
"ec2:CreateNetworkInterface",
"ec2:CreateSecurityGroup",
"ec2:CreateTags",
"ec2:DeleteNetworkInterface",
"ec2:DeleteSecurityGroup",
"ec2:DeleteTags",
"ec2:DescribeAvailabilityZones",
"ec2:DescribeAccountAttributes",
"ec2:DescribeDhcpOptions",
"ec2:DescribeInstanceStatus",
"ec2:DescribeInstances",
"ec2:DescribeKeyPairs",
"ec2:DescribeNetworkAcls",
"ec2:DescribeNetworkInterfaces",
"ec2:DescribePrefixLists",
"ec2:DescribeRouteTables",
"ec2:DescribeSecurityGroups",
"ec2:DescribeSpotInstanceRequests",
"ec2:DescribeSpotPriceHistory",
"ec2:DescribeSubnets",
"ec2:DescribeVpcAttribute",
"ec2:DescribeVpcEndpoints",
"ec2:DescribeVpcEndpointServices",
"ec2:DescribeVpcs",
"ec2:DetachNetworkInterface",
"ec2:ModifyImageAttribute",
"ec2:ModifyInstanceAttribute",
"ec2:RequestSpotInstances",
"ec2:RevokeSecurityGroupEgress",
"ec2:RunInstances",
"ec2:TerminateInstances",
"ec2:DeleteVolume",
"ec2:DescribeVolumeStatus",
"iam:GetRole",
"iam:GetRolePolicy",
"iam:ListInstanceProfiles",
"iam:ListRolePolicies",
"iam:PassRole",
"s3:CreateBucket",
"s3:Get*",
"s3:List*",
"sdb:BatchPutAttributes",
"sdb:Select",
"sqs:CreateQueue",
"sqs:Delete*",
"sqs:GetQueue*",
"sqs:PurgeQueue",
"sqs:ReceiveMessage"
]
}]
}
EOT
}
resource "aws_iam_policy" "iam_emr_profile_policy" {
name = "%s_profile"
policy = <<EOT
{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Resource": "*",
"Action": [
"cloudwatch:*",
"dynamodb:*",
"ec2:Describe*",
"elasticmapreduce:Describe*",
"elasticmapreduce:ListBootstrapActions",
"elasticmapreduce:ListClusters",
"elasticmapreduce:ListInstanceGroups",
"elasticmapreduce:ListInstances",
"elasticmapreduce:ListSteps",
"kinesis:CreateStream",
"kinesis:DeleteStream",
"kinesis:DescribeStream",
"kinesis:GetRecords",
"kinesis:GetShardIterator",
"kinesis:MergeShards",
"kinesis:PutRecord",
"kinesis:SplitShard",
"rds:Describe*",
"s3:*",
"sdb:*",
"sns:*",
"sqs:*"
]
}]
}
EOT
}
resource "aws_vpc" "main" {
cidr_block = "168.31.0.0/16"
enable_dns_hostnames = true
tags {
name = "emr_test_cts"
}
}
resource "aws_subnet" "main" {
vpc_id = "${aws_vpc.main.id}"
cidr_block = "168.31.0.0/20"
tags {
name = "emr_test_cts"
}
}
resource "aws_internet_gateway" "gw" {
vpc_id = "${aws_vpc.main.id}"
}
resource "aws_route_table" "r" {
vpc_id = "${aws_vpc.main.id}"
route {
cidr_block = "0.0.0.0/0"
gateway_id = "${aws_internet_gateway.gw.id}"
}
}
resource "aws_main_route_table_association" "a" {
vpc_id = "${aws_vpc.main.id}"
route_table_id = "${aws_route_table.r.id}"
}
resource "aws_security_group" "allow_all" {
name = "allow_all"
description = "Allow all inbound traffic"
vpc_id = "${aws_vpc.main.id}"
ingress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
depends_on = ["aws_subnet.main"]
lifecycle {
ignore_changes = ["ingress", "egress"]
}
tags {
name = "emr_test"
}
}
output "cluser_id" {
value = "${aws_emr_cluster.test.id}"
}
resource "aws_s3_bucket" "tester" {
bucket = "%s"
acl = "public-read"
}
resource "aws_s3_bucket_object" "testobject" {
bucket = "${aws_s3_bucket.tester.bucket}"
key = "testscript.sh"
#source = "testscript.sh"
content = "${data.template_file.testscript.rendered}"
acl = "public-read"
}
data "template_file" "testscript" {
template = <<POLICY
#!/bin/bash
echo $@
POLICY
}`, r, r, r, r, r, r, r)
}
func testAccAWSEmrClusterConfig(r int) string {
return fmt.Sprintf(`
provider "aws" {

7
helper/acctest/random.go
View File

@ -24,6 +24,13 @@ func RandInt() int {
return rand.New(rand.NewSource(time.Now().UnixNano())).Int()
}
// RandomWithPrefix is used to generate a unique name with a prefix, for
// randomizing names in acceptance tests
func RandomWithPrefix(name string) string {
reseed()
return fmt.Sprintf("%s-%d", name, rand.New(rand.NewSource(time.Now().UnixNano())).Int())
}
func RandIntRange(min int, max int) int {
reseed()
source := rand.New(rand.NewSource(time.Now().UnixNano()))