provider/aws: Fix EMR Bootstrap Action Ordering (#13580)
* provider/aws: Add failing test for EMR Bootstrap Actions * aws_emr_cluster: Fix bootstrap action parameter ordering * provider/aws: Fix EMR Bootstrap arguments * provider/aws: Args needs to be ForceNew, because we can't update them
This commit is contained in:
parent
491cc0b725
commit
9ef9501e65
|
@ -138,10 +138,10 @@ func resourceAwsEMRCluster() *schema.Resource {
|
||||||
Required: true,
|
Required: true,
|
||||||
},
|
},
|
||||||
"args": {
|
"args": {
|
||||||
Type: schema.TypeSet,
|
Type: schema.TypeList,
|
||||||
Optional: true,
|
Optional: true,
|
||||||
|
ForceNew: true,
|
||||||
Elem: &schema.Schema{Type: schema.TypeString},
|
Elem: &schema.Schema{Type: schema.TypeString},
|
||||||
Set: schema.HashString,
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -381,6 +381,18 @@ func resourceAwsEMRClusterRead(d *schema.ResourceData, meta interface{}) error {
|
||||||
if err := d.Set("ec2_attributes", flattenEc2Attributes(cluster.Ec2InstanceAttributes)); err != nil {
|
if err := d.Set("ec2_attributes", flattenEc2Attributes(cluster.Ec2InstanceAttributes)); err != nil {
|
||||||
log.Printf("[ERR] Error setting EMR Ec2 Attributes: %s", err)
|
log.Printf("[ERR] Error setting EMR Ec2 Attributes: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
respBootstraps, err := emrconn.ListBootstrapActions(&emr.ListBootstrapActionsInput{
|
||||||
|
ClusterId: cluster.Id,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[WARN] Error listing bootstrap actions: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := d.Set("bootstrap_action", flattenBootstrapArguments(respBootstraps.BootstrapActions)); err != nil {
|
||||||
|
log.Printf("[WARN] Error setting Bootstrap Actions: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -589,6 +601,20 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func flattenBootstrapArguments(actions []*emr.Command) []map[string]interface{} {
|
||||||
|
result := make([]map[string]interface{}, 0)
|
||||||
|
|
||||||
|
for _, b := range actions {
|
||||||
|
attrs := make(map[string]interface{})
|
||||||
|
attrs["name"] = *b.Name
|
||||||
|
attrs["path"] = *b.ScriptPath
|
||||||
|
attrs["args"] = flattenStringList(b.Args)
|
||||||
|
result = append(result, attrs)
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func loadGroups(d *schema.ResourceData, meta interface{}) ([]*emr.InstanceGroup, error) {
|
func loadGroups(d *schema.ResourceData, meta interface{}) ([]*emr.InstanceGroup, error) {
|
||||||
emrconn := meta.(*AWSClient).emrconn
|
emrconn := meta.(*AWSClient).emrconn
|
||||||
reqGrps := &emr.ListInstanceGroupsInput{
|
reqGrps := &emr.ListInstanceGroupsInput{
|
||||||
|
@ -699,7 +725,7 @@ func expandBootstrapActions(bootstrapActions []interface{}) []*emr.BootstrapActi
|
||||||
actionAttributes := raw.(map[string]interface{})
|
actionAttributes := raw.(map[string]interface{})
|
||||||
actionName := actionAttributes["name"].(string)
|
actionName := actionAttributes["name"].(string)
|
||||||
actionPath := actionAttributes["path"].(string)
|
actionPath := actionAttributes["path"].(string)
|
||||||
actionArgs := actionAttributes["args"].(*schema.Set).List()
|
actionArgs := actionAttributes["args"].([]interface{})
|
||||||
|
|
||||||
action := &emr.BootstrapActionConfig{
|
action := &emr.BootstrapActionConfig{
|
||||||
Name: aws.String(actionName),
|
Name: aws.String(actionName),
|
||||||
|
|
|
@ -3,6 +3,7 @@ package aws
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/aws/aws-sdk-go/aws"
|
"github.com/aws/aws-sdk-go/aws"
|
||||||
|
@ -14,7 +15,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestAccAWSEMRCluster_basic(t *testing.T) {
|
func TestAccAWSEMRCluster_basic(t *testing.T) {
|
||||||
var jobFlow emr.RunJobFlowOutput
|
var cluster emr.Cluster
|
||||||
r := acctest.RandInt()
|
r := acctest.RandInt()
|
||||||
resource.Test(t, resource.TestCase{
|
resource.Test(t, resource.TestCase{
|
||||||
PreCheck: func() { testAccPreCheck(t) },
|
PreCheck: func() { testAccPreCheck(t) },
|
||||||
|
@ -23,14 +24,51 @@ func TestAccAWSEMRCluster_basic(t *testing.T) {
|
||||||
Steps: []resource.TestStep{
|
Steps: []resource.TestStep{
|
||||||
{
|
{
|
||||||
Config: testAccAWSEmrClusterConfig(r),
|
Config: testAccAWSEmrClusterConfig(r),
|
||||||
Check: testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
|
Check: testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccAWSEMRCluster_bootstrap_ordering(t *testing.T) {
|
||||||
|
var cluster emr.Cluster
|
||||||
|
rName := acctest.RandomWithPrefix("tf-emr-bootstrap")
|
||||||
|
argsInts := []string{
|
||||||
|
"1",
|
||||||
|
"2",
|
||||||
|
"3",
|
||||||
|
"4",
|
||||||
|
"5",
|
||||||
|
"6",
|
||||||
|
"7",
|
||||||
|
"8",
|
||||||
|
"9",
|
||||||
|
"10",
|
||||||
|
}
|
||||||
|
|
||||||
|
argsStrings := []string{
|
||||||
|
"instance.isMaster=true",
|
||||||
|
"echo running on master node",
|
||||||
|
}
|
||||||
|
|
||||||
|
resource.Test(t, resource.TestCase{
|
||||||
|
PreCheck: func() { testAccPreCheck(t) },
|
||||||
|
Providers: testAccProviders,
|
||||||
|
CheckDestroy: testAccCheckAWSEmrDestroy,
|
||||||
|
Steps: []resource.TestStep{
|
||||||
|
{
|
||||||
|
Config: testAccAWSEmrClusterConfig_bootstrap(rName),
|
||||||
|
Check: resource.ComposeTestCheckFunc(
|
||||||
|
testAccCheckAWSEmrClusterExists("aws_emr_cluster.test", &cluster),
|
||||||
|
testAccCheck_bootstrap_order(&cluster, argsInts, argsStrings),
|
||||||
|
),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAccAWSEMRCluster_terminationProtected(t *testing.T) {
|
func TestAccAWSEMRCluster_terminationProtected(t *testing.T) {
|
||||||
var jobFlow emr.RunJobFlowOutput
|
var cluster emr.Cluster
|
||||||
r := acctest.RandInt()
|
r := acctest.RandInt()
|
||||||
resource.Test(t, resource.TestCase{
|
resource.Test(t, resource.TestCase{
|
||||||
PreCheck: func() { testAccPreCheck(t) },
|
PreCheck: func() { testAccPreCheck(t) },
|
||||||
|
@ -40,7 +78,7 @@ func TestAccAWSEMRCluster_terminationProtected(t *testing.T) {
|
||||||
{
|
{
|
||||||
Config: testAccAWSEmrClusterConfig(r),
|
Config: testAccAWSEmrClusterConfig(r),
|
||||||
Check: resource.ComposeTestCheckFunc(
|
Check: resource.ComposeTestCheckFunc(
|
||||||
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
|
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
|
||||||
resource.TestCheckResourceAttr(
|
resource.TestCheckResourceAttr(
|
||||||
"aws_emr_cluster.tf-test-cluster", "termination_protection", "false"),
|
"aws_emr_cluster.tf-test-cluster", "termination_protection", "false"),
|
||||||
),
|
),
|
||||||
|
@ -48,7 +86,7 @@ func TestAccAWSEMRCluster_terminationProtected(t *testing.T) {
|
||||||
{
|
{
|
||||||
Config: testAccAWSEmrClusterConfigTerminationPolicyUpdated(r),
|
Config: testAccAWSEmrClusterConfigTerminationPolicyUpdated(r),
|
||||||
Check: resource.ComposeTestCheckFunc(
|
Check: resource.ComposeTestCheckFunc(
|
||||||
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
|
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
|
||||||
resource.TestCheckResourceAttr(
|
resource.TestCheckResourceAttr(
|
||||||
"aws_emr_cluster.tf-test-cluster", "termination_protection", "true"),
|
"aws_emr_cluster.tf-test-cluster", "termination_protection", "true"),
|
||||||
),
|
),
|
||||||
|
@ -57,7 +95,7 @@ func TestAccAWSEMRCluster_terminationProtected(t *testing.T) {
|
||||||
//Need to turn off termination_protection to allow the job to be deleted
|
//Need to turn off termination_protection to allow the job to be deleted
|
||||||
Config: testAccAWSEmrClusterConfig(r),
|
Config: testAccAWSEmrClusterConfig(r),
|
||||||
Check: resource.ComposeTestCheckFunc(
|
Check: resource.ComposeTestCheckFunc(
|
||||||
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
|
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -65,7 +103,7 @@ func TestAccAWSEMRCluster_terminationProtected(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) {
|
func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) {
|
||||||
var jobFlow emr.RunJobFlowOutput
|
var cluster emr.Cluster
|
||||||
r := acctest.RandInt()
|
r := acctest.RandInt()
|
||||||
resource.Test(t, resource.TestCase{
|
resource.Test(t, resource.TestCase{
|
||||||
PreCheck: func() { testAccPreCheck(t) },
|
PreCheck: func() { testAccPreCheck(t) },
|
||||||
|
@ -75,7 +113,7 @@ func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) {
|
||||||
{
|
{
|
||||||
Config: testAccAWSEmrClusterConfig(r),
|
Config: testAccAWSEmrClusterConfig(r),
|
||||||
Check: resource.ComposeTestCheckFunc(
|
Check: resource.ComposeTestCheckFunc(
|
||||||
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
|
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
|
||||||
resource.TestCheckResourceAttr(
|
resource.TestCheckResourceAttr(
|
||||||
"aws_emr_cluster.tf-test-cluster", "visible_to_all_users", "true"),
|
"aws_emr_cluster.tf-test-cluster", "visible_to_all_users", "true"),
|
||||||
),
|
),
|
||||||
|
@ -83,7 +121,7 @@ func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) {
|
||||||
{
|
{
|
||||||
Config: testAccAWSEmrClusterConfigVisibleToAllUsersUpdated(r),
|
Config: testAccAWSEmrClusterConfigVisibleToAllUsersUpdated(r),
|
||||||
Check: resource.ComposeTestCheckFunc(
|
Check: resource.ComposeTestCheckFunc(
|
||||||
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
|
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
|
||||||
resource.TestCheckResourceAttr(
|
resource.TestCheckResourceAttr(
|
||||||
"aws_emr_cluster.tf-test-cluster", "visible_to_all_users", "false"),
|
"aws_emr_cluster.tf-test-cluster", "visible_to_all_users", "false"),
|
||||||
),
|
),
|
||||||
|
@ -93,7 +131,7 @@ func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAccAWSEMRCluster_tags(t *testing.T) {
|
func TestAccAWSEMRCluster_tags(t *testing.T) {
|
||||||
var jobFlow emr.RunJobFlowOutput
|
var cluster emr.Cluster
|
||||||
r := acctest.RandInt()
|
r := acctest.RandInt()
|
||||||
resource.Test(t, resource.TestCase{
|
resource.Test(t, resource.TestCase{
|
||||||
PreCheck: func() { testAccPreCheck(t) },
|
PreCheck: func() { testAccPreCheck(t) },
|
||||||
|
@ -103,7 +141,7 @@ func TestAccAWSEMRCluster_tags(t *testing.T) {
|
||||||
{
|
{
|
||||||
Config: testAccAWSEmrClusterConfig(r),
|
Config: testAccAWSEmrClusterConfig(r),
|
||||||
Check: resource.ComposeTestCheckFunc(
|
Check: resource.ComposeTestCheckFunc(
|
||||||
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
|
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
|
||||||
resource.TestCheckResourceAttr("aws_emr_cluster.tf-test-cluster", "tags.%", "4"),
|
resource.TestCheckResourceAttr("aws_emr_cluster.tf-test-cluster", "tags.%", "4"),
|
||||||
resource.TestCheckResourceAttr(
|
resource.TestCheckResourceAttr(
|
||||||
"aws_emr_cluster.tf-test-cluster", "tags.role", "rolename"),
|
"aws_emr_cluster.tf-test-cluster", "tags.role", "rolename"),
|
||||||
|
@ -117,7 +155,7 @@ func TestAccAWSEMRCluster_tags(t *testing.T) {
|
||||||
{
|
{
|
||||||
Config: testAccAWSEmrClusterConfigUpdatedTags(r),
|
Config: testAccAWSEmrClusterConfigUpdatedTags(r),
|
||||||
Check: resource.ComposeTestCheckFunc(
|
Check: resource.ComposeTestCheckFunc(
|
||||||
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow),
|
testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster),
|
||||||
resource.TestCheckResourceAttr("aws_emr_cluster.tf-test-cluster", "tags.%", "3"),
|
resource.TestCheckResourceAttr("aws_emr_cluster.tf-test-cluster", "tags.%", "3"),
|
||||||
resource.TestCheckResourceAttr(
|
resource.TestCheckResourceAttr(
|
||||||
"aws_emr_cluster.tf-test-cluster", "tags.dns_zone", "new_zone"),
|
"aws_emr_cluster.tf-test-cluster", "tags.dns_zone", "new_zone"),
|
||||||
|
@ -131,6 +169,45 @@ func TestAccAWSEMRCluster_tags(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testAccCheck_bootstrap_order(cluster *emr.Cluster, argsInts, argsStrings []string) resource.TestCheckFunc {
|
||||||
|
return func(s *terraform.State) error {
|
||||||
|
|
||||||
|
emrconn := testAccProvider.Meta().(*AWSClient).emrconn
|
||||||
|
req := emr.ListBootstrapActionsInput{
|
||||||
|
ClusterId: cluster.Id,
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := emrconn.ListBootstrapActions(&req)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("[ERR] Error listing boostrap actions in test: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// make sure we actually checked something
|
||||||
|
var ran bool
|
||||||
|
for _, ba := range resp.BootstrapActions {
|
||||||
|
// assume name matches the config
|
||||||
|
rArgs := aws.StringValueSlice(ba.Args)
|
||||||
|
if *ba.Name == "test" {
|
||||||
|
ran = true
|
||||||
|
if !reflect.DeepEqual(argsInts, rArgs) {
|
||||||
|
return fmt.Errorf("Error matching Bootstrap args:\n\texpected: %#v\n\tgot: %#v", argsInts, rArgs)
|
||||||
|
}
|
||||||
|
} else if *ba.Name == "runif" {
|
||||||
|
ran = true
|
||||||
|
if !reflect.DeepEqual(argsStrings, rArgs) {
|
||||||
|
return fmt.Errorf("Error matching Bootstrap args:\n\texpected: %#v\n\tgot: %#v", argsStrings, rArgs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !ran {
|
||||||
|
return fmt.Errorf("Expected to compare bootstrap actions, but no checks were ran")
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func testAccCheckAWSEmrDestroy(s *terraform.State) error {
|
func testAccCheckAWSEmrDestroy(s *terraform.State) error {
|
||||||
conn := testAccProvider.Meta().(*AWSClient).emrconn
|
conn := testAccProvider.Meta().(*AWSClient).emrconn
|
||||||
|
|
||||||
|
@ -163,7 +240,7 @@ func testAccCheckAWSEmrDestroy(s *terraform.State) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func testAccCheckAWSEmrClusterExists(n string, v *emr.RunJobFlowOutput) resource.TestCheckFunc {
|
func testAccCheckAWSEmrClusterExists(n string, v *emr.Cluster) resource.TestCheckFunc {
|
||||||
return func(s *terraform.State) error {
|
return func(s *terraform.State) error {
|
||||||
rs, ok := s.RootModule().Resources[n]
|
rs, ok := s.RootModule().Resources[n]
|
||||||
if !ok {
|
if !ok {
|
||||||
|
@ -185,6 +262,8 @@ func testAccCheckAWSEmrClusterExists(n string, v *emr.RunJobFlowOutput) resource
|
||||||
return fmt.Errorf("EMR cluser not found")
|
return fmt.Errorf("EMR cluser not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
*v = *describe.Cluster
|
||||||
|
|
||||||
if describe.Cluster != nil &&
|
if describe.Cluster != nil &&
|
||||||
*describe.Cluster.Status.State != "WAITING" {
|
*describe.Cluster.Status.State != "WAITING" {
|
||||||
return fmt.Errorf("EMR cluser is not up yet")
|
return fmt.Errorf("EMR cluser is not up yet")
|
||||||
|
@ -194,6 +273,308 @@ func testAccCheckAWSEmrClusterExists(n string, v *emr.RunJobFlowOutput) resource
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func testAccAWSEmrClusterConfig_bootstrap(r string) string {
|
||||||
|
return fmt.Sprintf(`
|
||||||
|
resource "aws_emr_cluster" "test" {
|
||||||
|
count = 1
|
||||||
|
name = "%s"
|
||||||
|
release_label = "emr-5.0.0"
|
||||||
|
applications = ["Hadoop", "Hive"]
|
||||||
|
log_uri = "s3n://terraform/testlog/"
|
||||||
|
master_instance_type = "m4.large"
|
||||||
|
core_instance_type = "m1.small"
|
||||||
|
core_instance_count = 1
|
||||||
|
service_role = "${aws_iam_role.iam_emr_default_role.arn}"
|
||||||
|
|
||||||
|
depends_on = ["aws_main_route_table_association.a"]
|
||||||
|
|
||||||
|
ec2_attributes {
|
||||||
|
subnet_id = "${aws_subnet.main.id}"
|
||||||
|
|
||||||
|
emr_managed_master_security_group = "${aws_security_group.allow_all.id}"
|
||||||
|
emr_managed_slave_security_group = "${aws_security_group.allow_all.id}"
|
||||||
|
instance_profile = "${aws_iam_instance_profile.emr_profile.arn}"
|
||||||
|
}
|
||||||
|
|
||||||
|
bootstrap_action {
|
||||||
|
path = "s3://elasticmapreduce/bootstrap-actions/run-if"
|
||||||
|
name = "runif"
|
||||||
|
args = ["instance.isMaster=true", "echo running on master node"]
|
||||||
|
}
|
||||||
|
|
||||||
|
bootstrap_action = [
|
||||||
|
{
|
||||||
|
path = "s3://${aws_s3_bucket.tester.bucket}/testscript.sh"
|
||||||
|
name = "test"
|
||||||
|
|
||||||
|
args = ["1",
|
||||||
|
"2",
|
||||||
|
"3",
|
||||||
|
"4",
|
||||||
|
"5",
|
||||||
|
"6",
|
||||||
|
"7",
|
||||||
|
"8",
|
||||||
|
"9",
|
||||||
|
"10",
|
||||||
|
]
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_iam_instance_profile" "emr_profile" {
|
||||||
|
name = "%s_profile"
|
||||||
|
role = "${aws_iam_role.iam_emr_profile_role.name}"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_iam_role" "iam_emr_default_role" {
|
||||||
|
name = "%s_default_role"
|
||||||
|
|
||||||
|
assume_role_policy = <<EOT
|
||||||
|
{
|
||||||
|
"Version": "2008-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Sid": "",
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Principal": {
|
||||||
|
"Service": "elasticmapreduce.amazonaws.com"
|
||||||
|
},
|
||||||
|
"Action": "sts:AssumeRole"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
EOT
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_iam_role" "iam_emr_profile_role" {
|
||||||
|
name = "%s_profile_role"
|
||||||
|
|
||||||
|
assume_role_policy = <<EOT
|
||||||
|
{
|
||||||
|
"Version": "2008-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Sid": "",
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Principal": {
|
||||||
|
"Service": "ec2.amazonaws.com"
|
||||||
|
},
|
||||||
|
"Action": "sts:AssumeRole"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
EOT
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_iam_role_policy_attachment" "profile-attach" {
|
||||||
|
role = "${aws_iam_role.iam_emr_profile_role.id}"
|
||||||
|
policy_arn = "${aws_iam_policy.iam_emr_profile_policy.arn}"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_iam_role_policy_attachment" "service-attach" {
|
||||||
|
role = "${aws_iam_role.iam_emr_default_role.id}"
|
||||||
|
policy_arn = "${aws_iam_policy.iam_emr_default_policy.arn}"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_iam_policy" "iam_emr_default_policy" {
|
||||||
|
name = "%s_emr"
|
||||||
|
|
||||||
|
policy = <<EOT
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Resource": "*",
|
||||||
|
"Action": [
|
||||||
|
"ec2:AuthorizeSecurityGroupEgress",
|
||||||
|
"ec2:AuthorizeSecurityGroupIngress",
|
||||||
|
"ec2:CancelSpotInstanceRequests",
|
||||||
|
"ec2:CreateNetworkInterface",
|
||||||
|
"ec2:CreateSecurityGroup",
|
||||||
|
"ec2:CreateTags",
|
||||||
|
"ec2:DeleteNetworkInterface",
|
||||||
|
"ec2:DeleteSecurityGroup",
|
||||||
|
"ec2:DeleteTags",
|
||||||
|
"ec2:DescribeAvailabilityZones",
|
||||||
|
"ec2:DescribeAccountAttributes",
|
||||||
|
"ec2:DescribeDhcpOptions",
|
||||||
|
"ec2:DescribeInstanceStatus",
|
||||||
|
"ec2:DescribeInstances",
|
||||||
|
"ec2:DescribeKeyPairs",
|
||||||
|
"ec2:DescribeNetworkAcls",
|
||||||
|
"ec2:DescribeNetworkInterfaces",
|
||||||
|
"ec2:DescribePrefixLists",
|
||||||
|
"ec2:DescribeRouteTables",
|
||||||
|
"ec2:DescribeSecurityGroups",
|
||||||
|
"ec2:DescribeSpotInstanceRequests",
|
||||||
|
"ec2:DescribeSpotPriceHistory",
|
||||||
|
"ec2:DescribeSubnets",
|
||||||
|
"ec2:DescribeVpcAttribute",
|
||||||
|
"ec2:DescribeVpcEndpoints",
|
||||||
|
"ec2:DescribeVpcEndpointServices",
|
||||||
|
"ec2:DescribeVpcs",
|
||||||
|
"ec2:DetachNetworkInterface",
|
||||||
|
"ec2:ModifyImageAttribute",
|
||||||
|
"ec2:ModifyInstanceAttribute",
|
||||||
|
"ec2:RequestSpotInstances",
|
||||||
|
"ec2:RevokeSecurityGroupEgress",
|
||||||
|
"ec2:RunInstances",
|
||||||
|
"ec2:TerminateInstances",
|
||||||
|
"ec2:DeleteVolume",
|
||||||
|
"ec2:DescribeVolumeStatus",
|
||||||
|
"iam:GetRole",
|
||||||
|
"iam:GetRolePolicy",
|
||||||
|
"iam:ListInstanceProfiles",
|
||||||
|
"iam:ListRolePolicies",
|
||||||
|
"iam:PassRole",
|
||||||
|
"s3:CreateBucket",
|
||||||
|
"s3:Get*",
|
||||||
|
"s3:List*",
|
||||||
|
"sdb:BatchPutAttributes",
|
||||||
|
"sdb:Select",
|
||||||
|
"sqs:CreateQueue",
|
||||||
|
"sqs:Delete*",
|
||||||
|
"sqs:GetQueue*",
|
||||||
|
"sqs:PurgeQueue",
|
||||||
|
"sqs:ReceiveMessage"
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
EOT
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_iam_policy" "iam_emr_profile_policy" {
|
||||||
|
name = "%s_profile"
|
||||||
|
|
||||||
|
policy = <<EOT
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [{
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Resource": "*",
|
||||||
|
"Action": [
|
||||||
|
"cloudwatch:*",
|
||||||
|
"dynamodb:*",
|
||||||
|
"ec2:Describe*",
|
||||||
|
"elasticmapreduce:Describe*",
|
||||||
|
"elasticmapreduce:ListBootstrapActions",
|
||||||
|
"elasticmapreduce:ListClusters",
|
||||||
|
"elasticmapreduce:ListInstanceGroups",
|
||||||
|
"elasticmapreduce:ListInstances",
|
||||||
|
"elasticmapreduce:ListSteps",
|
||||||
|
"kinesis:CreateStream",
|
||||||
|
"kinesis:DeleteStream",
|
||||||
|
"kinesis:DescribeStream",
|
||||||
|
"kinesis:GetRecords",
|
||||||
|
"kinesis:GetShardIterator",
|
||||||
|
"kinesis:MergeShards",
|
||||||
|
"kinesis:PutRecord",
|
||||||
|
"kinesis:SplitShard",
|
||||||
|
"rds:Describe*",
|
||||||
|
"s3:*",
|
||||||
|
"sdb:*",
|
||||||
|
"sns:*",
|
||||||
|
"sqs:*"
|
||||||
|
]
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
EOT
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_vpc" "main" {
|
||||||
|
cidr_block = "168.31.0.0/16"
|
||||||
|
enable_dns_hostnames = true
|
||||||
|
|
||||||
|
tags {
|
||||||
|
name = "emr_test_cts"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_subnet" "main" {
|
||||||
|
vpc_id = "${aws_vpc.main.id}"
|
||||||
|
cidr_block = "168.31.0.0/20"
|
||||||
|
|
||||||
|
tags {
|
||||||
|
name = "emr_test_cts"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_internet_gateway" "gw" {
|
||||||
|
vpc_id = "${aws_vpc.main.id}"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_route_table" "r" {
|
||||||
|
vpc_id = "${aws_vpc.main.id}"
|
||||||
|
|
||||||
|
route {
|
||||||
|
cidr_block = "0.0.0.0/0"
|
||||||
|
gateway_id = "${aws_internet_gateway.gw.id}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_main_route_table_association" "a" {
|
||||||
|
vpc_id = "${aws_vpc.main.id}"
|
||||||
|
route_table_id = "${aws_route_table.r.id}"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_security_group" "allow_all" {
|
||||||
|
name = "allow_all"
|
||||||
|
description = "Allow all inbound traffic"
|
||||||
|
vpc_id = "${aws_vpc.main.id}"
|
||||||
|
|
||||||
|
ingress {
|
||||||
|
from_port = 0
|
||||||
|
to_port = 0
|
||||||
|
protocol = "-1"
|
||||||
|
cidr_blocks = ["0.0.0.0/0"]
|
||||||
|
}
|
||||||
|
|
||||||
|
egress {
|
||||||
|
from_port = 0
|
||||||
|
to_port = 0
|
||||||
|
protocol = "-1"
|
||||||
|
cidr_blocks = ["0.0.0.0/0"]
|
||||||
|
}
|
||||||
|
|
||||||
|
depends_on = ["aws_subnet.main"]
|
||||||
|
|
||||||
|
lifecycle {
|
||||||
|
ignore_changes = ["ingress", "egress"]
|
||||||
|
}
|
||||||
|
|
||||||
|
tags {
|
||||||
|
name = "emr_test"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
output "cluser_id" {
|
||||||
|
value = "${aws_emr_cluster.test.id}"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket" "tester" {
|
||||||
|
bucket = "%s"
|
||||||
|
acl = "public-read"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "aws_s3_bucket_object" "testobject" {
|
||||||
|
bucket = "${aws_s3_bucket.tester.bucket}"
|
||||||
|
key = "testscript.sh"
|
||||||
|
|
||||||
|
#source = "testscript.sh"
|
||||||
|
content = "${data.template_file.testscript.rendered}"
|
||||||
|
acl = "public-read"
|
||||||
|
}
|
||||||
|
|
||||||
|
data "template_file" "testscript" {
|
||||||
|
template = <<POLICY
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
echo $@
|
||||||
|
POLICY
|
||||||
|
}`, r, r, r, r, r, r, r)
|
||||||
|
}
|
||||||
|
|
||||||
func testAccAWSEmrClusterConfig(r int) string {
|
func testAccAWSEmrClusterConfig(r int) string {
|
||||||
return fmt.Sprintf(`
|
return fmt.Sprintf(`
|
||||||
provider "aws" {
|
provider "aws" {
|
||||||
|
|
|
@ -24,6 +24,13 @@ func RandInt() int {
|
||||||
return rand.New(rand.NewSource(time.Now().UnixNano())).Int()
|
return rand.New(rand.NewSource(time.Now().UnixNano())).Int()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RandomWithPrefix is used to generate a unique name with a prefix, for
|
||||||
|
// randomizing names in acceptance tests
|
||||||
|
func RandomWithPrefix(name string) string {
|
||||||
|
reseed()
|
||||||
|
return fmt.Sprintf("%s-%d", name, rand.New(rand.NewSource(time.Now().UnixNano())).Int())
|
||||||
|
}
|
||||||
|
|
||||||
func RandIntRange(min int, max int) int {
|
func RandIntRange(min int, max int) int {
|
||||||
reseed()
|
reseed()
|
||||||
source := rand.New(rand.NewSource(time.Now().UnixNano()))
|
source := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||||
|
|
Loading…
Reference in New Issue