From 34e9e313774db97de45a5e44eb67f5b4058983a2 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Wed, 3 Jun 2015 17:05:02 -0700 Subject: [PATCH 1/7] Adding DynamoDB resource --- builtin/providers/aws/config.go | 31 +- builtin/providers/aws/provider.go | 1 + .../aws/resource_aws_dynamodb_table.go | 610 ++++++++++++++++++ 3 files changed, 629 insertions(+), 13 deletions(-) create mode 100644 builtin/providers/aws/resource_aws_dynamodb_table.go diff --git a/builtin/providers/aws/config.go b/builtin/providers/aws/config.go index 7908daf16..8ffd3be5e 100644 --- a/builtin/providers/aws/config.go +++ b/builtin/providers/aws/config.go @@ -7,19 +7,20 @@ import ( "github.com/hashicorp/terraform/helper/multierror" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/service/autoscaling" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/elasticache" - "github.com/aws/aws-sdk-go/service/elb" - "github.com/aws/aws-sdk-go/service/iam" - "github.com/aws/aws-sdk-go/service/kinesis" - "github.com/aws/aws-sdk-go/service/rds" - "github.com/aws/aws-sdk-go/service/route53" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/sns" - "github.com/aws/aws-sdk-go/service/sqs" + "github.com/awslabs/aws-sdk-go/aws" + "github.com/awslabs/aws-sdk-go/aws/credentials" + "github.com/awslabs/aws-sdk-go/service/autoscaling" + "github.com/awslabs/aws-sdk-go/service/ec2" + "github.com/awslabs/aws-sdk-go/service/elasticache" + "github.com/awslabs/aws-sdk-go/service/elb" + "github.com/awslabs/aws-sdk-go/service/iam" + "github.com/awslabs/aws-sdk-go/service/kinesis" + "github.com/awslabs/aws-sdk-go/service/rds" + "github.com/awslabs/aws-sdk-go/service/route53" + "github.com/awslabs/aws-sdk-go/service/s3" + "github.com/awslabs/aws-sdk-go/service/sns" + "github.com/awslabs/aws-sdk-go/service/sqs" + "github.com/awslabs/aws-sdk-go/service/dynamodb" ) type Config struct { @@ -34,6 +35,7 @@ type Config struct { } type AWSClient struct { + dynamodbconn *dynamodb.DynamoDB ec2conn *ec2.EC2 elbconn *elb.ELB autoscalingconn *autoscaling.AutoScaling @@ -84,6 +86,9 @@ func (c *Config) Client() (interface{}, error) { MaxRetries: c.MaxRetries, } + log.Println("[INFO] Initializing DynamoDB connection") + client.dynamodbconn = dynamodb.New(awsConfig) + log.Println("[INFO] Initializing ELB connection") client.elbconn = elb.New(awsConfig) diff --git a/builtin/providers/aws/provider.go b/builtin/providers/aws/provider.go index 9e0f928a4..885161765 100644 --- a/builtin/providers/aws/provider.go +++ b/builtin/providers/aws/provider.go @@ -90,6 +90,7 @@ func Provider() terraform.ResourceProvider { "aws_db_parameter_group": resourceAwsDbParameterGroup(), "aws_db_security_group": resourceAwsDbSecurityGroup(), "aws_db_subnet_group": resourceAwsDbSubnetGroup(), + "aws_dynamodb_table": resourceAwsDynamoDbTable(), "aws_ebs_volume": resourceAwsEbsVolume(), "aws_eip": resourceAwsEip(), "aws_elasticache_cluster": resourceAwsElasticacheCluster(), diff --git a/builtin/providers/aws/resource_aws_dynamodb_table.go b/builtin/providers/aws/resource_aws_dynamodb_table.go new file mode 100644 index 000000000..a4131eb66 --- /dev/null +++ b/builtin/providers/aws/resource_aws_dynamodb_table.go @@ -0,0 +1,610 @@ +package aws + +import ( + "bytes" + "fmt" + "log" + "time" + + "github.com/hashicorp/terraform/helper/schema" + + "github.com/awslabs/aws-sdk-go/aws" + "github.com/awslabs/aws-sdk-go/service/dynamodb" + "github.com/hashicorp/terraform/helper/hashcode" +) + +// A number of these are marked as computed because if you don't +// provide a value, DynamoDB will provide you with defaults (which are the +// default values specified below) +func resourceAwsDynamoDbTable() *schema.Resource { + return &schema.Resource{ + Create: resourceAwsDynamoDbTableCreate, + Read: resourceAwsDynamoDbTableRead, + Update: resourceAwsDynamoDbTableUpdate, + Delete: resourceAwsDynamoDbTableDelete, + + Schema: map[string]*schema.Schema{ + "name": &schema.Schema{ + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "hash_key": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "range_key": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + }, + "write_capacity": &schema.Schema{ + Type: schema.TypeInt, + Required: true, + }, + "read_capacity": &schema.Schema{ + Type: schema.TypeInt, + Required: true, + }, + "attribute": &schema.Schema{ + Type: schema.TypeSet, + Required: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "name": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "type": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + }, + }, + Set: func(v interface{}) int { + var buf bytes.Buffer + m := v.(map[string]interface{}) + buf.WriteString(fmt.Sprintf("%s-", m["name"].(string))) + return hashcode.String(buf.String()) + }, + }, + "local_secondary_index": &schema.Schema{ + Type: schema.TypeSet, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "name": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "range_key": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "projection_type": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "non_key_attributes": &schema.Schema{ + Type: schema.TypeList, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + }, + }, + Set: func(v interface{}) int { + var buf bytes.Buffer + m := v.(map[string]interface{}) + buf.WriteString(fmt.Sprintf("%s-", m["name"].(string))) + return hashcode.String(buf.String()) + }, + }, + "global_secondary_index": &schema.Schema{ + Type: schema.TypeSet, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "name": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "write_capacity": &schema.Schema{ + Type: schema.TypeInt, + Required: true, + }, + "read_capacity": &schema.Schema{ + Type: schema.TypeInt, + Required: true, + }, + "hash_key": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "range_key": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "projection_type": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "non_key_attributes": &schema.Schema{ + Type: schema.TypeList, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + }, + }, + // GSI names are the uniqueness constraint + Set: func(v interface{}) int { + var buf bytes.Buffer + m := v.(map[string]interface{}) + buf.WriteString(fmt.Sprintf("%s-", m["name"].(string))) + return hashcode.String(buf.String()) + }, + }, + }, + } +} + +func resourceAwsDynamoDbTableCreate(d *schema.ResourceData, meta interface{}) error { + dynamodbconn := meta.(*AWSClient).dynamodbconn + + name := d.Get("name").(string) + + log.Printf("[DEBUG] DynamoDB table create: %s", name) + + throughput := &dynamodb.ProvisionedThroughput{ + ReadCapacityUnits: aws.Long(int64(d.Get("read_capacity").(int))), + WriteCapacityUnits: aws.Long(int64(d.Get("write_capacity").(int))), + } + + hash_key_name := d.Get("hash_key").(string) + keyschema := []*dynamodb.KeySchemaElement{ + &dynamodb.KeySchemaElement{ + AttributeName: aws.String(hash_key_name), + KeyType: aws.String("HASH"), + }, + } + + if range_key, ok := d.GetOk("range_key"); ok { + range_schema_element := &dynamodb.KeySchemaElement{ + AttributeName: aws.String(range_key.(string)), + KeyType: aws.String("RANGE"), + } + keyschema = append(keyschema, range_schema_element) + } + + req := &dynamodb.CreateTableInput{ + TableName: aws.String(name), + ProvisionedThroughput: throughput, + KeySchema: keyschema, + } + + if attributedata, ok := d.GetOk("attribute"); ok { + attributes := []*dynamodb.AttributeDefinition{} + attributeSet := attributedata.(*schema.Set) + for _, attribute := range attributeSet.List() { + attr := attribute.(map[string]interface{}) + attributes = append(attributes, &dynamodb.AttributeDefinition{ + AttributeName: aws.String(attr["name"].(string)), + AttributeType: aws.String(attr["type"].(string)), + }) + } + + req.AttributeDefinitions = attributes + } + + if lsidata, ok := d.GetOk("local_secondary_index"); ok { + fmt.Printf("[DEBUG] Adding LSI data to the table") + + lsiSet := lsidata.(*schema.Set) + localSecondaryIndexes := []*dynamodb.LocalSecondaryIndex{} + for _, lsiObject := range lsiSet.List() { + lsi := lsiObject.(map[string]interface{}) + + projection := &dynamodb.Projection{ + ProjectionType: aws.String(lsi["projection_type"].(string)), + } + + if lsi["projection_type"] != "ALL" { + non_key_attributes := []*string{} + for _, attr := range lsi["non_key_attributes"].([]interface{}) { + non_key_attributes = append(non_key_attributes, aws.String(attr.(string))) + } + projection.NonKeyAttributes = non_key_attributes + } + + localSecondaryIndexes = append(localSecondaryIndexes, &dynamodb.LocalSecondaryIndex{ + IndexName: aws.String(lsi["name"].(string)), + KeySchema: []*dynamodb.KeySchemaElement{ + &dynamodb.KeySchemaElement{ + AttributeName: aws.String(hash_key_name), + KeyType: aws.String("HASH"), + }, + &dynamodb.KeySchemaElement{ + AttributeName: aws.String(lsi["range_key"].(string)), + KeyType: aws.String("RANGE"), + }, + }, + Projection: projection, + }) + } + + req.LocalSecondaryIndexes = localSecondaryIndexes + + fmt.Printf("[DEBUG] Added %d LSI definitions", len(localSecondaryIndexes)) + } + + if gsidata, ok := d.GetOk("global_secondary_index"); ok { + globalSecondaryIndexes := []*dynamodb.GlobalSecondaryIndex{} + + gsiSet := gsidata.(*schema.Set) + for _, gsiObject := range gsiSet.List() { + gsi := gsiObject.(map[string]interface{}) + gsiObject := createGSIFromData(&gsi) + globalSecondaryIndexes = append(globalSecondaryIndexes, &gsiObject) + } + req.GlobalSecondaryIndexes = globalSecondaryIndexes + } + + output, err := dynamodbconn.CreateTable(req) + if err != nil { + return fmt.Errorf("Error creating DynamoDB table: %s", err) + } + + d.SetId(*output.TableDescription.TableName) + + // Creation complete, nothing to re-read + return nil +} + +func resourceAwsDynamoDbTableUpdate(d *schema.ResourceData, meta interface{}) error { + + log.Printf("[DEBUG] Updating DynamoDB table %s", d.Id()) + dynamodbconn := meta.(*AWSClient).dynamodbconn + + // Ensure table is active before trying to update + waitForTableToBeActive(d.Id(), meta) + + // LSI can only be done at create-time, abort if it's been changed + if d.HasChange("local_secondary_index") { + return fmt.Errorf("Local secondary indexes can only be built at creation, you cannot update them!") + } + + if d.HasChange("hash_key") { + return fmt.Errorf("Hash key can only be specified at creation, you cannot modify it.") + } + + if d.HasChange("range_key") { + return fmt.Errorf("Range key can only be specified at creation, you cannot modify it.") + } + + if d.HasChange("attribute") { + req := &dynamodb.UpdateTableInput{ + TableName: aws.String(d.Id()), + } + + newAttributes := []*dynamodb.AttributeDefinition{} + _, n := d.GetChange("attribute") + newAttributeSet := n.(*schema.Set) + for _, attribute := range newAttributeSet.List() { + attr := attribute.(map[string]interface{}) + newAttributes = append(newAttributes, &dynamodb.AttributeDefinition{ + AttributeName: aws.String(attr["name"].(string)), + AttributeType: aws.String(attr["type"].(string)), + }) + } + + req.AttributeDefinitions = newAttributes + } + + if d.HasChange("read_capacity") || d.HasChange("write_capacity") { + req := &dynamodb.UpdateTableInput{ + TableName: aws.String(d.Id()), + } + + throughput := &dynamodb.ProvisionedThroughput{ + ReadCapacityUnits: aws.Long(int64(d.Get("read_capacity").(int))), + WriteCapacityUnits: aws.Long(int64(d.Get("write_capacity").(int))), + } + req.ProvisionedThroughput = throughput + + _, err := dynamodbconn.UpdateTable(req) + + if err != nil { + return err + } + + waitForTableToBeActive(d.Id(), meta) + } + + if d.HasChange("global_secondary_index") { + req := &dynamodb.UpdateTableInput{ + TableName: aws.String(d.Id()), + } + + o, n := d.GetChange("global_secondary_index") + + oldSet := o.(*schema.Set) + newSet := n.(*schema.Set) + changedSet := newSet.Intersection(oldSet) + + // First determine what's new + for _, newgsidata := range newSet.List() { + updates := []*dynamodb.GlobalSecondaryIndexUpdate{} + if !oldSet.Contains(newgsidata) { + attributes := []*dynamodb.AttributeDefinition{} + gsidata := newgsidata.(map[string]interface{}) + gsi := createGSIFromData(&gsidata) + log.Printf("[DEBUG] Adding GSI %s", *gsi.IndexName) + update := &dynamodb.GlobalSecondaryIndexUpdate{ + Create: &dynamodb.CreateGlobalSecondaryIndexAction{ + IndexName: gsi.IndexName, + KeySchema: gsi.KeySchema, + ProvisionedThroughput: gsi.ProvisionedThroughput, + Projection: gsi.Projection, + }, + } + updates = append(updates, update) + hashkey_type, err := getAttributeType(d, *(gsi.KeySchema[0].AttributeName)) + if err != nil { + return err + } + + rangekey_type, err := getAttributeType(d, *(gsi.KeySchema[1].AttributeName)) + if err != nil { + return err + } + + attributes = append(attributes, &dynamodb.AttributeDefinition{ + AttributeName: gsi.KeySchema[0].AttributeName, + AttributeType: aws.String(hashkey_type), + }) + attributes = append(attributes, &dynamodb.AttributeDefinition{ + AttributeName: gsi.KeySchema[1].AttributeName, + AttributeType: aws.String(rangekey_type), + }) + + req.AttributeDefinitions = attributes + req.GlobalSecondaryIndexUpdates = updates + _, err = dynamodbconn.UpdateTable(req) + + if err != nil { + return err + } + + waitForTableToBeActive(d.Id(), meta) + waitForGSIToBeActive(d.Id(), *gsi.IndexName, meta) + + } + } + + for _, oldgsidata := range oldSet.List() { + updates := []*dynamodb.GlobalSecondaryIndexUpdate{} + if !newSet.Contains(oldgsidata) { + gsidata := oldgsidata.(map[string]interface{}) + log.Printf("[DEBUG] Deleting GSI %s", gsidata["name"].(string)) + update := &dynamodb.GlobalSecondaryIndexUpdate{ + Delete: &dynamodb.DeleteGlobalSecondaryIndexAction{ + IndexName: aws.String(gsidata["name"].(string)), + }, + } + updates = append(updates, update) + + req.GlobalSecondaryIndexUpdates = updates + _, err := dynamodbconn.UpdateTable(req) + + if err != nil { + return err + } + + waitForTableToBeActive(d.Id(), meta) + } + } + + for _, updatedgsidata := range changedSet.List() { + updates := []*dynamodb.GlobalSecondaryIndexUpdate{} + gsidata := updatedgsidata.(map[string]interface{}) + log.Printf("[DEBUG] Updating GSI %s", gsidata["name"].(string)) + update := &dynamodb.GlobalSecondaryIndexUpdate{ + Update: &dynamodb.UpdateGlobalSecondaryIndexAction{ + IndexName: aws.String(gsidata["name"].(string)), + ProvisionedThroughput: &dynamodb.ProvisionedThroughput{ + WriteCapacityUnits: aws.Long(int64(gsidata["write_capacity"].(int))), + ReadCapacityUnits: aws.Long(int64(gsidata["read_capacity"].(int))), + }, + }, + } + updates = append(updates, update) + + req.GlobalSecondaryIndexUpdates = updates + + _, err := dynamodbconn.UpdateTable(req) + + if err != nil { + log.Printf("[DEBUG] Error updating table: %s", err) + return err + } + } + } + + return resourceAwsDynamoDbTableRead(d, meta) +} + +func resourceAwsDynamoDbTableRead(d *schema.ResourceData, meta interface{}) error { + dynamodbconn := meta.(*AWSClient).dynamodbconn + req := &dynamodb.DescribeTableInput{ + TableName: aws.String(d.Id()), + } + + result, err := dynamodbconn.DescribeTable(req) + + if err != nil { + return err + } + + table := result.Table + + d.Set("write_capacity", table.ProvisionedThroughput.WriteCapacityUnits) + d.Set("read_capacity", table.ProvisionedThroughput.ReadCapacityUnits) + + attributes := []interface{}{} + for _, attrdef := range table.AttributeDefinitions { + attribute := make(map[string]string) + attribute["name"] = *(attrdef.AttributeName) + attribute["type"] = *(attrdef.AttributeType) + attributes = append(attributes, attribute) + } + + d.Set("attribute", attributes) + + gsiList := []interface{}{} + for _, gsiObject := range table.GlobalSecondaryIndexes { + gsi := make(map[string]interface{}) + gsi["write_capacity"] = gsiObject.ProvisionedThroughput.WriteCapacityUnits + gsi["read_capacity"] = gsiObject.ProvisionedThroughput.ReadCapacityUnits + gsi["name"] = gsiObject.IndexName + gsiList = append(gsiList, gsi) + } + + d.Set("global_secondary_index", gsiList) + + return nil +} + +func resourceAwsDynamoDbTableDelete(d *schema.ResourceData, meta interface{}) error { + dynamodbconn := meta.(*AWSClient).dynamodbconn + + waitForTableToBeActive(d.Id(), meta) + + log.Printf("[DEBUG] DynamoDB delete table: %s", d.Id()) + + _, err := dynamodbconn.DeleteTable(&dynamodb.DeleteTableInput{ + TableName: aws.String(d.Id()), + }) + if err != nil { + return err + } + return nil +} + +func createGSIFromData(data *map[string]interface{}) dynamodb.GlobalSecondaryIndex { + + projection := &dynamodb.Projection{ + ProjectionType: aws.String((*data)["projection_type"].(string)), + } + + if (*data)["projection_type"] != "ALL" { + non_key_attributes := []*string{} + for _, attr := range (*data)["non_key_attributes"].([]interface{}) { + non_key_attributes = append(non_key_attributes, aws.String(attr.(string))) + } + projection.NonKeyAttributes = non_key_attributes + } + + writeCapacity := (*data)["write_capacity"].(int) + readCapacity := (*data)["read_capacity"].(int) + + return dynamodb.GlobalSecondaryIndex{ + IndexName: aws.String((*data)["name"].(string)), + KeySchema: []*dynamodb.KeySchemaElement{ + &dynamodb.KeySchemaElement{ + AttributeName: aws.String((*data)["hash_key"].(string)), + KeyType: aws.String("HASH"), + }, + &dynamodb.KeySchemaElement{ + AttributeName: aws.String((*data)["range_key"].(string)), + KeyType: aws.String("RANGE"), + }, + }, + Projection: projection, + ProvisionedThroughput: &dynamodb.ProvisionedThroughput{ + WriteCapacityUnits: aws.Long(int64(writeCapacity)), + ReadCapacityUnits: aws.Long(int64(readCapacity)), + }, + } +} + +func getAttributeType(d *schema.ResourceData, attributeName string) (string, error) { + if attributedata, ok := d.GetOk("attribute"); ok { + attributeSet := attributedata.(*schema.Set) + for _, attribute := range attributeSet.List() { + attr := attribute.(map[string]interface{}) + if attr["name"] == attributeName { + return attr["type"].(string), nil + } + } + } + + return "", fmt.Errorf("Unable to find an attribute named %s", attributeName) +} + +func waitForGSIToBeActive(tableName string, gsiName string, meta interface{}) error { + dynamodbconn := meta.(*AWSClient).dynamodbconn + req := &dynamodb.DescribeTableInput{ + TableName: aws.String(tableName), + } + + activeIndex := false + + for activeIndex == false { + + result, err := dynamodbconn.DescribeTable(req) + + if err != nil { + return err + } + + table := result.Table + var targetGSI *dynamodb.GlobalSecondaryIndexDescription = nil + + for _, gsi := range table.GlobalSecondaryIndexes { + if *gsi.IndexName == gsiName { + targetGSI = gsi + } + } + + if targetGSI != nil { + activeIndex = *targetGSI.IndexStatus == "ACTIVE" + + if !activeIndex { + log.Printf("[DEBUG] Sleeping for 3 seconds for %s GSI to become active", gsiName) + time.Sleep(3 * time.Second) + } + } else { + log.Printf("[DEBUG] GSI %s did not exist, giving up", gsiName) + break + } + } + + return nil + +} + +func waitForTableToBeActive(tableName string, meta interface{}) error { + dynamodbconn := meta.(*AWSClient).dynamodbconn + req := &dynamodb.DescribeTableInput{ + TableName: aws.String(tableName), + } + + activeState := false + + for activeState == false { + result, err := dynamodbconn.DescribeTable(req) + + if err != nil { + return err + } + + activeState = *(result.Table.TableStatus) == "ACTIVE" + + // Wait for a few seconds + if !activeState { + log.Printf("[DEBUG] Sleeping for 3 seconds for table to become active") + time.Sleep(3 * time.Second) + } + } + + return nil + +} From 1669c35007f5c20fc156766456a5e1b593766868 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Wed, 3 Jun 2015 17:05:40 -0700 Subject: [PATCH 2/7] Adding DynamoDB acceptance tests --- .../aws/resource_aws_dynamodb_table_test.go | 296 ++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 builtin/providers/aws/resource_aws_dynamodb_table_test.go diff --git a/builtin/providers/aws/resource_aws_dynamodb_table_test.go b/builtin/providers/aws/resource_aws_dynamodb_table_test.go new file mode 100644 index 000000000..149f36a55 --- /dev/null +++ b/builtin/providers/aws/resource_aws_dynamodb_table_test.go @@ -0,0 +1,296 @@ +package aws + +import ( + "fmt" + "testing" + + "github.com/awslabs/aws-sdk-go/aws" + "github.com/awslabs/aws-sdk-go/aws/awserr" + "github.com/awslabs/aws-sdk-go/service/dynamodb" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" +) + +func TestAccAWSDynamoDbTable(t *testing.T) { + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSDynamoDbTableDestroy, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: testAccAWSDynamoDbConfigInitialState, + Check: resource.ComposeTestCheckFunc( + testAccCheckInitialAWSDynamoDbTableExists("aws_dynamodb_table.basic-dynamodb-table"), + ), + }, + resource.TestStep{ + Config: testAccAWSDynamoDbConfigAddSecondaryGSI, + Check: resource.ComposeTestCheckFunc( + testAccCheckDynamoDbTableWasUpdated("aws_dynamodb_table.basic-dynamodb-table"), + ), + }, + }, + }) +} + +func testAccCheckAWSDynamoDbTableDestroy(s *terraform.State) error { + conn := testAccProvider.Meta().(*AWSClient).dynamodbconn + + for _, rs := range s.RootModule().Resources { + if rs.Type != "aws_dynamodb_table" { + continue + } + + fmt.Printf("[DEBUG] Checking if DynamoDB table %s exists", rs.Primary.ID) + // Check if queue exists by checking for its attributes + params := &dynamodb.DescribeTableInput{ + TableName: aws.String(rs.Primary.ID), + } + _, err := conn.DescribeTable(params) + if err == nil { + return fmt.Errorf("DynamoDB table %s still exists. Failing!", rs.Primary.ID) + } + + // Verify the error is what we want + _, ok := err.(awserr.Error) + if !ok { + return err + } + } + + return nil +} + +func testAccCheckInitialAWSDynamoDbTableExists(n string) resource.TestCheckFunc { + return func(s *terraform.State) error { + fmt.Printf("[DEBUG] Trying to create initial table state!") + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Not found: %s", n) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("No DynamoDB table name specified!") + } + + conn := testAccProvider.Meta().(*AWSClient).dynamodbconn + + params := &dynamodb.DescribeTableInput{ + TableName: aws.String(rs.Primary.ID), + } + + resp, err := conn.DescribeTable(params) + + if err != nil { + fmt.Printf("[ERROR] Problem describing table '%s': %s", rs.Primary.ID, err) + return err + } + + table := resp.Table + + fmt.Printf("[DEBUG] Checking on table %s", rs.Primary.ID) + + if *table.ProvisionedThroughput.WriteCapacityUnits != 20 { + return fmt.Errorf("Provisioned write capacity was %d, not 20!", table.ProvisionedThroughput.WriteCapacityUnits) + } + + if *table.ProvisionedThroughput.ReadCapacityUnits != 10 { + return fmt.Errorf("Provisioned read capacity was %d, not 10!", table.ProvisionedThroughput.ReadCapacityUnits) + } + + attrCount := len(table.AttributeDefinitions) + gsiCount := len(table.GlobalSecondaryIndexes) + lsiCount := len(table.LocalSecondaryIndexes) + + if attrCount != 4 { + return fmt.Errorf("There were %d attributes, not 4 like there should have been!", attrCount) + } + + if gsiCount != 1 { + return fmt.Errorf("There were %d GSIs, not 1 like there should have been!", gsiCount) + } + + if lsiCount != 1 { + return fmt.Errorf("There were %d LSIs, not 1 like there should have been!", lsiCount) + } + + attrmap := dynamoDbAttributesToMap(&table.AttributeDefinitions) + if attrmap["TestTableHashKey"] != "S" { + return fmt.Errorf("Test table hash key was of type %s instead of S!", attrmap["TestTableHashKey"]) + } + if attrmap["TestTableRangeKey"] != "S" { + return fmt.Errorf("Test table range key was of type %s instead of S!", attrmap["TestTableRangeKey"]) + } + if attrmap["TestLSIRangeKey"] != "N" { + return fmt.Errorf("Test table LSI range key was of type %s instead of N!", attrmap["TestLSIRangeKey"]) + } + if attrmap["TestGSIRangeKey"] != "S" { + return fmt.Errorf("Test table GSI range key was of type %s instead of S!", attrmap["TestGSIRangeKey"]) + } + + return nil + } +} + +func testAccCheckDynamoDbTableWasUpdated(n string) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Not found: %s", n) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("No DynamoDB table name specified!") + } + + conn := testAccProvider.Meta().(*AWSClient).dynamodbconn + + params := &dynamodb.DescribeTableInput{ + TableName: aws.String(rs.Primary.ID), + } + resp, err := conn.DescribeTable(params) + table := resp.Table + + if err != nil { + return err + } + + attrCount := len(table.AttributeDefinitions) + gsiCount := len(table.GlobalSecondaryIndexes) + lsiCount := len(table.LocalSecondaryIndexes) + + if attrCount != 4 { + return fmt.Errorf("There were %d attributes, not 4 like there should have been!", attrCount) + } + + if gsiCount != 1 { + return fmt.Errorf("There were %d GSIs, not 1 like there should have been!", gsiCount) + } + + if lsiCount != 1 { + return fmt.Errorf("There were %d LSIs, not 1 like there should have been!", lsiCount) + } + + if dynamoDbGetGSIIndex(&table.GlobalSecondaryIndexes, "ReplacementTestTableGSI") == -1 { + return fmt.Errorf("Could not find GSI named 'ReplacementTestTableGSI' in the table!") + } + + if dynamoDbGetGSIIndex(&table.GlobalSecondaryIndexes, "InitialTestTableGSI") != -1 { + return fmt.Errorf("Should have removed 'InitialTestTableGSI' but it still exists!") + } + + attrmap := dynamoDbAttributesToMap(&table.AttributeDefinitions) + if attrmap["TestTableHashKey"] != "S" { + return fmt.Errorf("Test table hash key was of type %s instead of S!", attrmap["TestTableHashKey"]) + } + if attrmap["TestTableRangeKey"] != "S" { + return fmt.Errorf("Test table range key was of type %s instead of S!", attrmap["TestTableRangeKey"]) + } + if attrmap["TestLSIRangeKey"] != "N" { + return fmt.Errorf("Test table LSI range key was of type %s instead of N!", attrmap["TestLSIRangeKey"]) + } + if attrmap["ReplacementGSIRangeKey"] != "N" { + return fmt.Errorf("Test table replacement GSI range key was of type %s instead of N!", attrmap["ReplacementGSIRangeKey"]) + } + + return nil + } +} + +func dynamoDbGetGSIIndex(gsiList *[]*dynamodb.GlobalSecondaryIndexDescription, target string) int { + for idx, gsiObject := range *gsiList { + if *gsiObject.IndexName == target { + return idx + } + } + + return -1 +} + +func dynamoDbAttributesToMap(attributes *[]*dynamodb.AttributeDefinition) map[string]string { + attrmap := make(map[string]string) + + for _, attrdef := range *attributes { + attrmap[*(attrdef.AttributeName)] = *(attrdef.AttributeType) + } + + return attrmap +} + +const testAccAWSDynamoDbConfigInitialState = ` +resource "aws_dynamodb_table" "basic-dynamodb-table" { + name = "TerraformTestTable" + read_capacity = 10 + write_capacity = 20 + hash_key = "TestTableHashKey" + range_key = "TestTableRangeKey" + attribute { + name = "TestTableHashKey" + type = "S" + } + attribute { + name = "TestTableRangeKey" + type = "S" + } + attribute { + name = "TestLSIRangeKey" + type = "N" + } + attribute { + name = "TestGSIRangeKey" + type = "S" + } + local_secondary_index { + name = "TestTableLSI" + range_key = "TestLSIRangeKey" + projection_type = "ALL" + } + global_secondary_index { + name = "InitialTestTableGSI" + hash_key = "TestTableHashKey" + range_key = "TestGSIRangeKey" + write_capacity = 10 + read_capacity = 10 + projection_type = "ALL" + } +} +` + +const testAccAWSDynamoDbConfigAddSecondaryGSI = ` +resource "aws_dynamodb_table" "basic-dynamodb-table" { + name = "TerraformTestTable" + read_capacity = 20 + write_capacity = 20 + hash_key = "TestTableHashKey" + range_key = "TestTableRangeKey" + attribute { + name = "TestTableHashKey" + type = "S" + } + attribute { + name = "TestTableRangeKey" + type = "S" + } + attribute { + name = "TestLSIRangeKey" + type = "N" + } + attribute { + name = "ReplacementGSIRangeKey" + type = "N" + } + local_secondary_index { + name = "TestTableLSI" + range_key = "TestLSIRangeKey" + projection_type = "ALL" + } + global_secondary_index { + name = "ReplacementTestTableGSI" + hash_key = "TestTableHashKey" + range_key = "ReplacementGSIRangeKey" + write_capacity = 5 + read_capacity = 5 + projection_type = "ALL" + } +} +` From bbc6f14807e826120aa27c028521540b4234b9f3 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Wed, 3 Jun 2015 17:07:03 -0700 Subject: [PATCH 3/7] Adding DynamoDB documentation --- .../aws/r/dynamodb_table.html.markdown | 109 ++++++++++++++++++ website/source/layouts/aws.erb | 4 + 2 files changed, 113 insertions(+) create mode 100644 website/source/docs/providers/aws/r/dynamodb_table.html.markdown diff --git a/website/source/docs/providers/aws/r/dynamodb_table.html.markdown b/website/source/docs/providers/aws/r/dynamodb_table.html.markdown new file mode 100644 index 000000000..e176f39a0 --- /dev/null +++ b/website/source/docs/providers/aws/r/dynamodb_table.html.markdown @@ -0,0 +1,109 @@ +--- +layout: "aws" +page_title: "AWS: dynamodb_table" +sidebar_current: "docs-aws-resource-dynamodb-table" +description: |- + Provides a DynamoDB table resource +--- + +# aws\_dynamodb\_table + +Provides a DynamoDB table resource + +## Example Usage + +The following dynamodb table description models the table and GSI shown +in the [AWS SDK example documentation](http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GSI.html) + +``` +resource "aws_dynamodb_table" "basic-dynamodb-table" { + name = "GameScores" + read_capacity = 20 + write_capacity = 20 + hash_key = "UserId" + range_key = "GameTitle" + attribute { + name = "Username" + type = "S" + } + attribute { + name = "GameTitle" + type = "S" + } + attribute { + name = "TopScore" + type = "N" + } + attribute { + name = "TopScoreDateTime" + type = "S" + } + attribute { + name = "Wins" + type = "N" + } + attribute { + name = "Losses" + type = "N" + } + global_secondary_index { + name = "GameTitleIndex" + hash_key = "GameTitle" + range_key = "TopScore" + write_capacity = 10 + read_capacity = 10 + projection_type = "INCLUDE" + non_key_attributes = [ "UserId" ] + } +} +``` + +## Argument Reference + +The following arguments are supported: + +* `name` - (Required) The name of the table, this needs to be unique + within a region. +* `read_capacity` - (Required) The number of read units for this table +* `write_capacity` - (Required) The number of write units for this table +* `hash_key` - (Required) The attribute to use as the hash key (the + attribute must also be defined as an attribute record +* `range_key` - (Optional) The attribute to use as the range key (must + also be defined) +* `attribute` - Define an attribute, has two properties: + * `name` - The name of the attribute + * `type` - One of: S, N, or B for (S)tring, (N)umber or (B)inary data +* `local_secondary_index` - (Optional) Describe an LSI on the table; + these can only be allocated *at creation* so you cannot change this +definition after you have created the resource. +* `global_secondary_index` - (Optional) Describe a GSO for the table; + subject to the normal limits on the number of GSIs, projected +attributes, etc. + +For both `local_secondary_index` and `global_secondary_index` objects, +the following properties are supported: + +* `name` - (Required) The name of the LSI or GSI +* `hash_key` - (Required) The name of the hash key in the index; must be + defined as an attribute in the resource +* `range_key` - (Required) The name of the range key; must be defined +* `projection_type` - (Required) One of "ALL", "INCLUDE" or "KEYS_ONLY" + where *ALL* projects every attribute into the index, *KEYS_ONLY* + projects just the hash and range key into the index, and *INCLUDE* + projects only the keys specified in the _non_key_attributes_ +parameter. +* `non_key_attributes` - (Optional) Only required with *INCLUDE* as a + projection type; a list of attributes to project into the index. For +each attribute listed, you need to make sure that it has been defined in +the table object. + +For `global_secondary_index` objects only, you need to specify +`write_capacity` and `read_capacity` in the same way you would for the +table as they have separate I/O capacity. + +## Attributes Reference + +The following attributes are exported: + +* `id` - The name of the table + diff --git a/website/source/layouts/aws.erb b/website/source/layouts/aws.erb index 219b3ed3c..2239975f3 100644 --- a/website/source/layouts/aws.erb +++ b/website/source/layouts/aws.erb @@ -37,6 +37,10 @@ aws_db_subnet_group + > + aws_dynamodb_table + + > aws_ebs_volume From 4784eff9cad317a2aae6e445a2d86d0333622e0c Mon Sep 17 00:00:00 2001 From: John Ewart Date: Wed, 3 Jun 2015 17:12:41 -0700 Subject: [PATCH 4/7] Fix AWS SDK imports --- builtin/providers/aws/config.go | 28 +++++++++---------- .../aws/resource_aws_dynamodb_table.go | 4 +-- .../aws/resource_aws_dynamodb_table_test.go | 6 ++-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/builtin/providers/aws/config.go b/builtin/providers/aws/config.go index 8ffd3be5e..ee491b59f 100644 --- a/builtin/providers/aws/config.go +++ b/builtin/providers/aws/config.go @@ -7,20 +7,20 @@ import ( "github.com/hashicorp/terraform/helper/multierror" - "github.com/awslabs/aws-sdk-go/aws" - "github.com/awslabs/aws-sdk-go/aws/credentials" - "github.com/awslabs/aws-sdk-go/service/autoscaling" - "github.com/awslabs/aws-sdk-go/service/ec2" - "github.com/awslabs/aws-sdk-go/service/elasticache" - "github.com/awslabs/aws-sdk-go/service/elb" - "github.com/awslabs/aws-sdk-go/service/iam" - "github.com/awslabs/aws-sdk-go/service/kinesis" - "github.com/awslabs/aws-sdk-go/service/rds" - "github.com/awslabs/aws-sdk-go/service/route53" - "github.com/awslabs/aws-sdk-go/service/s3" - "github.com/awslabs/aws-sdk-go/service/sns" - "github.com/awslabs/aws-sdk-go/service/sqs" - "github.com/awslabs/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/elasticache" + "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/kinesis" + "github.com/aws/aws-sdk-go/service/rds" + "github.com/aws/aws-sdk-go/service/route53" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/sns" + "github.com/aws/aws-sdk-go/service/sqs" ) type Config struct { diff --git a/builtin/providers/aws/resource_aws_dynamodb_table.go b/builtin/providers/aws/resource_aws_dynamodb_table.go index a4131eb66..75cf664db 100644 --- a/builtin/providers/aws/resource_aws_dynamodb_table.go +++ b/builtin/providers/aws/resource_aws_dynamodb_table.go @@ -8,8 +8,8 @@ import ( "github.com/hashicorp/terraform/helper/schema" - "github.com/awslabs/aws-sdk-go/aws" - "github.com/awslabs/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/hashicorp/terraform/helper/hashcode" ) diff --git a/builtin/providers/aws/resource_aws_dynamodb_table_test.go b/builtin/providers/aws/resource_aws_dynamodb_table_test.go index 149f36a55..786a946b6 100644 --- a/builtin/providers/aws/resource_aws_dynamodb_table_test.go +++ b/builtin/providers/aws/resource_aws_dynamodb_table_test.go @@ -4,9 +4,9 @@ import ( "fmt" "testing" - "github.com/awslabs/aws-sdk-go/aws" - "github.com/awslabs/aws-sdk-go/aws/awserr" - "github.com/awslabs/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/hashicorp/terraform/helper/resource" "github.com/hashicorp/terraform/terraform" ) From f458521be9fd630257d17ecb9af03f53d31995dd Mon Sep 17 00:00:00 2001 From: John Ewart Date: Mon, 8 Jun 2015 15:58:42 -0700 Subject: [PATCH 5/7] Remove request for attribute changes --- .../aws/resource_aws_dynamodb_table.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/builtin/providers/aws/resource_aws_dynamodb_table.go b/builtin/providers/aws/resource_aws_dynamodb_table.go index 75cf664db..e8e8add8c 100644 --- a/builtin/providers/aws/resource_aws_dynamodb_table.go +++ b/builtin/providers/aws/resource_aws_dynamodb_table.go @@ -279,25 +279,6 @@ func resourceAwsDynamoDbTableUpdate(d *schema.ResourceData, meta interface{}) er return fmt.Errorf("Range key can only be specified at creation, you cannot modify it.") } - if d.HasChange("attribute") { - req := &dynamodb.UpdateTableInput{ - TableName: aws.String(d.Id()), - } - - newAttributes := []*dynamodb.AttributeDefinition{} - _, n := d.GetChange("attribute") - newAttributeSet := n.(*schema.Set) - for _, attribute := range newAttributeSet.List() { - attr := attribute.(map[string]interface{}) - newAttributes = append(newAttributes, &dynamodb.AttributeDefinition{ - AttributeName: aws.String(attr["name"].(string)), - AttributeType: aws.String(attr["type"].(string)), - }) - } - - req.AttributeDefinitions = newAttributes - } - if d.HasChange("read_capacity") || d.HasChange("write_capacity") { req := &dynamodb.UpdateTableInput{ TableName: aws.String(d.Id()), From 320e4b222c4905f7550708541b489db07703ad8f Mon Sep 17 00:00:00 2001 From: John Ewart Date: Mon, 8 Jun 2015 16:04:22 -0700 Subject: [PATCH 6/7] Change sleep time for DynamoDB table waits from 3 seconds to 5 seconds --- builtin/providers/aws/resource_aws_dynamodb_table.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builtin/providers/aws/resource_aws_dynamodb_table.go b/builtin/providers/aws/resource_aws_dynamodb_table.go index e8e8add8c..bbc0fd861 100644 --- a/builtin/providers/aws/resource_aws_dynamodb_table.go +++ b/builtin/providers/aws/resource_aws_dynamodb_table.go @@ -549,8 +549,8 @@ func waitForGSIToBeActive(tableName string, gsiName string, meta interface{}) er activeIndex = *targetGSI.IndexStatus == "ACTIVE" if !activeIndex { - log.Printf("[DEBUG] Sleeping for 3 seconds for %s GSI to become active", gsiName) - time.Sleep(3 * time.Second) + log.Printf("[DEBUG] Sleeping for 5 seconds for %s GSI to become active", gsiName) + time.Sleep(5 * time.Second) } } else { log.Printf("[DEBUG] GSI %s did not exist, giving up", gsiName) From 4e219b3bad3b34532b58f2f8119ff658b7fe97f1 Mon Sep 17 00:00:00 2001 From: John Ewart Date: Mon, 15 Jun 2015 17:05:50 -0700 Subject: [PATCH 7/7] Fixes support for changing just the read / write capacity of a GSI --- .../aws/resource_aws_dynamodb_table.go | 217 +++++++++++++----- 1 file changed, 165 insertions(+), 52 deletions(-) diff --git a/builtin/providers/aws/resource_aws_dynamodb_table.go b/builtin/providers/aws/resource_aws_dynamodb_table.go index bbc0fd861..163d14d8d 100644 --- a/builtin/providers/aws/resource_aws_dynamodb_table.go +++ b/builtin/providers/aws/resource_aws_dynamodb_table.go @@ -121,7 +121,7 @@ func resourceAwsDynamoDbTable() *schema.Resource { }, "range_key": &schema.Schema{ Type: schema.TypeString, - Required: true, + Optional: true, }, "projection_type": &schema.Schema{ Type: schema.TypeString, @@ -139,6 +139,8 @@ func resourceAwsDynamoDbTable() *schema.Resource { var buf bytes.Buffer m := v.(map[string]interface{}) buf.WriteString(fmt.Sprintf("%s-", m["name"].(string))) + buf.WriteString(fmt.Sprintf("%d-", m["write_capacity"].(int))) + buf.WriteString(fmt.Sprintf("%d-", m["read_capacity"].(int))) return hashcode.String(buf.String()) }, }, @@ -300,6 +302,7 @@ func resourceAwsDynamoDbTableUpdate(d *schema.ResourceData, meta interface{}) er } if d.HasChange("global_secondary_index") { + log.Printf("[DEBUG] Changed GSI data") req := &dynamodb.UpdateTableInput{ TableName: aws.String(d.Id()), } @@ -308,12 +311,29 @@ func resourceAwsDynamoDbTableUpdate(d *schema.ResourceData, meta interface{}) er oldSet := o.(*schema.Set) newSet := n.(*schema.Set) - changedSet := newSet.Intersection(oldSet) + + // Track old names so we can know which ones we need to just update based on + // capacity changes, terraform appears to only diff on the set hash, not the + // contents so we need to make sure we don't delete any indexes that we + // just want to update the capacity for + oldGsiNameSet := make(map[string]bool) + newGsiNameSet := make(map[string]bool) + + for _, gsidata := range oldSet.List() { + gsiName := gsidata.(map[string]interface{})["name"].(string) + oldGsiNameSet[gsiName] = true + } + + for _, gsidata := range newSet.List() { + gsiName := gsidata.(map[string]interface{})["name"].(string) + newGsiNameSet[gsiName] = true + } // First determine what's new for _, newgsidata := range newSet.List() { updates := []*dynamodb.GlobalSecondaryIndexUpdate{} - if !oldSet.Contains(newgsidata) { + newGsiName := newgsidata.(map[string]interface{})["name"].(string) + if _, exists := oldGsiNameSet[newGsiName]; !exists { attributes := []*dynamodb.AttributeDefinition{} gsidata := newgsidata.(map[string]interface{}) gsi := createGSIFromData(&gsidata) @@ -327,12 +347,9 @@ func resourceAwsDynamoDbTableUpdate(d *schema.ResourceData, meta interface{}) er }, } updates = append(updates, update) - hashkey_type, err := getAttributeType(d, *(gsi.KeySchema[0].AttributeName)) - if err != nil { - return err - } - rangekey_type, err := getAttributeType(d, *(gsi.KeySchema[1].AttributeName)) + // Hash key is required, range key isn't + hashkey_type, err := getAttributeType(d, *(gsi.KeySchema[0].AttributeName)) if err != nil { return err } @@ -341,10 +358,19 @@ func resourceAwsDynamoDbTableUpdate(d *schema.ResourceData, meta interface{}) er AttributeName: gsi.KeySchema[0].AttributeName, AttributeType: aws.String(hashkey_type), }) - attributes = append(attributes, &dynamodb.AttributeDefinition{ - AttributeName: gsi.KeySchema[1].AttributeName, - AttributeType: aws.String(rangekey_type), - }) + + // If there's a range key, there will be 2 elements in KeySchema + if len(gsi.KeySchema) == 2 { + rangekey_type, err := getAttributeType(d, *(gsi.KeySchema[1].AttributeName)) + if err != nil { + return err + } + + attributes = append(attributes, &dynamodb.AttributeDefinition{ + AttributeName: gsi.KeySchema[1].AttributeName, + AttributeType: aws.String(rangekey_type), + }) + } req.AttributeDefinitions = attributes req.GlobalSecondaryIndexUpdates = updates @@ -362,7 +388,8 @@ func resourceAwsDynamoDbTableUpdate(d *schema.ResourceData, meta interface{}) er for _, oldgsidata := range oldSet.List() { updates := []*dynamodb.GlobalSecondaryIndexUpdate{} - if !newSet.Contains(oldgsidata) { + oldGsiName := oldgsidata.(map[string]interface{})["name"].(string) + if _, exists := newGsiNameSet[oldGsiName]; !exists { gsidata := oldgsidata.(map[string]interface{}) log.Printf("[DEBUG] Deleting GSI %s", gsidata["name"].(string)) update := &dynamodb.GlobalSecondaryIndexUpdate{ @@ -382,31 +409,80 @@ func resourceAwsDynamoDbTableUpdate(d *schema.ResourceData, meta interface{}) er waitForTableToBeActive(d.Id(), meta) } } + } - for _, updatedgsidata := range changedSet.List() { - updates := []*dynamodb.GlobalSecondaryIndexUpdate{} - gsidata := updatedgsidata.(map[string]interface{}) - log.Printf("[DEBUG] Updating GSI %s", gsidata["name"].(string)) - update := &dynamodb.GlobalSecondaryIndexUpdate{ - Update: &dynamodb.UpdateGlobalSecondaryIndexAction{ - IndexName: aws.String(gsidata["name"].(string)), - ProvisionedThroughput: &dynamodb.ProvisionedThroughput{ - WriteCapacityUnits: aws.Long(int64(gsidata["write_capacity"].(int))), - ReadCapacityUnits: aws.Long(int64(gsidata["read_capacity"].(int))), - }, - }, - } - updates = append(updates, update) + // Update any out-of-date read / write capacity + if gsiObjects, ok := d.GetOk("global_secondary_index"); ok { + gsiSet := gsiObjects.(*schema.Set) + if len(gsiSet.List()) > 0 { + log.Printf("Updating capacity as needed!") - req.GlobalSecondaryIndexUpdates = updates - - _, err := dynamodbconn.UpdateTable(req) + // We can only change throughput, but we need to make sure it's actually changed + tableDescription, err := dynamodbconn.DescribeTable(&dynamodb.DescribeTableInput{ + TableName: aws.String(d.Id()), + }) if err != nil { - log.Printf("[DEBUG] Error updating table: %s", err) return err } + + table := tableDescription.Table + + updates := []*dynamodb.GlobalSecondaryIndexUpdate{} + + for _, updatedgsidata := range gsiSet.List() { + gsidata := updatedgsidata.(map[string]interface{}) + gsiName := gsidata["name"].(string) + gsiWriteCapacity := gsidata["write_capacity"].(int) + gsiReadCapacity := gsidata["read_capacity"].(int) + + log.Printf("[DEBUG] Updating GSI %s", gsiName) + gsi, err := getGlobalSecondaryIndex(gsiName, table.GlobalSecondaryIndexes) + + if err != nil { + return err + } + + capacityUpdated := false + + if int64(gsiReadCapacity) != *(gsi.ProvisionedThroughput.ReadCapacityUnits) || + int64(gsiWriteCapacity) != *(gsi.ProvisionedThroughput.WriteCapacityUnits) { + capacityUpdated = true + } + + if capacityUpdated { + update := &dynamodb.GlobalSecondaryIndexUpdate{ + Update: &dynamodb.UpdateGlobalSecondaryIndexAction{ + IndexName: aws.String(gsidata["name"].(string)), + ProvisionedThroughput: &dynamodb.ProvisionedThroughput{ + WriteCapacityUnits: aws.Long(int64(gsiWriteCapacity)), + ReadCapacityUnits: aws.Long(int64(gsiReadCapacity)), + }, + }, + } + updates = append(updates, update) + + } + + if len(updates) > 0 { + + req := &dynamodb.UpdateTableInput{ + TableName: aws.String(d.Id()), + } + + req.GlobalSecondaryIndexUpdates = updates + + log.Printf("[DEBUG] Updating GSI read / write capacity on %s", d.Id()) + _, err := dynamodbconn.UpdateTable(req) + + if err != nil { + log.Printf("[DEBUG] Error updating table: %s", err) + return err + } + } + } } + } return resourceAwsDynamoDbTableRead(d, meta) @@ -414,6 +490,7 @@ func resourceAwsDynamoDbTableUpdate(d *schema.ResourceData, meta interface{}) er func resourceAwsDynamoDbTableRead(d *schema.ResourceData, meta interface{}) error { dynamodbconn := meta.(*AWSClient).dynamodbconn + log.Printf("[DEBUG] Loading data for DynamoDB table '%s'", d.Id()) req := &dynamodb.DescribeTableInput{ TableName: aws.String(d.Id()), } @@ -431,21 +508,39 @@ func resourceAwsDynamoDbTableRead(d *schema.ResourceData, meta interface{}) erro attributes := []interface{}{} for _, attrdef := range table.AttributeDefinitions { - attribute := make(map[string]string) - attribute["name"] = *(attrdef.AttributeName) - attribute["type"] = *(attrdef.AttributeType) + attribute := map[string]string{ + "name": *(attrdef.AttributeName), + "type": *(attrdef.AttributeType), + } attributes = append(attributes, attribute) + log.Printf("[DEBUG] Added Attribute: %s", attribute["name"]) } d.Set("attribute", attributes) - gsiList := []interface{}{} + gsiList := make([]map[string]interface{}, 0, len(table.GlobalSecondaryIndexes)) for _, gsiObject := range table.GlobalSecondaryIndexes { - gsi := make(map[string]interface{}) - gsi["write_capacity"] = gsiObject.ProvisionedThroughput.WriteCapacityUnits - gsi["read_capacity"] = gsiObject.ProvisionedThroughput.ReadCapacityUnits - gsi["name"] = gsiObject.IndexName + gsi := map[string]interface{}{ + "write_capacity": *(gsiObject.ProvisionedThroughput.WriteCapacityUnits), + "read_capacity": *(gsiObject.ProvisionedThroughput.ReadCapacityUnits), + "name": *(gsiObject.IndexName), + } + + for _, attribute := range gsiObject.KeySchema { + if *attribute.KeyType == "HASH" { + gsi["hash_key"] = *attribute.AttributeName + } + + if *attribute.KeyType == "RANGE" { + gsi["range_key"] = *attribute.AttributeName + } + } + + gsi["projection_type"] = *(gsiObject.Projection.ProjectionType) + gsi["non_key_attributes"] = gsiObject.Projection.NonKeyAttributes + gsiList = append(gsiList, gsi) + log.Printf("[DEBUG] Added GSI: %s - Read: %d / Write: %d", gsi["name"], gsi["read_capacity"], gsi["write_capacity"]) } d.Set("global_secondary_index", gsiList) @@ -486,18 +581,26 @@ func createGSIFromData(data *map[string]interface{}) dynamodb.GlobalSecondaryInd writeCapacity := (*data)["write_capacity"].(int) readCapacity := (*data)["read_capacity"].(int) - return dynamodb.GlobalSecondaryIndex{ - IndexName: aws.String((*data)["name"].(string)), - KeySchema: []*dynamodb.KeySchemaElement{ - &dynamodb.KeySchemaElement{ - AttributeName: aws.String((*data)["hash_key"].(string)), - KeyType: aws.String("HASH"), - }, - &dynamodb.KeySchemaElement{ - AttributeName: aws.String((*data)["range_key"].(string)), - KeyType: aws.String("RANGE"), - }, + key_schema := []*dynamodb.KeySchemaElement{ + &dynamodb.KeySchemaElement{ + AttributeName: aws.String((*data)["hash_key"].(string)), + KeyType: aws.String("HASH"), }, + } + + range_key_name := (*data)["range_key"] + if range_key_name != "" { + range_key_element := &dynamodb.KeySchemaElement{ + AttributeName: aws.String(range_key_name.(string)), + KeyType: aws.String("RANGE"), + } + + key_schema = append(key_schema, range_key_element) + } + + return dynamodb.GlobalSecondaryIndex{ + IndexName: aws.String((*data)["name"].(string)), + KeySchema: key_schema, Projection: projection, ProvisionedThroughput: &dynamodb.ProvisionedThroughput{ WriteCapacityUnits: aws.Long(int64(writeCapacity)), @@ -506,6 +609,16 @@ func createGSIFromData(data *map[string]interface{}) dynamodb.GlobalSecondaryInd } } +func getGlobalSecondaryIndex(indexName string, indexList []*dynamodb.GlobalSecondaryIndexDescription) (*dynamodb.GlobalSecondaryIndexDescription, error) { + for _, gsi := range indexList { + if *(gsi.IndexName) == indexName { + return gsi, nil + } + } + + return &dynamodb.GlobalSecondaryIndexDescription{}, fmt.Errorf("Can't find a GSI by that name...") +} + func getAttributeType(d *schema.ResourceData, attributeName string) (string, error) { if attributedata, ok := d.GetOk("attribute"); ok { attributeSet := attributedata.(*schema.Set) @@ -581,8 +694,8 @@ func waitForTableToBeActive(tableName string, meta interface{}) error { // Wait for a few seconds if !activeState { - log.Printf("[DEBUG] Sleeping for 3 seconds for table to become active") - time.Sleep(3 * time.Second) + log.Printf("[DEBUG] Sleeping for 5 seconds for table to become active") + time.Sleep(5 * time.Second) } }