diff --git a/builtin/providers/aws/data_source_aws_s3_object.go b/builtin/providers/aws/data_source_aws_s3_object.go new file mode 100644 index 000000000..d5fbbd31c --- /dev/null +++ b/builtin/providers/aws/data_source_aws_s3_object.go @@ -0,0 +1,221 @@ +package aws + +import ( + "bytes" + "fmt" + "log" + "regexp" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/hashicorp/terraform/helper/schema" +) + +func dataSourceAwsS3BucketObject() *schema.Resource { + return &schema.Resource{ + Read: dataSourceAwsS3BucketObjectRead, + + Schema: map[string]*schema.Schema{ + "body": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "bucket": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "cache_control": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "content_disposition": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "content_encoding": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "content_language": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "content_length": &schema.Schema{ + Type: schema.TypeInt, + Computed: true, + }, + "content_type": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "etag": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "expiration": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "expires": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "key": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + "last_modified": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "metadata": &schema.Schema{ + Type: schema.TypeMap, + Computed: true, + }, + "range": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + }, + "server_side_encryption": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "sse_kms_key_id": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "storage_class": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + "version_id": &schema.Schema{ + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + "website_redirect_location": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, + }, + } +} + +func dataSourceAwsS3BucketObjectRead(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).s3conn + + bucket := d.Get("bucket").(string) + key := d.Get("key").(string) + + input := s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + } + if v, ok := d.GetOk("range"); ok { + input.Range = aws.String(v.(string)) + } + if v, ok := d.GetOk("version_id"); ok { + input.VersionId = aws.String(v.(string)) + } + + versionText := "" + uniqueId := bucket + "/" + key + if v, ok := d.GetOk("version_id"); ok { + versionText = fmt.Sprintf(" of version %q", v.(string)) + uniqueId += "@" + v.(string) + } + + log.Printf("[DEBUG] Reading S3 object: %s", input) + out, err := conn.HeadObject(&input) + if err != nil { + return fmt.Errorf("Failed getting S3 object: %s", err) + } + if out.DeleteMarker != nil && *out.DeleteMarker == true { + return fmt.Errorf("Requested S3 object %q%s has been deleted", + bucket+key, versionText) + } + + log.Printf("[DEBUG] Received S3 object: %s", out) + + d.SetId(uniqueId) + + d.Set("cache_control", out.CacheControl) + d.Set("content_disposition", out.ContentDisposition) + d.Set("content_encoding", out.ContentEncoding) + d.Set("content_language", out.ContentLanguage) + d.Set("content_length", out.ContentLength) + d.Set("content_type", out.ContentType) + // See https://forums.aws.amazon.com/thread.jspa?threadID=44003 + d.Set("etag", strings.Trim(*out.ETag, `"`)) + d.Set("expiration", out.Expiration) + d.Set("expires", out.Expires) + d.Set("last_modified", out.LastModified.Format(time.RFC1123)) + d.Set("metadata", pointersMapToStringList(out.Metadata)) + d.Set("server_side_encryption", out.ServerSideEncryption) + d.Set("sse_kms_key_id", out.SSEKMSKeyId) + d.Set("storage_class", out.StorageClass) + d.Set("version_id", out.VersionId) + d.Set("website_redirect_location", out.WebsiteRedirectLocation) + + if isContentTypeAllowed(out.ContentType) { + input := s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + } + if v, ok := d.GetOk("range"); ok { + input.Range = aws.String(v.(string)) + } + if out.VersionId != nil { + input.VersionId = out.VersionId + } + out, err := conn.GetObject(&input) + if err != nil { + return fmt.Errorf("Failed getting S3 object: %s", err) + } + + buf := new(bytes.Buffer) + bytesRead, err := buf.ReadFrom(out.Body) + if err != nil { + return fmt.Errorf("Failed reading content of S3 object (%s): %s", + uniqueId, err) + } + log.Printf("[INFO] Saving %d bytes from S3 object %s", bytesRead, uniqueId) + d.Set("body", buf.String()) + } else { + contentType := "" + if out.ContentType == nil { + contentType = "" + } else { + contentType = *out.ContentType + } + + log.Printf("[INFO] Ignoring body of S3 object %s with Content-Type %q", + uniqueId, contentType) + } + + return nil +} + +// This is to prevent potential issues w/ binary files +// and generally unprintable characters +// See https://github.com/hashicorp/terraform/pull/3858#issuecomment-156856738 +func isContentTypeAllowed(contentType *string) bool { + if contentType == nil { + return false + } + + allowedContentTypes := []*regexp.Regexp{ + regexp.MustCompile("^text/.+"), + regexp.MustCompile("^application/json$"), + } + + for _, r := range allowedContentTypes { + if r.MatchString(*contentType) { + return true + } + } + + return false +} diff --git a/builtin/providers/aws/data_source_aws_s3_object_test.go b/builtin/providers/aws/data_source_aws_s3_object_test.go new file mode 100644 index 000000000..f9210437d --- /dev/null +++ b/builtin/providers/aws/data_source_aws_s3_object_test.go @@ -0,0 +1,297 @@ +package aws + +import ( + "fmt" + "regexp" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/hashicorp/terraform/helper/acctest" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" +) + +func TestAccDataSourceAWSS3BucketObject_basic(t *testing.T) { + rInt := acctest.RandInt() + resourceOnlyConf, conf := testAccAWSDataSourceS3ObjectConfig_basic(rInt) + + var rObj s3.GetObjectOutput + var dsObj s3.GetObjectOutput + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + PreventPostDestroyRefresh: true, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: resourceOnlyConf, + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSS3BucketObjectExists("aws_s3_bucket_object.object", &rObj), + ), + }, + resource.TestStep{ + Config: conf, + Check: resource.ComposeTestCheckFunc( + testAccCheckAwsS3ObjectDataSourceExists("data.aws_s3_bucket_object.obj", &dsObj), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "content_length", "11"), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "content_type", "binary/octet-stream"), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "etag", "b10a8db164e0754105b7a99be72e3fe5"), + resource.TestMatchResourceAttr("data.aws_s3_bucket_object.obj", "last_modified", + regexp.MustCompile("^[a-zA-Z]{3}, [0-9]+ [a-zA-Z]+ [0-9]{4} [0-9:]+ [A-Z]+$")), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "body", ""), + ), + }, + }, + }) +} + +func TestAccDataSourceAWSS3BucketObject_readableBody(t *testing.T) { + rInt := acctest.RandInt() + resourceOnlyConf, conf := testAccAWSDataSourceS3ObjectConfig_readableBody(rInt) + + var rObj s3.GetObjectOutput + var dsObj s3.GetObjectOutput + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + PreventPostDestroyRefresh: true, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: resourceOnlyConf, + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSS3BucketObjectExists("aws_s3_bucket_object.object", &rObj), + ), + }, + resource.TestStep{ + Config: conf, + Check: resource.ComposeTestCheckFunc( + testAccCheckAwsS3ObjectDataSourceExists("data.aws_s3_bucket_object.obj", &dsObj), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "content_length", "3"), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "content_type", "text/plain"), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "etag", "a6105c0a611b41b08f1209506350279e"), + resource.TestMatchResourceAttr("data.aws_s3_bucket_object.obj", "last_modified", + regexp.MustCompile("^[a-zA-Z]{3}, [0-9]+ [a-zA-Z]+ [0-9]{4} [0-9:]+ [A-Z]+$")), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "body", "yes"), + ), + }, + }, + }) +} + +func TestAccDataSourceAWSS3BucketObject_kmsEncrypted(t *testing.T) { + rInt := acctest.RandInt() + resourceOnlyConf, conf := testAccAWSDataSourceS3ObjectConfig_kmsEncrypted(rInt) + + var rObj s3.GetObjectOutput + var dsObj s3.GetObjectOutput + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + PreventPostDestroyRefresh: true, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: resourceOnlyConf, + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSS3BucketObjectExists("aws_s3_bucket_object.object", &rObj), + ), + }, + resource.TestStep{ + Config: conf, + Check: resource.ComposeTestCheckFunc( + testAccCheckAwsS3ObjectDataSourceExists("data.aws_s3_bucket_object.obj", &dsObj), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "content_length", "22"), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "content_type", "text/plain"), + resource.TestMatchResourceAttr("data.aws_s3_bucket_object.obj", "etag", regexp.MustCompile("^[a-f0-9]{32}$")), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "server_side_encryption", "aws:kms"), + resource.TestMatchResourceAttr("data.aws_s3_bucket_object.obj", "sse_kms_key_id", + regexp.MustCompile("^arn:aws:kms:us-west-2:[0-9]{12}:key/[a-z0-9-]{36}$")), + resource.TestMatchResourceAttr("data.aws_s3_bucket_object.obj", "last_modified", + regexp.MustCompile("^[a-zA-Z]{3}, [0-9]+ [a-zA-Z]+ [0-9]{4} [0-9:]+ [A-Z]+$")), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "body", "Keep Calm and Carry On"), + ), + }, + }, + }) +} + +func TestAccDataSourceAWSS3BucketObject_allParams(t *testing.T) { + rInt := acctest.RandInt() + resourceOnlyConf, conf := testAccAWSDataSourceS3ObjectConfig_allParams(rInt) + + var rObj s3.GetObjectOutput + var dsObj s3.GetObjectOutput + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + PreventPostDestroyRefresh: true, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: resourceOnlyConf, + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSS3BucketObjectExists("aws_s3_bucket_object.object", &rObj), + ), + }, + resource.TestStep{ + Config: conf, + Check: resource.ComposeTestCheckFunc( + testAccCheckAwsS3ObjectDataSourceExists("data.aws_s3_bucket_object.obj", &dsObj), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "content_length", "21"), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "content_type", "application/unknown"), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "etag", "723f7a6ac0c57b445790914668f98640"), + resource.TestMatchResourceAttr("data.aws_s3_bucket_object.obj", "last_modified", + regexp.MustCompile("^[a-zA-Z]{3}, [0-9]+ [a-zA-Z]+ [0-9]{4} [0-9:]+ [A-Z]+$")), + resource.TestMatchResourceAttr("data.aws_s3_bucket_object.obj", "version_id", regexp.MustCompile("^.{32}$")), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "body", ""), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "cache_control", "no-cache"), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "content_disposition", "attachment"), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "content_encoding", "gzip"), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "content_language", "en-GB"), + // Encryption is off + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "server_side_encryption", ""), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "sse_kms_key_id", ""), + // Supported, but difficult to reproduce in short testing time + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "storage_class", ""), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "expiration", ""), + // Currently unsupported in aws_s3_bucket_object resource + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "expires", ""), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "website_redirect_location", ""), + resource.TestCheckResourceAttr("data.aws_s3_bucket_object.obj", "metadata.#", "0"), + ), + }, + }, + }) +} + +func testAccCheckAwsS3ObjectDataSourceExists(n string, obj *s3.GetObjectOutput) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Can't find S3 object data source: %s", n) + } + + if rs.Primary.ID == "" { + return fmt.Errorf("S3 object data source ID not set") + } + + s3conn := testAccProvider.Meta().(*AWSClient).s3conn + out, err := s3conn.GetObject( + &s3.GetObjectInput{ + Bucket: aws.String(rs.Primary.Attributes["bucket"]), + Key: aws.String(rs.Primary.Attributes["key"]), + }) + if err != nil { + return fmt.Errorf("Failed getting S3 Object from %s: %s", + rs.Primary.Attributes["bucket"]+"/"+rs.Primary.Attributes["key"], err) + } + + *obj = *out + + return nil + } +} + +func testAccAWSDataSourceS3ObjectConfig_basic(randInt int) (string, string) { + resources := fmt.Sprintf(` +resource "aws_s3_bucket" "object_bucket" { + bucket = "tf-object-test-bucket-%d" +} +resource "aws_s3_bucket_object" "object" { + bucket = "${aws_s3_bucket.object_bucket.bucket}" + key = "tf-testing-obj-%d" + content = "Hello World" +} +`, randInt, randInt) + + both := fmt.Sprintf(`%s +data "aws_s3_bucket_object" "obj" { + bucket = "tf-object-test-bucket-%d" + key = "tf-testing-obj-%d" +}`, resources, randInt, randInt) + + return resources, both +} + +func testAccAWSDataSourceS3ObjectConfig_readableBody(randInt int) (string, string) { + resources := fmt.Sprintf(` +resource "aws_s3_bucket" "object_bucket" { + bucket = "tf-object-test-bucket-%d" +} +resource "aws_s3_bucket_object" "object" { + bucket = "${aws_s3_bucket.object_bucket.bucket}" + key = "tf-testing-obj-%d-readable" + content = "yes" + content_type = "text/plain" +} +`, randInt, randInt) + + both := fmt.Sprintf(`%s +data "aws_s3_bucket_object" "obj" { + bucket = "tf-object-test-bucket-%d" + key = "tf-testing-obj-%d-readable" +}`, resources, randInt, randInt) + + return resources, both +} + +func testAccAWSDataSourceS3ObjectConfig_kmsEncrypted(randInt int) (string, string) { + resources := fmt.Sprintf(` +resource "aws_s3_bucket" "object_bucket" { + bucket = "tf-object-test-bucket-%d" +} +resource "aws_kms_key" "example" { + description = "TF Acceptance Test KMS key" + deletion_window_in_days = 7 +} +resource "aws_s3_bucket_object" "object" { + bucket = "${aws_s3_bucket.object_bucket.bucket}" + key = "tf-testing-obj-%d-encrypted" + content = "Keep Calm and Carry On" + content_type = "text/plain" + kms_key_id = "${aws_kms_key.example.arn}" +} +`, randInt, randInt) + + both := fmt.Sprintf(`%s +data "aws_s3_bucket_object" "obj" { + bucket = "tf-object-test-bucket-%d" + key = "tf-testing-obj-%d-encrypted" +}`, resources, randInt, randInt) + + return resources, both +} + +func testAccAWSDataSourceS3ObjectConfig_allParams(randInt int) (string, string) { + resources := fmt.Sprintf(` +resource "aws_s3_bucket" "object_bucket" { + bucket = "tf-object-test-bucket-%d" + versioning { + enabled = true + } +} + +resource "aws_s3_bucket_object" "object" { + bucket = "${aws_s3_bucket.object_bucket.bucket}" + key = "tf-testing-obj-%d-all-params" + content = < **Note:** The content of an object (`body` field) is available only for objects which have a human-readable `Content-Type` (`text/*` and `application/json`). This is to prevent printing unsafe characters and potentially downloading large amount of data which would be thrown away in favour of metadata. + +## Example Usage + +``` +data "aws_s3_object" "lambda" { + bucket = "my-lambda-functions" + key = "hello-world.zip" +} + +resource "aws_iam_role" "iam_for_lambda" { + name = "iam_for_lambda" + assume_role_policy = <> aws_availability_zones - > aws_iam_policy_document + > + aws_s3_object +