From 8e2ff8e6cabe19192ec90d9c511a06a251b245bf Mon Sep 17 00:00:00 2001 From: Marcin Suterski Date: Mon, 10 Apr 2017 07:04:14 -0400 Subject: [PATCH] Add support for Sumologic logging to Fastly provider (#12541) --- .../fastly/resource_fastly_service_v1.go | 145 +++++++++++++++++- ...source_fastly_service_v1_sumologic_test.go | 126 +++++++++++++++ builtin/providers/fastly/validators.go | 18 ++- builtin/providers/fastly/validators_test.go | 32 +++- .../github.com/sethvargo/go-fastly/request.go | 4 +- .../sethvargo/go-fastly/sumologic.go | 6 + .../github.com/sethvargo/go-fastly/syslog.go | 3 + vendor/vendor.json | 6 +- .../fastly/r/service_v1.html.markdown | 12 ++ 9 files changed, 341 insertions(+), 11 deletions(-) create mode 100644 builtin/providers/fastly/resource_fastly_service_v1_sumologic_test.go diff --git a/builtin/providers/fastly/resource_fastly_service_v1.go b/builtin/providers/fastly/resource_fastly_service_v1.go index 852af30ce..e3b4a7c7b 100644 --- a/builtin/providers/fastly/resource_fastly_service_v1.go +++ b/builtin/providers/fastly/resource_fastly_service_v1.go @@ -539,7 +539,7 @@ func resourceServiceV1() *schema.Resource { Optional: true, Default: 1, Description: "The version of the custom logging format used for the configured endpoint. Can be either 1 or 2. (Default: 1)", - ValidateFunc: validateS3FormatVersion, + ValidateFunc: validateLoggingFormatVersion, }, "timestamp_format": { Type: schema.TypeString, @@ -594,6 +594,52 @@ func resourceServiceV1() *schema.Resource { }, }, }, + "sumologic": { + Type: schema.TypeSet, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + // Required fields + "name": { + Type: schema.TypeString, + Required: true, + Description: "Unique name to refer to this logging setup", + }, + "url": { + Type: schema.TypeString, + Required: true, + Description: "The URL to POST to.", + }, + // Optional fields + "format": { + Type: schema.TypeString, + Optional: true, + Default: "%h %l %u %t %r %>s", + Description: "Apache-style string or VCL variables to use for log formatting", + }, + "format_version": { + Type: schema.TypeInt, + Optional: true, + Default: 1, + Description: "The version of the custom logging format used for the configured endpoint. Can be either 1 or 2. (Default: 1)", + ValidateFunc: validateLoggingFormatVersion, + }, + "response_condition": { + Type: schema.TypeString, + Optional: true, + Default: "", + Description: "Name of a condition to apply this logging.", + }, + "message_type": { + Type: schema.TypeString, + Optional: true, + Default: "classic", + Description: "How the message should be formatted.", + ValidateFunc: validateLoggingMessageType, + }, + }, + }, + }, "response_object": { Type: schema.TypeSet, @@ -1344,6 +1390,59 @@ func resourceServiceV1Update(d *schema.ResourceData, meta interface{}) error { } } + // find difference in Sumologic + if d.HasChange("sumologic") { + os, ns := d.GetChange("sumologic") + if os == nil { + os = new(schema.Set) + } + if ns == nil { + ns = new(schema.Set) + } + + oss := os.(*schema.Set) + nss := ns.(*schema.Set) + removeSumologic := oss.Difference(nss).List() + addSumologic := nss.Difference(oss).List() + + // DELETE old sumologic configurations + for _, pRaw := range removeSumologic { + sf := pRaw.(map[string]interface{}) + opts := gofastly.DeleteSumologicInput{ + Service: d.Id(), + Version: latestVersion, + Name: sf["name"].(string), + } + + log.Printf("[DEBUG] Fastly Sumologic removal opts: %#v", opts) + err := conn.DeleteSumologic(&opts) + if err != nil { + return err + } + } + + // POST new/updated Sumologic + for _, pRaw := range addSumologic { + sf := pRaw.(map[string]interface{}) + opts := gofastly.CreateSumologicInput{ + Service: d.Id(), + Version: latestVersion, + Name: sf["name"].(string), + URL: sf["url"].(string), + Format: sf["format"].(string), + FormatVersion: sf["format_version"].(int), + ResponseCondition: sf["response_condition"].(string), + MessageType: sf["message_type"].(string), + } + + log.Printf("[DEBUG] Create Sumologic Opts: %#v", opts) + _, err := conn.CreateSumologic(&opts) + if err != nil { + return err + } + } + } + // find difference in Response Object if d.HasChange("response_object") { or, nr := d.GetChange("response_object") @@ -1761,6 +1860,22 @@ func resourceServiceV1Read(d *schema.ResourceData, meta interface{}) error { log.Printf("[WARN] Error setting Papertrail for (%s): %s", d.Id(), err) } + // refresh Sumologic Logging + log.Printf("[DEBUG] Refreshing Sumologic for (%s)", d.Id()) + sumologicList, err := conn.ListSumologics(&gofastly.ListSumologicsInput{ + Service: d.Id(), + Version: s.ActiveVersion.Number, + }) + + if err != nil { + return fmt.Errorf("[ERR] Error looking up Sumologic for (%s), version (%s): %s", d.Id(), s.ActiveVersion.Number, err) + } + + sul := flattenSumologics(sumologicList) + if err := d.Set("sumologic", sul); err != nil { + log.Printf("[WARN] Error setting Sumologic for (%s): %s", d.Id(), err) + } + // refresh Response Objects log.Printf("[DEBUG] Refreshing Response Object for (%s)", d.Id()) responseObjectList, err := conn.ListResponseObjects(&gofastly.ListResponseObjectsInput{ @@ -2179,7 +2294,7 @@ func flattenS3s(s3List []*gofastly.S3) []map[string]interface{} { func flattenPapertrails(papertrailList []*gofastly.Papertrail) []map[string]interface{} { var pl []map[string]interface{} for _, p := range papertrailList { - // Convert S3s to a map for saving to state. + // Convert Papertrails to a map for saving to state. ns := map[string]interface{}{ "name": p.Name, "address": p.Address, @@ -2201,6 +2316,32 @@ func flattenPapertrails(papertrailList []*gofastly.Papertrail) []map[string]inte return pl } +func flattenSumologics(sumologicList []*gofastly.Sumologic) []map[string]interface{} { + var l []map[string]interface{} + for _, p := range sumologicList { + // Convert Sumologic to a map for saving to state. + ns := map[string]interface{}{ + "name": p.Name, + "url": p.URL, + "format": p.Format, + "response_condition": p.ResponseCondition, + "message_type": p.MessageType, + "format_version": int(p.FormatVersion), + } + + // prune any empty values that come from the default string value in structs + for k, v := range ns { + if v == "" { + delete(ns, k) + } + } + + l = append(l, ns) + } + + return l +} + func flattenResponseObjects(responseObjectList []*gofastly.ResponseObject) []map[string]interface{} { var rol []map[string]interface{} for _, ro := range responseObjectList { diff --git a/builtin/providers/fastly/resource_fastly_service_v1_sumologic_test.go b/builtin/providers/fastly/resource_fastly_service_v1_sumologic_test.go new file mode 100644 index 000000000..f3c4b0f69 --- /dev/null +++ b/builtin/providers/fastly/resource_fastly_service_v1_sumologic_test.go @@ -0,0 +1,126 @@ +package fastly + +import ( + "fmt" + "reflect" + "testing" + + "github.com/hashicorp/terraform/helper/acctest" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" + gofastly "github.com/sethvargo/go-fastly" +) + +func TestResourceFastlyFlattenSumologic(t *testing.T) { + cases := []struct { + remote []*gofastly.Sumologic + local []map[string]interface{} + }{ + { + remote: []*gofastly.Sumologic{ + &gofastly.Sumologic{ + Name: "sumo collector", + URL: "https://sumologic.com/collector/1", + Format: "log format", + FormatVersion: 2, + MessageType: "classic", + ResponseCondition: "condition 1", + }, + }, + local: []map[string]interface{}{ + map[string]interface{}{ + "name": "sumo collector", + "url": "https://sumologic.com/collector/1", + "format": "log format", + "format_version": 2, + "message_type": "classic", + "response_condition": "condition 1", + }, + }, + }, + } + + for _, c := range cases { + out := flattenSumologics(c.remote) + if !reflect.DeepEqual(out, c.local) { + t.Fatalf("Error matching:\nexpected: %#v\ngot: %#v", c.local, out) + } + } +} + +func TestAccFastlyServiceV1_sumologic(t *testing.T) { + var service gofastly.ServiceDetail + name := fmt.Sprintf("tf-test-%s", acctest.RandString(10)) + sumologicName := fmt.Sprintf("sumologic %s", acctest.RandString(3)) + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckServiceV1Destroy, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: testAccServiceV1Config_sumologic(name, sumologicName), + Check: resource.ComposeTestCheckFunc( + testAccCheckServiceV1Exists("fastly_service_v1.foo", &service), + testAccCheckFastlyServiceV1Attributes_sumologic(&service, name, sumologicName), + ), + }, + }, + }) +} + +func testAccCheckFastlyServiceV1Attributes_sumologic(service *gofastly.ServiceDetail, name, sumologic string) resource.TestCheckFunc { + return func(s *terraform.State) error { + + if service.Name != name { + return fmt.Errorf("Bad name, expected (%s), got (%s)", name, service.Name) + } + + conn := testAccProvider.Meta().(*FastlyClient).conn + sumologicList, err := conn.ListSumologics(&gofastly.ListSumologicsInput{ + Service: service.ID, + Version: service.ActiveVersion.Number, + }) + + if err != nil { + return fmt.Errorf("[ERR] Error looking up Sumologics for (%s), version (%s): %s", service.Name, service.ActiveVersion.Number, err) + } + + if len(sumologicList) != 1 { + return fmt.Errorf("Sumologic missing, expected: 1, got: %d", len(sumologicList)) + } + + if sumologicList[0].Name != sumologic { + return fmt.Errorf("Sumologic name mismatch, expected: %s, got: %#v", sumologic, sumologicList[0].Name) + } + + return nil + } +} + +func testAccServiceV1Config_sumologic(name, sumologic string) string { + backendName := fmt.Sprintf("%s.aws.amazon.com", acctest.RandString(3)) + + return fmt.Sprintf(` +resource "fastly_service_v1" "foo" { + name = "%s" + + domain { + name = "test.notadomain.com" + comment = "tf-testing-domain" + } + + backend { + address = "%s" + name = "tf -test backend" + } + + sumologic { + name = "%s" + url = "https://sumologic.com/collector/1" + format_version = 2 + } + + force_destroy = true +}`, name, backendName, sumologic) +} diff --git a/builtin/providers/fastly/validators.go b/builtin/providers/fastly/validators.go index 1f2fb22d4..1903d8daf 100644 --- a/builtin/providers/fastly/validators.go +++ b/builtin/providers/fastly/validators.go @@ -2,7 +2,7 @@ package fastly import "fmt" -func validateS3FormatVersion(v interface{}, k string) (ws []string, errors []error) { +func validateLoggingFormatVersion(v interface{}, k string) (ws []string, errors []error) { value := uint(v.(int)) validVersions := map[uint]struct{}{ 1: {}, @@ -15,3 +15,19 @@ func validateS3FormatVersion(v interface{}, k string) (ws []string, errors []err } return } + +func validateLoggingMessageType(v interface{}, k string) (ws []string, errors []error) { + value := v.(string) + validTypes := map[string]struct{}{ + "classic": {}, + "loggly": {}, + "logplex": {}, + "blank": {}, + } + + if _, ok := validTypes[value]; !ok { + errors = append(errors, fmt.Errorf( + "%q must be one of ['classic', 'loggly', 'logplex', 'blank']", k)) + } + return +} diff --git a/builtin/providers/fastly/validators_test.go b/builtin/providers/fastly/validators_test.go index d563c7f5d..9ec1a4e8e 100644 --- a/builtin/providers/fastly/validators_test.go +++ b/builtin/providers/fastly/validators_test.go @@ -2,13 +2,13 @@ package fastly import "testing" -func TestValidateS3FormatVersion(t *testing.T) { +func TestValidateLoggingFormatVersion(t *testing.T) { validVersions := []int{ 1, 2, } for _, v := range validVersions { - _, errors := validateS3FormatVersion(v, "format_version") + _, errors := validateLoggingFormatVersion(v, "format_version") if len(errors) != 0 { t.Fatalf("%q should be a valid format version: %q", v, errors) } @@ -21,9 +21,35 @@ func TestValidateS3FormatVersion(t *testing.T) { 5, } for _, v := range invalidVersions { - _, errors := validateS3FormatVersion(v, "format_version") + _, errors := validateLoggingFormatVersion(v, "format_version") if len(errors) != 1 { t.Fatalf("%q should not be a valid format version", v) } } } + +func TestValidateLoggingMessageType(t *testing.T) { + validTypes := []string{ + "classic", + "loggly", + "logplex", + "blank", + } + for _, v := range validTypes { + _, errors := validateLoggingMessageType(v, "message_type") + if len(errors) != 0 { + t.Fatalf("%q should be a valid message type: %q", v, errors) + } + } + + invalidTypes := []string{ + "invalid_type_1", + "invalid_type_2", + } + for _, v := range invalidTypes { + _, errors := validateLoggingMessageType(v, "message_type") + if len(errors) != 1 { + t.Fatalf("%q should not be a valid message type", v) + } + } +} diff --git a/vendor/github.com/sethvargo/go-fastly/request.go b/vendor/github.com/sethvargo/go-fastly/request.go index 3e1de3a3b..5f2af4959 100644 --- a/vendor/github.com/sethvargo/go-fastly/request.go +++ b/vendor/github.com/sethvargo/go-fastly/request.go @@ -4,7 +4,7 @@ import ( "io" "net/http" "net/url" - "path" + "strings" ) // RequestOptions is the list of options to pass to the request. @@ -31,7 +31,7 @@ func (c *Client) RawRequest(verb, p string, ro *RequestOptions) (*http.Request, // Append the path to the URL. u := *c.url - u.Path = path.Join(c.url.Path, p) + u.Path = strings.TrimRight(c.url.Path, "/") + "/" + strings.TrimLeft(p, "/") // Add the token and other params. var params = make(url.Values) diff --git a/vendor/github.com/sethvargo/go-fastly/sumologic.go b/vendor/github.com/sethvargo/go-fastly/sumologic.go index 371e7a6e8..2e6b3fba7 100644 --- a/vendor/github.com/sethvargo/go-fastly/sumologic.go +++ b/vendor/github.com/sethvargo/go-fastly/sumologic.go @@ -16,6 +16,8 @@ type Sumologic struct { URL string `mapstructure:"url"` Format string `mapstructure:"format"` ResponseCondition string `mapstructure:"response_condition"` + MessageType string `mapstructure:"message_type"` + FormatVersion int `mapstructure:"format_version"` CreatedAt *time.Time `mapstructure:"created_at"` UpdatedAt *time.Time `mapstructure:"updated_at"` DeletedAt *time.Time `mapstructure:"deleted_at"` @@ -76,6 +78,8 @@ type CreateSumologicInput struct { URL string `form:"url,omitempty"` Format string `form:"format,omitempty"` ResponseCondition string `form:"response_condition,omitempty"` + MessageType string `form:"message_type,omitempty"` + FormatVersion int `form:"format_version,omitempty"` } // CreateSumologic creates a new Fastly sumologic. @@ -154,6 +158,8 @@ type UpdateSumologicInput struct { URL string `form:"url,omitempty"` Format string `form:"format,omitempty"` ResponseCondition string `form:"response_condition,omitempty"` + MessageType string `form:"message_type,omitempty"` + FormatVersion int `form:"format_version,omitempty"` } // UpdateSumologic updates a specific sumologic. diff --git a/vendor/github.com/sethvargo/go-fastly/syslog.go b/vendor/github.com/sethvargo/go-fastly/syslog.go index 5d2a08303..56d61ce31 100644 --- a/vendor/github.com/sethvargo/go-fastly/syslog.go +++ b/vendor/github.com/sethvargo/go-fastly/syslog.go @@ -18,6 +18,7 @@ type Syslog struct { TLSCACert string `mapstructure:"tls_ca_cert"` Token string `mapstructure:"token"` Format string `mapstructure:"format"` + FormatVersion uint `mapstructure:"format_version"` ResponseCondition string `mapstructure:"response_condition"` CreatedAt *time.Time `mapstructure:"created_at"` UpdatedAt *time.Time `mapstructure:"updated_at"` @@ -81,6 +82,7 @@ type CreateSyslogInput struct { TLSCACert string `form:"tls_ca_cert,omitempty"` Token string `form:"token,omitempty"` Format string `form:"format,omitempty"` + FormatVersion uint `form:"format_version,omitempty"` ResponseCondition string `form:"response_condition,omitempty"` } @@ -162,6 +164,7 @@ type UpdateSyslogInput struct { TLSCACert string `form:"tls_ca_cert,omitempty"` Token string `form:"token,omitempty"` Format string `form:"format,omitempty"` + FormatVersion uint `form:"format_version,omitempty"` ResponseCondition string `form:"response_condition,omitempty"` } diff --git a/vendor/vendor.json b/vendor/vendor.json index aee3d291f..641847d6b 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -2858,10 +2858,10 @@ "revisionTime": "2017-03-13T16:33:22Z" }, { - "checksumSHA1": "ySSmShoczI/i/5PzurH8Uhi/dbA=", + "checksumSHA1": "bCpL8ZdY+y7OGwiN3hZzbQI5oM0=", "path": "github.com/sethvargo/go-fastly", - "revision": "247f42f7ecc6677aa1b6e30978d06fcc38f5f769", - "revisionTime": "2017-02-06T18:56:52Z" + "revision": "43b7f97296d6c8e3a7bc083ab91101fbbc8c2f94", + "revisionTime": "2017-02-28T16:12:19Z" }, { "checksumSHA1": "8tEiK6vhVXuUbnWME5XNWLgvtSo=", diff --git a/website/source/docs/providers/fastly/r/service_v1.html.markdown b/website/source/docs/providers/fastly/r/service_v1.html.markdown index fa995c367..3429c86aa 100644 --- a/website/source/docs/providers/fastly/r/service_v1.html.markdown +++ b/website/source/docs/providers/fastly/r/service_v1.html.markdown @@ -153,6 +153,8 @@ order to destroy the Service, set `force_destroy` to `true`. Default `false`. Defined below. * `papertrail` - (Optional) A Papertrail endpoint to send streaming logs too. Defined below. +* `sumologic` - (Optional) A Sumologic endpoint to send streaming logs too. +Defined below. * `response_object` - (Optional) Allows you to create synthetic responses that exist entirely on the varnish machine. Useful for creating error or maintenance pages that exists outside the scope of your datacenter. Best when used with Condition objects. * `vcl` - (Optional) A set of custom VCL configuration blocks. The ability to upload custom VCL code is not enabled by default for new Fastly @@ -315,6 +317,15 @@ The `papertrail` block supports: * `response_condition` - (Optional) Name of already defined `condition` to apply. This `condition` must be of type `RESPONSE`. For detailed information about Conditionals, see [Fastly's Documentation on Conditionals][fastly-conditionals]. +The `sumologic` block supports: + +* `name` - (Required) A unique name to identify this Sumologic endpoint. +* `url` - (Required) The URL to Sumologic collector endpoint +* `format` - (Optional) Apache-style string or VCL variables to use for log formatting. Defaults to Apache Common Log format (`%h %l %u %t %r %>s`) +* `format_version` - (Optional) The version of the custom logging format used for the configured endpoint. Can be either 1 (the default, version 1 log format) or 2 (the version 2 log format). +* `response_condition` - (Optional) Name of already defined `condition` to apply. This `condition` must be of type `RESPONSE`. For detailed information about Conditionals, see [Fastly's Documentation on Conditionals][fastly-conditionals]. +* `message_type` - (Optional) How the message should be formatted. One of: classic, loggly, logplex, blank. See [Fastly's Documentation on Sumologic][fastly-sumologic] + The `response_object` block supports: * `name` - (Required) A unique name to identify this Response Object. @@ -357,3 +368,4 @@ Service. [fastly-s3]: https://docs.fastly.com/guides/integrations/amazon-s3 [fastly-cname]: https://docs.fastly.com/guides/basic-setup/adding-cname-records [fastly-conditionals]: https://docs.fastly.com/guides/conditions/using-conditions +[fastly-sumologic]: https://docs.fastly.com/api/logging#logging_sumologic