From 1425205348c8899f85179255936fb16883838d68 Mon Sep 17 00:00:00 2001 From: tombuildsstuff Date: Mon, 4 Sep 2017 11:18:38 +0100 Subject: [PATCH] Upgrading to the latest Azure SDK's --- .../arm/resources/resources/version.go | 9 +- .../azure-sdk-for-go/arm/storage/version.go | 9 +- .../azure-sdk-for-go/storage/appendblob.go | 2 + .../Azure/azure-sdk-for-go/storage/blob.go | 34 ++++-- .../azure-sdk-for-go/storage/blockblob.go | 35 +++++- .../Azure/azure-sdk-for-go/storage/client.go | 95 ++++++++------- .../azure-sdk-for-go/storage/container.go | 3 + .../azure-sdk-for-go/storage/copyblob.go | 4 +- .../azure-sdk-for-go/storage/directory.go | 2 + .../Azure/azure-sdk-for-go/storage/file.go | 20 ++-- .../azure-sdk-for-go/storage/pageblob.go | 2 + .../Azure/azure-sdk-for-go/storage/util.go | 2 +- .../azure-sdk-for-go/storage/util_1.7.go | 12 ++ .../azure-sdk-for-go/storage/util_1.8.go | 18 +++ .../Azure/go-autorest/autorest/adal/token.go | 6 + .../go-autorest/autorest/authorization.go | 110 ++++++++++++++++++ .../Azure/go-autorest/autorest/client.go | 1 + .../Azure/go-autorest/autorest/error.go | 4 + .../go-autorest/autorest/retriablerequest.go | 38 ++++++ .../autorest/retriablerequest_1.7.go | 44 +++++++ .../autorest/retriablerequest_1.8.go | 56 +++++++++ .../Azure/go-autorest/autorest/sender.go | 53 ++++++--- .../autorest/validation/validation.go | 15 ++- vendor/vendor.json | 87 +++++++------- 24 files changed, 522 insertions(+), 139 deletions(-) create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/storage/util_1.7.go create mode 100644 vendor/github.com/Azure/azure-sdk-for-go/storage/util_1.8.go create mode 100644 vendor/github.com/Azure/go-autorest/autorest/retriablerequest.go create mode 100644 vendor/github.com/Azure/go-autorest/autorest/retriablerequest_1.7.go create mode 100644 vendor/github.com/Azure/go-autorest/autorest/retriablerequest_1.8.go diff --git a/vendor/github.com/Azure/azure-sdk-for-go/arm/resources/resources/version.go b/vendor/github.com/Azure/azure-sdk-for-go/arm/resources/resources/version.go index 998788856..ab4005676 100755 --- a/vendor/github.com/Azure/azure-sdk-for-go/arm/resources/resources/version.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/arm/resources/resources/version.go @@ -14,16 +14,15 @@ package resources // See the License for the specific language governing permissions and // limitations under the License. // -// Code generated by Microsoft (R) AutoRest Code Generator 1.0.1.0 -// Changes may cause incorrect behavior and will be lost if the code is -// regenerated. +// Code generated by Microsoft (R) AutoRest Code Generator 1.2.2.0 +// Changes may cause incorrect behavior and will be lost if the code is regenerated. // UserAgent returns the UserAgent string to use when sending http.Requests. func UserAgent() string { - return "Azure-SDK-For-Go/v10.0.2-beta arm-resources/2016-09-01" + return "Azure-SDK-For-Go/v10.3.0-beta arm-resources/2017-05-10" } // Version returns the semantic version (see http://semver.org) of the client. func Version() string { - return "v10.0.2-beta" + return "v10.3.0-beta" } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/arm/storage/version.go b/vendor/github.com/Azure/azure-sdk-for-go/arm/storage/version.go index ac97c159f..8d944c739 100755 --- a/vendor/github.com/Azure/azure-sdk-for-go/arm/storage/version.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/arm/storage/version.go @@ -14,16 +14,15 @@ package storage // See the License for the specific language governing permissions and // limitations under the License. // -// Code generated by Microsoft (R) AutoRest Code Generator 1.0.1.0 -// Changes may cause incorrect behavior and will be lost if the code is -// regenerated. +// Code generated by Microsoft (R) AutoRest Code Generator 1.2.2.0 +// Changes may cause incorrect behavior and will be lost if the code is regenerated. // UserAgent returns the UserAgent string to use when sending http.Requests. func UserAgent() string { - return "Azure-SDK-For-Go/v10.0.2-beta arm-storage/2016-12-01" + return "Azure-SDK-For-Go/v10.3.0-beta arm-storage/2017-06-01" } // Version returns the semantic version (see http://semver.org) of the client. func Version() string { - return "v10.0.2-beta" + return "v10.3.0-beta" } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/appendblob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/appendblob.go index 3292cb556..c13d409b7 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/appendblob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/appendblob.go @@ -11,6 +11,8 @@ import ( // PutAppendBlob initializes an empty append blob with specified name. An // append blob must be created using this method before appending blocks. // +// See CreateBlockBlobFromReader for more info on creating blobs. +// // See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob func (b *Blob) PutAppendBlob(options *PutBlobOptions) error { params := url.Values{} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go index dd9eb386c..12f61ac2a 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/blob.go @@ -174,11 +174,17 @@ type BlobRange struct { } func (br BlobRange) String() string { + if br.End == 0 { + return fmt.Sprintf("bytes=%d-", br.Start) + } return fmt.Sprintf("bytes=%d-%d", br.Start, br.End) } // Get returns a stream to read the blob. Caller must call both Read and Close() // to correctly close the underlying connection. +// +// See the GetRange method for use with a Range header. +// // See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Get-Blob func (b *Blob) Get(options *GetBlobOptions) (io.ReadCloser, error) { rangeOptions := GetBlobRangeOptions{ @@ -192,7 +198,7 @@ func (b *Blob) Get(options *GetBlobOptions) (io.ReadCloser, error) { if err := checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { return nil, err } - if err := b.writePropoerties(resp.headers); err != nil { + if err := b.writeProperties(resp.headers, true); err != nil { return resp.body, err } return resp.body, nil @@ -212,7 +218,9 @@ func (b *Blob) GetRange(options *GetBlobRangeOptions) (io.ReadCloser, error) { if err := checkRespCode(resp.statusCode, []int{http.StatusPartialContent}); err != nil { return nil, err } - if err := b.writePropoerties(resp.headers); err != nil { + // Content-Length header should not be updated, as the service returns the range length + // (which is not alwys the full blob length) + if err := b.writeProperties(resp.headers, false); err != nil { return resp.body, err } return resp.body, nil @@ -225,7 +233,9 @@ func (b *Blob) getRange(options *GetBlobRangeOptions) (*storageResponse, error) if options != nil { if options.Range != nil { headers["Range"] = options.Range.String() - headers["x-ms-range-get-content-md5"] = fmt.Sprintf("%v", options.GetRangeContentMD5) + if options.GetRangeContentMD5 { + headers["x-ms-range-get-content-md5"] = "true" + } } if options.GetBlobOptions != nil { headers = mergeHeaders(headers, headersFromStruct(*options.GetBlobOptions)) @@ -322,18 +332,20 @@ func (b *Blob) GetProperties(options *GetBlobPropertiesOptions) error { if err = checkRespCode(resp.statusCode, []int{http.StatusOK}); err != nil { return err } - return b.writePropoerties(resp.headers) + return b.writeProperties(resp.headers, true) } -func (b *Blob) writePropoerties(h http.Header) error { +func (b *Blob) writeProperties(h http.Header, includeContentLen bool) error { var err error - var contentLength int64 - contentLengthStr := h.Get("Content-Length") - if contentLengthStr != "" { - contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64) - if err != nil { - return err + contentLength := b.Properties.ContentLength + if includeContentLen { + contentLengthStr := h.Get("Content-Length") + if contentLengthStr != "" { + contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64) + if err != nil { + return err + } } } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go index 09b13cc18..5258f24fd 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "net/url" + "strconv" "strings" "time" ) @@ -67,6 +68,8 @@ type BlockResponse struct { // CreateBlockBlob initializes an empty block blob with no blocks. // +// See CreateBlockBlobFromReader for more info on creating blobs. +// // See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob func (b *Blob) CreateBlockBlob(options *PutBlobOptions) error { return b.CreateBlockBlobFromReader(nil, options) @@ -76,16 +79,46 @@ func (b *Blob) CreateBlockBlob(options *PutBlobOptions) error { // reader. Size must be the number of bytes read from reader. To // create an empty blob, use size==0 and reader==nil. // +// Any headers set in blob.Properties or metadata in blob.Metadata +// will be set on the blob. +// // The API rejects requests with size > 256 MiB (but this limit is not // checked by the SDK). To write a larger blob, use CreateBlockBlob, // PutBlock, and PutBlockList. // +// To create a blob from scratch, call container.GetBlobReference() to +// get an empty blob, fill in blob.Properties and blob.Metadata as +// appropriate then call this method. +// // See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob func (b *Blob) CreateBlockBlobFromReader(blob io.Reader, options *PutBlobOptions) error { params := url.Values{} headers := b.Container.bsc.client.getStandardHeaders() headers["x-ms-blob-type"] = string(BlobTypeBlock) - headers["Content-Length"] = fmt.Sprintf("%d", b.Properties.ContentLength) + + headers["Content-Length"] = "0" + var n int64 + var err error + if blob != nil { + type lener interface { + Len() int + } + // TODO(rjeczalik): handle io.ReadSeeker, in case blob is *os.File etc. + if l, ok := blob.(lener); ok { + n = int64(l.Len()) + } else { + var buf bytes.Buffer + n, err = io.Copy(&buf, blob) + if err != nil { + return err + } + blob = &buf + } + + headers["Content-Length"] = strconv.FormatInt(n, 10) + } + b.Properties.ContentLength = n + headers = mergeHeaders(headers, headersFromStruct(b.Properties)) headers = b.Container.bsc.client.addMetadataToHeaders(headers, b.Metadata) diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go index b4ac22d82..42fa702f6 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/client.go @@ -17,7 +17,6 @@ import ( "net/url" "regexp" "runtime" - "strconv" "strings" "time" @@ -76,19 +75,13 @@ type DefaultSender struct { // Send is the default retry strategy in the client func (ds *DefaultSender) Send(c *Client, req *http.Request) (resp *http.Response, err error) { - b := []byte{} - if req.Body != nil { - b, err = ioutil.ReadAll(req.Body) + rr := autorest.NewRetriableRequest(req) + for attempts := 0; attempts < ds.RetryAttempts; attempts++ { + err = rr.Prepare() if err != nil { return resp, err } - } - - for attempts := 0; attempts < ds.RetryAttempts; attempts++ { - if len(b) > 0 { - req.Body = ioutil.NopCloser(bytes.NewBuffer(b)) - } - resp, err = c.HTTPClient.Do(req) + resp, err = c.HTTPClient.Do(rr.Request()) if err != nil || !autorest.ResponseHasStatusCode(resp, ds.ValidStatusCodes...) { return resp, err } @@ -129,7 +122,7 @@ type storageResponse struct { type odataResponse struct { storageResponse - odata odataErrorMessage + odata odataErrorWrapper } // AzureStorageServiceError contains fields of the error response from @@ -142,24 +135,25 @@ type AzureStorageServiceError struct { QueryParameterName string `xml:"QueryParameterName"` QueryParameterValue string `xml:"QueryParameterValue"` Reason string `xml:"Reason"` + Lang string StatusCode int RequestID string Date string APIVersion string } -type odataErrorMessageMessage struct { +type odataErrorMessage struct { Lang string `json:"lang"` Value string `json:"value"` } -type odataErrorMessageInternal struct { - Code string `json:"code"` - Message odataErrorMessageMessage `json:"message"` +type odataError struct { + Code string `json:"code"` + Message odataErrorMessage `json:"message"` } -type odataErrorMessage struct { - Err odataErrorMessageInternal `json:"odata.error"` +type odataErrorWrapper struct { + Err odataError `json:"odata.error"` } // UnexpectedStatusCodeError is returned when a storage service responds with neither an error @@ -404,18 +398,17 @@ func (c Client) exec(verb, url string, headers map[string]string, body io.Reader return nil, errors.New("azure/storage: error creating request: " + err.Error()) } - if clstr, ok := headers["Content-Length"]; ok { - // content length header is being signed, but completely ignored by golang. - // instead we have to use the ContentLength property on the request struct - // (see https://golang.org/src/net/http/request.go?s=18140:18370#L536 and - // https://golang.org/src/net/http/transfer.go?s=1739:2467#L49) - req.ContentLength, err = strconv.ParseInt(clstr, 10, 64) - if err != nil { - return nil, err + // if a body was provided ensure that the content length was set. + // http.NewRequest() will automatically do this for a handful of types + // and for those that it doesn't we will handle here. + if body != nil && req.ContentLength < 1 { + if lr, ok := body.(*io.LimitedReader); ok { + setContentLengthFromLimitedReader(req, lr) } } + for k, v := range headers { - req.Header.Add(k, v) + req.Header[k] = append(req.Header[k], v) // Must bypass case munging present in `Add` by using map functions directly. See https://github.com/Azure/azure-sdk-for-go/issues/645 } resp, err := c.Sender.Send(&c, req) @@ -423,8 +416,7 @@ func (c Client) exec(verb, url string, headers map[string]string, body io.Reader return nil, err } - statusCode := resp.StatusCode - if statusCode >= 400 && statusCode <= 505 { + if resp.StatusCode >= 400 && resp.StatusCode <= 505 { var respBody []byte respBody, err = readAndCloseBody(resp.Body) if err != nil { @@ -436,10 +428,23 @@ func (c Client) exec(verb, url string, headers map[string]string, body io.Reader // no error in response body, might happen in HEAD requests err = serviceErrFromStatusCode(resp.StatusCode, resp.Status, requestID, date, version) } else { + storageErr := AzureStorageServiceError{ + StatusCode: resp.StatusCode, + RequestID: requestID, + Date: date, + APIVersion: version, + } // response contains storage service error object, unmarshal - storageErr, errIn := serviceErrFromXML(respBody, resp.StatusCode, requestID, date, version) - if err != nil { // error unmarshaling the error response - err = errIn + if resp.Header.Get("Content-Type") == "application/xml" { + errIn := serviceErrFromXML(respBody, &storageErr) + if err != nil { // error unmarshaling the error response + err = errIn + } + } else { + errIn := serviceErrFromJSON(respBody, &storageErr) + if err != nil { // error unmarshaling the error response + err = errIn + } } err = storageErr } @@ -595,18 +600,24 @@ func readAndCloseBody(body io.ReadCloser) ([]byte, error) { return out, err } -func serviceErrFromXML(body []byte, statusCode int, requestID, date, version string) (AzureStorageServiceError, error) { - storageErr := AzureStorageServiceError{ - StatusCode: statusCode, - RequestID: requestID, - Date: date, - APIVersion: version, - } - if err := xml.Unmarshal(body, &storageErr); err != nil { +func serviceErrFromXML(body []byte, storageErr *AzureStorageServiceError) error { + if err := xml.Unmarshal(body, storageErr); err != nil { storageErr.Message = fmt.Sprintf("Response body could no be unmarshaled: %v. Body: %v.", err, string(body)) - return storageErr, err + return err } - return storageErr, nil + return nil +} + +func serviceErrFromJSON(body []byte, storageErr *AzureStorageServiceError) error { + odataError := odataErrorWrapper{} + if err := json.Unmarshal(body, &odataError); err != nil { + storageErr.Message = fmt.Sprintf("Response body could no be unmarshaled: %v. Body: %v.", err, string(body)) + return err + } + storageErr.Code = odataError.Err.Code + storageErr.Message = odataError.Err.Message.Value + storageErr.Lang = odataError.Err.Message.Lang + return nil } func serviceErrFromStatusCode(code int, status string, requestID, date, version string) AzureStorageServiceError { diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/container.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/container.go index b20dfcf35..c2c9c055b 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/container.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/container.go @@ -414,6 +414,9 @@ func (c *Container) ListBlobs(params ListBlobsParameters) (BlobListResponse, err defer resp.body.Close() err = xmlUnmarshal(resp.body, &out) + for i := range out.Blobs { + out.Blobs[i].Container = c + } return out, err } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/copyblob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/copyblob.go index 377a3c622..f14342618 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/copyblob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/copyblob.go @@ -50,7 +50,7 @@ type IncrementalCopyOptionsConditions struct { // Copy starts a blob copy operation and waits for the operation to // complete. sourceBlob parameter must be a canonical URL to the blob (can be -// obtained using GetBlobURL method.) There is no SLA on blob copy and therefore +// obtained using the GetURL method.) There is no SLA on blob copy and therefore // this helper method works faster on smaller files. // // See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Copy-Blob @@ -65,7 +65,7 @@ func (b *Blob) Copy(sourceBlob string, options *CopyOptions) error { // StartCopy starts a blob copy operation. // sourceBlob parameter must be a canonical URL to the blob (can be -// obtained using GetBlobURL method.) +// obtained using the GetURL method.) // // See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Copy-Blob func (b *Blob) StartCopy(sourceBlob string, options *CopyOptions) (string, error) { diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/directory.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/directory.go index 29610329e..57053efd1 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/directory.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/directory.go @@ -4,6 +4,7 @@ import ( "encoding/xml" "net/http" "net/url" + "sync" ) // Directory represents a directory on a share. @@ -169,6 +170,7 @@ func (d *Directory) GetFileReference(name string) *File { Name: name, parent: d, share: d.share, + mutex: &sync.Mutex{}, } } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go index 238ac6d6d..27dbdd1fc 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/file.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "strconv" + "sync" ) const fourMB = uint64(4194304) @@ -22,6 +23,7 @@ type File struct { Properties FileProperties `xml:"Properties"` share *Share FileCopyProperties FileCopyState + mutex *sync.Mutex } // FileProperties contains various properties of a file. @@ -148,7 +150,9 @@ func (f *File) CopyFile(sourceURL string, options *FileRequestOptions) error { return err } - f.updateEtagLastModifiedAndCopyHeaders(headers) + f.updateEtagAndLastModified(headers) + f.FileCopyProperties.ID = headers.Get("X-Ms-Copy-Id") + f.FileCopyProperties.Status = headers.Get("X-Ms-Copy-Status") return nil } @@ -399,14 +403,6 @@ func (f *File) updateEtagAndLastModified(headers http.Header) { f.Properties.LastModified = headers.Get("Last-Modified") } -// updates Etag, last modified date and x-ms-copy-id -func (f *File) updateEtagLastModifiedAndCopyHeaders(headers http.Header) { - f.Properties.Etag = headers.Get("Etag") - f.Properties.LastModified = headers.Get("Last-Modified") - f.FileCopyProperties.ID = headers.Get("X-Ms-Copy-Id") - f.FileCopyProperties.Status = headers.Get("X-Ms-Copy-Status") -} - // updates file properties from the specified HTTP header func (f *File) updateProperties(header http.Header) { size, err := strconv.ParseUint(header.Get("Content-Length"), 10, 64) @@ -456,7 +452,11 @@ func (f *File) WriteRange(bytes io.Reader, fileRange FileRange, options *WriteRa if err != nil { return err } - + // it's perfectly legal for multiple go routines to call WriteRange + // on the same *File (e.g. concurrently writing non-overlapping ranges) + // so we must take the file mutex before updating our properties. + f.mutex.Lock() f.updateEtagAndLastModified(headers) + f.mutex.Unlock() return nil } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/pageblob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/pageblob.go index bc5b398d3..468b3868a 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/pageblob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/pageblob.go @@ -160,6 +160,8 @@ func (b *Blob) GetPageRanges(options *GetPageRangesOptions) (GetPageRangesRespon // size in bytes (size must be aligned to a 512-byte boundary). A page blob must // be created using this method before writing pages. // +// See CreateBlockBlobFromReader for more info on creating blobs. +// // See https://docs.microsoft.com/en-us/rest/api/storageservices/fileservices/Put-Blob func (b *Blob) PutPageBlob(options *PutBlobOptions) error { if b.Properties.ContentLength%512 != 0 { diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go index 8a902be2f..d3ae9d092 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/util.go @@ -136,7 +136,7 @@ func addTimeout(params url.Values, timeout uint) url.Values { func addSnapshot(params url.Values, snapshot *time.Time) url.Values { if snapshot != nil { - params.Add("snapshot", timeRfc1123Formatted(*snapshot)) + params.Add("snapshot", snapshot.Format("2006-01-02T15:04:05.0000000Z")) } return params } diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/util_1.7.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/util_1.7.go new file mode 100644 index 000000000..345bb28f2 --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/util_1.7.go @@ -0,0 +1,12 @@ +// +build !go1.8 + +package storage + +import ( + "io" + "net/http" +) + +func setContentLengthFromLimitedReader(req *http.Request, lr *io.LimitedReader) { + req.ContentLength = lr.N +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/util_1.8.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/util_1.8.go new file mode 100644 index 000000000..ed8b77919 --- /dev/null +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/util_1.8.go @@ -0,0 +1,18 @@ +// +build go1.8 + +package storage + +import ( + "io" + "io/ioutil" + "net/http" +) + +func setContentLengthFromLimitedReader(req *http.Request, lr *io.LimitedReader) { + req.ContentLength = lr.N + snapshot := *lr + req.GetBody = func() (io.ReadCloser, error) { + r := snapshot + return ioutil.NopCloser(&r), nil + } +} diff --git a/vendor/github.com/Azure/go-autorest/autorest/adal/token.go b/vendor/github.com/Azure/go-autorest/autorest/adal/token.go index 559fc6653..55361139a 100644 --- a/vendor/github.com/Azure/go-autorest/autorest/adal/token.go +++ b/vendor/github.com/Azure/go-autorest/autorest/adal/token.go @@ -33,6 +33,9 @@ const ( // managedIdentitySettingsPath is the path to the MSI Extension settings file (to discover the endpoint) managedIdentitySettingsPath = "/var/lib/waagent/ManagedIdentity-Settings" + + // metadataHeader is the header required by MSI extension + metadataHeader = "Metadata" ) var expirationBase time.Time @@ -364,6 +367,9 @@ func (spt *ServicePrincipalToken) refreshInternal(resource string) error { req.ContentLength = int64(len(s)) req.Header.Set(contentType, mimeTypeFormPost) + if _, ok := spt.secret.(*ServicePrincipalMSISecret); ok { + req.Header.Set(metadataHeader, "true") + } resp, err := spt.sender.Do(req) if err != nil { return fmt.Errorf("adal: Failed to execute the refresh request. Error = '%v'", err) diff --git a/vendor/github.com/Azure/go-autorest/autorest/authorization.go b/vendor/github.com/Azure/go-autorest/autorest/authorization.go index 7f4e3d845..314ed7876 100644 --- a/vendor/github.com/Azure/go-autorest/autorest/authorization.go +++ b/vendor/github.com/Azure/go-autorest/autorest/authorization.go @@ -3,10 +3,18 @@ package autorest import ( "fmt" "net/http" + "net/url" + "strings" "github.com/Azure/go-autorest/autorest/adal" ) +const ( + bearerChallengeHeader = "Www-Authenticate" + bearer = "Bearer" + tenantID = "tenantID" +) + // Authorizer is the interface that provides a PrepareDecorator used to supply request // authorization. Most often, the Authorizer decorator runs last so it has access to the full // state of the formed HTTP request. @@ -55,3 +63,105 @@ func (ba *BearerAuthorizer) WithAuthorization() PrepareDecorator { }) } } + +// BearerAuthorizerCallbackFunc is the authentication callback signature. +type BearerAuthorizerCallbackFunc func(tenantID, resource string) (*BearerAuthorizer, error) + +// BearerAuthorizerCallback implements bearer authorization via a callback. +type BearerAuthorizerCallback struct { + sender Sender + callback BearerAuthorizerCallbackFunc +} + +// NewBearerAuthorizerCallback creates a bearer authorization callback. The callback +// is invoked when the HTTP request is submitted. +func NewBearerAuthorizerCallback(sender Sender, callback BearerAuthorizerCallbackFunc) *BearerAuthorizerCallback { + if sender == nil { + sender = &http.Client{} + } + return &BearerAuthorizerCallback{sender: sender, callback: callback} +} + +// WithAuthorization returns a PrepareDecorator that adds an HTTP Authorization header whose value +// is "Bearer " followed by the token. The BearerAuthorizer is obtained via a user-supplied callback. +// +// By default, the token will be automatically refreshed through the Refresher interface. +func (bacb *BearerAuthorizerCallback) WithAuthorization() PrepareDecorator { + return func(p Preparer) Preparer { + return PreparerFunc(func(r *http.Request) (*http.Request, error) { + // make a copy of the request and remove the body as it's not + // required and avoids us having to create a copy of it. + rCopy := *r + removeRequestBody(&rCopy) + + resp, err := bacb.sender.Do(&rCopy) + if err == nil && resp.StatusCode == 401 { + defer resp.Body.Close() + if hasBearerChallenge(resp) { + bc, err := newBearerChallenge(resp) + if err != nil { + return r, err + } + if bacb.callback != nil { + ba, err := bacb.callback(bc.values[tenantID], bc.values["resource"]) + if err != nil { + return r, err + } + return ba.WithAuthorization()(p).Prepare(r) + } + } + } + return r, err + }) + } +} + +// returns true if the HTTP response contains a bearer challenge +func hasBearerChallenge(resp *http.Response) bool { + authHeader := resp.Header.Get(bearerChallengeHeader) + if len(authHeader) == 0 || strings.Index(authHeader, bearer) < 0 { + return false + } + return true +} + +type bearerChallenge struct { + values map[string]string +} + +func newBearerChallenge(resp *http.Response) (bc bearerChallenge, err error) { + challenge := strings.TrimSpace(resp.Header.Get(bearerChallengeHeader)) + trimmedChallenge := challenge[len(bearer)+1:] + + // challenge is a set of key=value pairs that are comma delimited + pairs := strings.Split(trimmedChallenge, ",") + if len(pairs) < 1 { + err = fmt.Errorf("challenge '%s' contains no pairs", challenge) + return bc, err + } + + bc.values = make(map[string]string) + for i := range pairs { + trimmedPair := strings.TrimSpace(pairs[i]) + pair := strings.Split(trimmedPair, "=") + if len(pair) == 2 { + // remove the enclosing quotes + key := strings.Trim(pair[0], "\"") + value := strings.Trim(pair[1], "\"") + + switch key { + case "authorization", "authorization_uri": + // strip the tenant ID from the authorization URL + asURL, err := url.Parse(value) + if err != nil { + return bc, err + } + bc.values[tenantID] = asURL.Path[1:] + default: + bc.values[key] = value + } + } + } + + return bc, err +} diff --git a/vendor/github.com/Azure/go-autorest/autorest/client.go b/vendor/github.com/Azure/go-autorest/autorest/client.go index b5f94b5c3..5f1e72fbe 100644 --- a/vendor/github.com/Azure/go-autorest/autorest/client.go +++ b/vendor/github.com/Azure/go-autorest/autorest/client.go @@ -35,6 +35,7 @@ var ( statusCodesForRetry = []int{ http.StatusRequestTimeout, // 408 + http.StatusTooManyRequests, // 429 http.StatusInternalServerError, // 500 http.StatusBadGateway, // 502 http.StatusServiceUnavailable, // 503 diff --git a/vendor/github.com/Azure/go-autorest/autorest/error.go b/vendor/github.com/Azure/go-autorest/autorest/error.go index 4bcb8f27b..aaef2ac8e 100644 --- a/vendor/github.com/Azure/go-autorest/autorest/error.go +++ b/vendor/github.com/Azure/go-autorest/autorest/error.go @@ -31,6 +31,9 @@ type DetailedError struct { // Service Error is the response body of failed API in bytes ServiceError []byte + + // Response is the response object that was returned during failure if applicable. + Response *http.Response } // NewError creates a new Error conforming object from the passed packageType, method, and @@ -67,6 +70,7 @@ func NewErrorWithError(original error, packageType string, method string, resp * Method: method, StatusCode: statusCode, Message: fmt.Sprintf(message, args...), + Response: resp, } } diff --git a/vendor/github.com/Azure/go-autorest/autorest/retriablerequest.go b/vendor/github.com/Azure/go-autorest/autorest/retriablerequest.go new file mode 100644 index 000000000..0ab1eb300 --- /dev/null +++ b/vendor/github.com/Azure/go-autorest/autorest/retriablerequest.go @@ -0,0 +1,38 @@ +package autorest + +import ( + "bytes" + "io" + "io/ioutil" + "net/http" +) + +// NewRetriableRequest returns a wrapper around an HTTP request that support retry logic. +func NewRetriableRequest(req *http.Request) *RetriableRequest { + return &RetriableRequest{req: req} +} + +// Request returns the wrapped HTTP request. +func (rr *RetriableRequest) Request() *http.Request { + return rr.req +} + +func (rr *RetriableRequest) prepareFromByteReader() (err error) { + // fall back to making a copy (only do this once) + b := []byte{} + if rr.req.ContentLength > 0 { + b = make([]byte, rr.req.ContentLength) + _, err = io.ReadFull(rr.req.Body, b) + if err != nil { + return err + } + } else { + b, err = ioutil.ReadAll(rr.req.Body) + if err != nil { + return err + } + } + rr.br = bytes.NewReader(b) + rr.req.Body = ioutil.NopCloser(rr.br) + return err +} diff --git a/vendor/github.com/Azure/go-autorest/autorest/retriablerequest_1.7.go b/vendor/github.com/Azure/go-autorest/autorest/retriablerequest_1.7.go new file mode 100644 index 000000000..e28eb2cbd --- /dev/null +++ b/vendor/github.com/Azure/go-autorest/autorest/retriablerequest_1.7.go @@ -0,0 +1,44 @@ +// +build !go1.8 + +package autorest + +import ( + "bytes" + "net/http" +) + +// RetriableRequest provides facilities for retrying an HTTP request. +type RetriableRequest struct { + req *http.Request + br *bytes.Reader + reset bool +} + +// Prepare signals that the request is about to be sent. +func (rr *RetriableRequest) Prepare() (err error) { + // preserve the request body; this is to support retry logic as + // the underlying transport will always close the reqeust body + if rr.req.Body != nil { + if rr.reset { + if rr.br != nil { + _, err = rr.br.Seek(0, 0 /*io.SeekStart*/) + } + rr.reset = false + if err != nil { + return err + } + } + if rr.br == nil { + // fall back to making a copy (only do this once) + err = rr.prepareFromByteReader() + } + // indicates that the request body needs to be reset + rr.reset = true + } + return err +} + +func removeRequestBody(req *http.Request) { + req.Body = nil + req.ContentLength = 0 +} diff --git a/vendor/github.com/Azure/go-autorest/autorest/retriablerequest_1.8.go b/vendor/github.com/Azure/go-autorest/autorest/retriablerequest_1.8.go new file mode 100644 index 000000000..8c1d1aec8 --- /dev/null +++ b/vendor/github.com/Azure/go-autorest/autorest/retriablerequest_1.8.go @@ -0,0 +1,56 @@ +// +build go1.8 + +package autorest + +import ( + "bytes" + "io" + "net/http" +) + +// RetriableRequest provides facilities for retrying an HTTP request. +type RetriableRequest struct { + req *http.Request + rc io.ReadCloser + br *bytes.Reader + reset bool +} + +// Prepare signals that the request is about to be sent. +func (rr *RetriableRequest) Prepare() (err error) { + // preserve the request body; this is to support retry logic as + // the underlying transport will always close the reqeust body + if rr.req.Body != nil { + if rr.reset { + if rr.rc != nil { + rr.req.Body = rr.rc + } else if rr.br != nil { + _, err = rr.br.Seek(0, io.SeekStart) + } + rr.reset = false + if err != nil { + return err + } + } + if rr.req.GetBody != nil { + // this will allow us to preserve the body without having to + // make a copy. note we need to do this on each iteration + rr.rc, err = rr.req.GetBody() + if err != nil { + return err + } + } else if rr.br == nil { + // fall back to making a copy (only do this once) + err = rr.prepareFromByteReader() + } + // indicates that the request body needs to be reset + rr.reset = true + } + return err +} + +func removeRequestBody(req *http.Request) { + req.Body = nil + req.GetBody = nil + req.ContentLength = 0 +} diff --git a/vendor/github.com/Azure/go-autorest/autorest/sender.go b/vendor/github.com/Azure/go-autorest/autorest/sender.go index 9c0697815..94b029847 100644 --- a/vendor/github.com/Azure/go-autorest/autorest/sender.go +++ b/vendor/github.com/Azure/go-autorest/autorest/sender.go @@ -1,12 +1,11 @@ package autorest import ( - "bytes" "fmt" - "io/ioutil" "log" "math" "net/http" + "strconv" "time" ) @@ -175,8 +174,13 @@ func DoPollForStatusCodes(duration time.Duration, delay time.Duration, codes ... func DoRetryForAttempts(attempts int, backoff time.Duration) SendDecorator { return func(s Sender) Sender { return SenderFunc(func(r *http.Request) (resp *http.Response, err error) { + rr := NewRetriableRequest(r) for attempt := 0; attempt < attempts; attempt++ { - resp, err = s.Do(r) + err = rr.Prepare() + if err != nil { + return resp, err + } + resp, err = s.Do(rr.Request()) if err == nil { return resp, err } @@ -194,29 +198,43 @@ func DoRetryForAttempts(attempts int, backoff time.Duration) SendDecorator { func DoRetryForStatusCodes(attempts int, backoff time.Duration, codes ...int) SendDecorator { return func(s Sender) Sender { return SenderFunc(func(r *http.Request) (resp *http.Response, err error) { - b := []byte{} - if r.Body != nil { - b, err = ioutil.ReadAll(r.Body) - if err != nil { - return resp, err - } - } - + rr := NewRetriableRequest(r) // Increment to add the first call (attempts denotes number of retries) attempts++ for attempt := 0; attempt < attempts; attempt++ { - r.Body = ioutil.NopCloser(bytes.NewBuffer(b)) - resp, err = s.Do(r) + err = rr.Prepare() + if err != nil { + return resp, err + } + resp, err = s.Do(rr.Request()) if err != nil || !ResponseHasStatusCode(resp, codes...) { return resp, err } - DelayForBackoff(backoff, attempt, r.Cancel) + delayed := DelayWithRetryAfter(resp, r.Cancel) + if !delayed { + DelayForBackoff(backoff, attempt, r.Cancel) + } } return resp, err }) } } +// DelayWithRetryAfter invokes time.After for the duration specified in the "Retry-After" header in +// responses with status code 429 +func DelayWithRetryAfter(resp *http.Response, cancel <-chan struct{}) bool { + retryAfter, _ := strconv.Atoi(resp.Header.Get("Retry-After")) + if resp.StatusCode == http.StatusTooManyRequests && retryAfter > 0 { + select { + case <-time.After(time.Duration(retryAfter) * time.Second): + return true + case <-cancel: + return false + } + } + return false +} + // DoRetryForDuration returns a SendDecorator that retries the request until the total time is equal // to or greater than the specified duration, exponentially backing off between requests using the // supplied backoff time.Duration (which may be zero). Retrying may be canceled by closing the @@ -224,9 +242,14 @@ func DoRetryForStatusCodes(attempts int, backoff time.Duration, codes ...int) Se func DoRetryForDuration(d time.Duration, backoff time.Duration) SendDecorator { return func(s Sender) Sender { return SenderFunc(func(r *http.Request) (resp *http.Response, err error) { + rr := NewRetriableRequest(r) end := time.Now().Add(d) for attempt := 0; time.Now().Before(end); attempt++ { - resp, err = s.Do(r) + err = rr.Prepare() + if err != nil { + return resp, err + } + resp, err = s.Do(rr.Request()) if err == nil { return resp, err } diff --git a/vendor/github.com/Azure/go-autorest/autorest/validation/validation.go b/vendor/github.com/Azure/go-autorest/autorest/validation/validation.go index d7b0eadc5..38f0074d0 100644 --- a/vendor/github.com/Azure/go-autorest/autorest/validation/validation.go +++ b/vendor/github.com/Azure/go-autorest/autorest/validation/validation.go @@ -205,14 +205,14 @@ func validateString(x reflect.Value, v Constraint) error { return createError(x, v, fmt.Sprintf("rule must be integer value for %v constraint; got: %v", v.Name, v.Rule)) } if len(s) > v.Rule.(int) { - return createError(x, v, fmt.Sprintf("value length must be less than %v", v.Rule)) + return createError(x, v, fmt.Sprintf("value length must be less than or equal to %v", v.Rule)) } case MinLength: if _, ok := v.Rule.(int); !ok { return createError(x, v, fmt.Sprintf("rule must be integer value for %v constraint; got: %v", v.Name, v.Rule)) } if len(s) < v.Rule.(int) { - return createError(x, v, fmt.Sprintf("value length must be greater than %v", v.Rule)) + return createError(x, v, fmt.Sprintf("value length must be greater than or equal to %v", v.Rule)) } case ReadOnly: if len(s) > 0 { @@ -273,6 +273,17 @@ func validateArrayMap(x reflect.Value, v Constraint) error { if x.Len() != 0 { return createError(x, v, "readonly parameter; must send as nil or empty in request") } + case Pattern: + reg, err := regexp.Compile(v.Rule.(string)) + if err != nil { + return createError(x, v, err.Error()) + } + keys := x.MapKeys() + for _, k := range keys { + if !reg.MatchString(k.String()) { + return createError(k, v, fmt.Sprintf("map key doesn't match pattern %v", v.Rule)) + } + } default: return createError(x, v, fmt.Sprintf("constraint %v is not applicable to array, slice and map type", v.Name)) } diff --git a/vendor/vendor.json b/vendor/vendor.json index d6415ebe2..b29b6b090 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -15,79 +15,76 @@ "revisionTime": "2017-01-18T16:13:56Z" }, { - "checksumSHA1": "iJ8ZV+uhUAtNn91pkKjfIY0z+44=", + "checksumSHA1": "mD5cAEaOLqhUeaFHbE8CLkZwM0M=", "path": "github.com/Azure/azure-sdk-for-go/arm/resources/resources", - "revision": "5841475edc7c8725d79885d635aa8956f97fdf0e", - "revisionTime": "2017-05-10T22:14:13Z" + "revision": "57db66900881e9fd21fd041a9d013514700ecab3", + "revisionTime": "2017-08-18T20:19:01Z", + "version": "v10.3.0-beta", + "versionExact": "v10.3.0-beta" }, { - "checksumSHA1": "QiYT2buD7yqmgxfl/ESn8yJmQTY=", + "checksumSHA1": "IHdg51g2WKaTB5WXbbn9gNrtv0A=", "path": "github.com/Azure/azure-sdk-for-go/arm/storage", - "revision": "5841475edc7c8725d79885d635aa8956f97fdf0e", - "revisionTime": "2017-05-10T22:14:13Z", - "version": "v10.0.2-beta", - "versionExact": "v10.0.2-beta" + "revision": "57db66900881e9fd21fd041a9d013514700ecab3", + "revisionTime": "2017-08-18T20:19:01Z", + "version": "v10.3.0-beta", + "versionExact": "v10.3.0-beta" }, { - "checksumSHA1": "kaK7epAI09FJJHMAoVUKAkTD73E=", + "checksumSHA1": "KWdWO4eMy7/x85pgQhngfaTiqz8=", "path": "github.com/Azure/azure-sdk-for-go/storage", - "revision": "5841475edc7c8725d79885d635aa8956f97fdf0e", - "revisionTime": "2017-05-10T22:14:13Z", - "version": "v10.0.2-beta", - "versionExact": "v10.0.2-beta" + "revision": "57db66900881e9fd21fd041a9d013514700ecab3", + "revisionTime": "2017-08-18T20:19:01Z", + "version": "v10.3.0-beta", + "versionExact": "v10.3.0-beta" }, { - "checksumSHA1": "NwbvjCz9Xo4spo0C96Tq6WCLX7U=", - "comment": "v8.0.0", + "checksumSHA1": "+4d+Y67AMKKuyR1EO33Zdt+RVx0=", "path": "github.com/Azure/go-autorest/autorest", - "revision": "58f6f26e200fa5dfb40c9cd1c83f3e2c860d779d", - "revisionTime": "2017-04-28T17:52:31Z", - "version": "v8.0.0", - "versionExact": "v8.0.0" + "revision": "5432abe734f8d95c78340cd56712f912906e6514", + "revisionTime": "2017-08-29T19:03:17Z", + "version": "v8.3.1", + "versionExact": "v8.3.1" }, { - "checksumSHA1": "KOETWLsF6QW+lrPVPsMNHDZP+xA=", - "comment": "v8.0.0", + "checksumSHA1": "hebqp0dsOKrcolVlLEzz6AVW8do=", "path": "github.com/Azure/go-autorest/autorest/adal", - "revision": "58f6f26e200fa5dfb40c9cd1c83f3e2c860d779d", - "revisionTime": "2017-04-28T17:52:31Z", - "version": "v8.0.0", - "versionExact": "v8.0.0" + "revision": "5432abe734f8d95c78340cd56712f912906e6514", + "revisionTime": "2017-08-29T19:03:17Z", + "version": "v8.3.1", + "versionExact": "v8.3.1" }, { "checksumSHA1": "2KdBFgT4qY+fMOkBTa5vA9V0AiM=", - "comment": "v8.0.0", "path": "github.com/Azure/go-autorest/autorest/azure", - "revision": "58f6f26e200fa5dfb40c9cd1c83f3e2c860d779d", - "revisionTime": "2017-04-28T17:52:31Z", - "version": "v8.0.0", - "versionExact": "v8.0.0" + "revision": "5432abe734f8d95c78340cd56712f912906e6514", + "revisionTime": "2017-08-29T19:03:17Z", + "version": "v8.3.1", + "versionExact": "v8.3.1" }, { "checksumSHA1": "LSF/pNrjhIxl6jiS6bKooBFCOxI=", - "comment": "v8.0.0", "path": "github.com/Azure/go-autorest/autorest/date", - "revision": "58f6f26e200fa5dfb40c9cd1c83f3e2c860d779d", - "revisionTime": "2017-04-28T17:52:31Z", - "version": "v8.0.0", - "versionExact": "v8.0.0" + "revision": "5432abe734f8d95c78340cd56712f912906e6514", + "revisionTime": "2017-08-29T19:03:17Z", + "version": "v8.3.1", + "versionExact": "v8.3.1" }, { "checksumSHA1": "Ev8qCsbFjDlMlX0N2tYAhYQFpUc=", "path": "github.com/Azure/go-autorest/autorest/to", - "revision": "58f6f26e200fa5dfb40c9cd1c83f3e2c860d779d", - "revisionTime": "2017-04-28T17:52:31Z", - "version": "v8.0.0", - "versionExact": "v8.0.0" + "revision": "5432abe734f8d95c78340cd56712f912906e6514", + "revisionTime": "2017-08-29T19:03:17Z", + "version": "v8.3.1", + "versionExact": "v8.3.1" }, { - "checksumSHA1": "oBixceM+55gdk47iff8DSEIh3po=", - "comment": "v8.0.0", + "checksumSHA1": "rGkTfIycpeix5TAbZS74ceGAPHI=", "path": "github.com/Azure/go-autorest/autorest/validation", - "revision": "58f6f26e200fa5dfb40c9cd1c83f3e2c860d779d", - "revisionTime": "2017-04-28T17:52:31Z", - "version": "v8.0.0", - "versionExact": "v8.0.0" + "revision": "5432abe734f8d95c78340cd56712f912906e6514", + "revisionTime": "2017-08-29T19:03:17Z", + "version": "v8.3.1", + "versionExact": "v8.3.1" }, { "checksumSHA1": "PYNaEEt9v8iAvGVUD4do0YIeR1A=",