terraform/vendor/github.com/joyent/gomanta/manta/manta.go

460 lines
15 KiB
Go
Raw Normal View History

2016-09-13 21:13:20 +02:00
/*
The gomanta/manta package interacts with the Manta API (http://apidocs.joyent.com/manta/api.html).
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
Copyright (c) 2016 Joyent Inc.
Written by Daniele Stroppa <daniele.stroppa@joyent.com>
*/
package manta
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"path"
"time"
"github.com/joyent/gocommon/client"
"github.com/joyent/gocommon/errors"
jh "github.com/joyent/gocommon/http"
)
const (
// The default version of the Manta API to use
DefaultAPIVersion = "7.1"
// Manta API URL parts
apiStorage = "stor"
apiJobs = "jobs"
apiJobsLive = "live"
apiJobsIn = "in"
apiJobsOut = "out"
apiJobsFail = "fail"
apiJobsErr = "err"
apiJobsEnd = "end"
apiJobsCancel = "cancel"
apiJobsStatus = "status"
)
// Client provides a means to access Joyent Manta
type Client struct {
client client.Client
}
// New creates a new Client.
func New(client client.Client) *Client {
return &Client{client}
}
// request represents an API request
type request struct {
method string
url string
reqValue interface{}
reqHeader http.Header
reqReader io.Reader
reqLength int
resp interface{}
respHeader *http.Header
expectedStatus int
}
// Helper method to send an API request
func (c *Client) sendRequest(req request) (*jh.ResponseData, error) {
request := jh.RequestData{
ReqValue: req.reqValue,
ReqHeaders: req.reqHeader,
ReqReader: req.reqReader,
ReqLength: req.reqLength,
}
if req.expectedStatus == 0 {
req.expectedStatus = http.StatusOK
}
respData := jh.ResponseData{
RespValue: req.resp,
RespHeaders: req.respHeader,
ExpectedStatus: []int{req.expectedStatus},
}
err := c.client.SendRequest(req.method, req.url, "", &request, &respData)
return &respData, err
}
// Helper method to create the API URL
func makeURL(parts ...string) string {
return path.Join(parts...)
}
// ListDirectoryOpts represent the option that can be specified
// when listing a directory.
type ListDirectoryOpts struct {
Limit int `json:"limit"` // Limit to the number of records returned (default and max is 1000)
Marker string `json:"marker"` // Key name at which to start the next listing
}
// Entry represents an object stored in Manta, either a file or a directory
type Entry struct {
Name string `json:"name"` // Entry name
Etag string `json:"etag,omitempty"` // If type is 'object', object UUID
Size int `json:"size,omitempty"` // If type is 'object', object size (content-length)
Type string `json:"type"` // Entry type, one of 'directory' or 'object'
Mtime string `json:"mtime"` // ISO8601 timestamp of the last update
}
// Creates a directory at the specified path. Any parent directory must exist.
// See API docs: http://apidocs.joyent.com/manta/api.html#PutDirectory
func (c *Client) PutDirectory(path string) error {
requestHeaders := make(http.Header)
requestHeaders.Set("Content-Type", "application/json; type=directory")
requestHeaders.Set("Accept", "*/*")
req := request{
method: client.PUT,
url: makeURL(apiStorage, path),
reqHeader: requestHeaders,
expectedStatus: http.StatusNoContent,
}
if _, err := c.sendRequest(req); err != nil {
return errors.Newf(err, "failed to create directory: %s", path)
}
return nil
}
// Returns the content of the specified directory, using the specified options.
// See API docs: http://apidocs.joyent.com/manta/api.html#ListDirectory
func (c *Client) ListDirectory(directory string, opts ListDirectoryOpts) ([]Entry, error) {
var resp []Entry
requestHeaders := make(http.Header)
requestHeaders.Set("Accept", "*/*")
req := request{
method: client.GET,
url: makeURL(apiStorage, directory),
reqHeader: requestHeaders,
resp: &resp,
reqValue: opts,
}
if _, err := c.sendRequest(req); err != nil {
return nil, errors.Newf(err, "failed to list directory %s", directory)
}
return resp, nil
}
// Deletes the specified directory. Directory must be empty.
// See API docs: http://apidocs.joyent.com/manta/api.html#DeleteDirectory
func (c *Client) DeleteDirectory(path string) error {
req := request{
method: client.DELETE,
url: makeURL(apiStorage, path),
expectedStatus: http.StatusNoContent,
}
if _, err := c.sendRequest(req); err != nil {
return errors.Newf(err, "failed to delete directory %s", path)
}
return nil
}
// Creates an object at the specified path. Any parent directory must exist.
// See API docs: http://apidocs.joyent.com/manta/api.html#PutObject
func (c *Client) PutObject(path, objectName string, object []byte) error {
r := bytes.NewReader(object)
req := request{
method: client.PUT,
url: makeURL(apiStorage, path, objectName),
reqReader: r,
reqLength: len(object),
expectedStatus: http.StatusNoContent,
}
if _, err := c.sendRequest(req); err != nil {
return errors.Newf(err, "failed to create object: %s/%s", path, objectName)
}
return nil
}
// Retrieves the specified object from the specified location.
// See API docs: http://apidocs.joyent.com/manta/api.html#GetObject
func (c *Client) GetObject(path, objectName string) ([]byte, error) {
var resp []byte
requestHeaders := make(http.Header)
requestHeaders.Set("Accept", "*/*")
req := request{
method: client.GET,
url: makeURL(apiStorage, path, objectName),
reqHeader: requestHeaders,
resp: &resp,
}
respData, err := c.sendRequest(req)
if err != nil {
return nil, errors.Newf(err, "failed to get object %s/%s", path, objectName)
}
res, ok := respData.RespValue.(*[]byte)
if !ok {
return nil, errors.Newf(err, "failed to assert downloaded data as type *[]byte for object %s/%s", path, objectName)
}
return *res, nil
}
// Deletes the specified object from the specified location.
// See API docs: http://apidocs.joyent.com/manta/api.html#DeleteObject
func (c *Client) DeleteObject(path, objectName string) error {
req := request{
method: client.DELETE,
url: makeURL(apiStorage, path, objectName),
expectedStatus: http.StatusNoContent,
}
if _, err := c.sendRequest(req); err != nil {
return errors.Newf(err, "failed to delete object %s/%s", path, objectName)
}
return nil
}
// Creates a link (similar to a Unix hard link) from location to path/linkName.
// See API docs: http://apidocs.joyent.com/manta/api.html#PutSnapLink
func (c *Client) PutSnapLink(path, linkName, location string) error {
requestHeaders := make(http.Header)
requestHeaders.Set("Accept", "application/json; type=link")
requestHeaders.Set("Location", location)
req := request{
method: client.PUT,
url: makeURL(apiStorage, path, linkName),
reqHeader: requestHeaders,
expectedStatus: http.StatusNoContent,
}
if _, err := c.sendRequest(req); err != nil {
return errors.Newf(err, "failed to create snap link: %s/%s", path, linkName)
}
return nil
}
// CreateJobOpts represent the option that can be specified
// when creating a job.
type CreateJobOpts struct {
Name string `json:"name,omitempty"` // Job Name (optional)
Phases []Phase `json:"phases"` // Tasks to execute as part of this job
}
// Job represents the status of a job.
type Job struct {
Id string // Job unique identifier
Name string `json:"name,omitempty"` // Job Name
State string // Job state
Cancelled bool // Whether the job has been cancelled or not
InputDone bool // Whether the inputs for the job is still open or not
Stats JobStats `json:"stats,omitempty"` // Job statistics
TimeCreated string // Time the job was created at
TimeDone string `json:"timeDone,omitempty"` // Time the job was completed
TimeArchiveStarted string `json:"timeArchiveStarted,omitempty"` // Time the job archiving started
TimeArchiveDone string `json:"timeArchiveDone,omitempty"` // Time the job archiving completed
Phases []Phase `json:"phases"` // Job tasks
Options interface{} // Job options
}
// JobStats represents statistics about a job
type JobStats struct {
Errors int // Number or errors
Outputs int // Number of output produced
Retries int // Number of retries
Tasks int // Total number of task in the job
TasksDone int // number of tasks done
}
// Phase represents a task to be executed as part of a Job
type Phase struct {
Type string `json:"type,omitempty"` // Task type, one of 'map' or 'reduce' (optional)
Assets []string `json:"assets,omitempty"` // An array of objects to be placed in the compute zones (optional)
Exec string `json:"exec"` // The actual shell statement to execute
Init string `json:"init"` // Shell statement to execute in each compute zone before any tasks are executed
Count int `json:"count,omitempty"` // If type is 'reduce', an optional number of reducers for this phase (default is 1)
Memory int `json:"memory,omitempty"` // Amount of DRAM to give to your compute zone (in Mb, optional)
Disk int `json:"disk,omitempty"` // Amount of disk space to give to your compute zone (in Gb, optional)
}
// JobError represents an error occurred during a job execution
type JobError struct {
Id string // Job Id
Phase string // Phase number of the failure
What string // A human readable summary of what failed
Code string // Error code
Message string // Human readable error message
Stderr string // A key that saved the stderr for the given command (optional)
Key string // The input key being processed when the task failed (optional)
}
// Creates a job with the given options.
// See API docs: http://apidocs.joyent.com/manta/api.html#CreateJob
func (c *Client) CreateJob(opts CreateJobOpts) (string, error) {
var resp string
var respHeader http.Header
req := request{
method: client.POST,
url: apiJobs,
reqValue: opts,
respHeader: &respHeader,
resp: &resp,
expectedStatus: http.StatusCreated,
}
respData, err := c.sendRequest(req)
if err != nil {
return "", errors.Newf(err, "failed to create job with name: %s", opts.Name)
}
return respData.RespHeaders.Get("Location"), nil
}
// Submits inputs to an already created job.
// See API docs: http://apidocs.joyent.com/manta/api.html#AddJobInputs
func (c *Client) AddJobInputs(jobId string, jobInputs io.Reader) error {
inputData, errI := ioutil.ReadAll(jobInputs)
if errI != nil {
return errors.Newf(errI, "failed to read inputs for job %s", jobId)
}
requestHeaders := make(http.Header)
requestHeaders.Set("Accept", "*/*")
requestHeaders.Set("Content-Type", "text/plain")
req := request{
method: client.POST,
url: makeURL(apiJobs, jobId, apiJobsLive, apiJobsIn),
reqValue: string(inputData),
reqHeader: requestHeaders,
expectedStatus: http.StatusNoContent,
}
if _, err := c.sendRequest(req); err != nil {
return errors.Newf(err, "failed to add inputs to job %s", jobId)
}
return nil
}
// This closes input for a job, and finalize the job.
// See API docs: http://apidocs.joyent.com/manta/api.html#EndJobInput
func (c *Client) EndJobInputs(jobId string) error {
req := request{
method: client.POST,
url: makeURL(apiJobs, jobId, apiJobsLive, apiJobsIn, apiJobsEnd),
expectedStatus: http.StatusAccepted,
}
if _, err := c.sendRequest(req); err != nil {
return errors.Newf(err, "failed to end inputs for job %s", jobId)
}
return nil
}
// This cancels a job from doing any further work.
// Cancellation is asynchronous and "best effort"; there is no guarantee the job will actually stop
// See API docs: http://apidocs.joyent.com/manta/api.html#CancelJob
func (c *Client) CancelJob(jobId string) error {
req := request{
method: client.POST,
url: makeURL(apiJobs, jobId, apiJobsLive, apiJobsCancel),
expectedStatus: http.StatusAccepted,
}
if _, err := c.sendRequest(req); err != nil {
return errors.Newf(err, "failed to cancel job %s", jobId)
}
return nil
}
// Returns the list of jobs.
// Note you can filter the set of jobs down to only live jobs by setting the liveOnly flag.
// See API docs: http://apidocs.joyent.com/manta/api.html#ListJobs
func (c *Client) ListJobs(liveOnly bool) ([]Entry, error) {
var resp []Entry
var url string
if liveOnly {
url = fmt.Sprintf("%s?state=running", apiJobs)
} else {
url = apiJobs
}
req := request{
method: client.GET,
url: url,
resp: &resp,
}
if _, err := c.sendRequest(req); err != nil {
return nil, errors.Newf(err, "failed to list jobs")
}
return resp, nil
}
// Gets the high-level job container object for a given job.
// See API docs: http://apidocs.joyent.com/manta/api.html#GetJob
func (c *Client) GetJob(jobId string) (Job, error) {
var resp Job
req := request{
method: client.GET,
url: makeURL(apiJobs, jobId, apiJobsLive, apiJobsStatus),
resp: &resp,
}
if _, err := c.sendRequest(req); err != nil {
return Job{}, errors.Newf(err, "failed to get job with id: %s", jobId)
}
return resp, nil
}
// Returns the current "live" set of outputs from a given job.
// See API docs: http://apidocs.joyent.com/manta/api.html#GetJobOutput
func (c *Client) GetJobOutput(jobId string) (string, error) {
var resp string
req := request{
method: client.GET,
url: makeURL(apiJobs, jobId, apiJobsLive, apiJobsOut),
resp: &resp,
}
if _, err := c.sendRequest(req); err != nil {
return "", errors.Newf(err, "failed to get output for job with id: %s", jobId)
}
return resp, nil
}
// Returns the submitted input objects for a given job, available while the job is running.
// See API docs: http://apidocs.joyent.com/manta/api.html#GetJobInput
func (c *Client) GetJobInput(jobId string) (string, error) {
var resp string
req := request{
method: client.GET,
url: makeURL(apiJobs, jobId, apiJobsLive, apiJobsIn),
resp: &resp,
}
if _, err := c.sendRequest(req); err != nil {
return "", errors.Newf(err, "failed to get input for job with id: %s", jobId)
}
return resp, nil
}
// Returns the current "live" set of failures from a given job.
// See API docs: http://apidocs.joyent.com/manta/api.html#GetJobFailures
func (c *Client) GetJobFailures(jobId string) (interface{}, error) {
var resp interface{}
req := request{
method: client.GET,
url: makeURL(apiJobs, jobId, apiJobsLive, apiJobsFail),
resp: &resp,
}
if _, err := c.sendRequest(req); err != nil {
return nil, errors.Newf(err, "failed to get failures for job with id: %s", jobId)
}
return resp, nil
}
// Returns the current "live" set of errors from a given job.
// See API docs: http://apidocs.joyent.com/manta/api.html#GetJobErrors
func (c *Client) GetJobErrors(jobId string) ([]JobError, error) {
var resp []JobError
req := request{
method: client.GET,
url: makeURL(apiJobs, jobId, apiJobsLive, apiJobsErr),
resp: &resp,
}
if _, err := c.sendRequest(req); err != nil {
return nil, errors.Newf(err, "failed to get errors for job with id: %s", jobId)
}
return resp, nil
}
// Returns a signed URL to retrieve the object at path.
func (c *Client) SignURL(path string, expires time.Time) (string, error) {
return c.client.SignURL(path, expires)
}