173 lines
3.6 KiB
Go
173 lines
3.6 KiB
Go
package getter
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/url"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"cloud.google.com/go/storage"
|
|
"google.golang.org/api/iterator"
|
|
)
|
|
|
|
// GCSGetter is a Getter implementation that will download a module from
|
|
// a GCS bucket.
|
|
type GCSGetter struct {
|
|
getter
|
|
}
|
|
|
|
func (g *GCSGetter) ClientMode(u *url.URL) (ClientMode, error) {
|
|
ctx := g.Context()
|
|
|
|
// Parse URL
|
|
bucket, object, err := g.parseURL(u)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
client, err := storage.NewClient(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
iter := client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: object})
|
|
for {
|
|
obj, err := iter.Next()
|
|
if err != nil && err != iterator.Done {
|
|
return 0, err
|
|
}
|
|
|
|
if err == iterator.Done {
|
|
break
|
|
}
|
|
if strings.HasSuffix(obj.Name, "/") {
|
|
// A directory matched the prefix search, so this must be a directory
|
|
return ClientModeDir, nil
|
|
} else if obj.Name != object {
|
|
// A file matched the prefix search and doesn't have the same name
|
|
// as the query, so this must be a directory
|
|
return ClientModeDir, nil
|
|
}
|
|
}
|
|
// There are no directories or subdirectories, and if a match was returned,
|
|
// it was exactly equal to the prefix search. So return File mode
|
|
return ClientModeFile, nil
|
|
}
|
|
|
|
func (g *GCSGetter) Get(dst string, u *url.URL) error {
|
|
ctx := g.Context()
|
|
|
|
// Parse URL
|
|
bucket, object, err := g.parseURL(u)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Remove destination if it already exists
|
|
_, err = os.Stat(dst)
|
|
if err != nil && !os.IsNotExist(err) {
|
|
return err
|
|
}
|
|
if err == nil {
|
|
// Remove the destination
|
|
if err := os.RemoveAll(dst); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
// Create all the parent directories
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
|
|
client, err := storage.NewClient(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Iterate through all matching objects.
|
|
iter := client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: object})
|
|
for {
|
|
obj, err := iter.Next()
|
|
if err != nil && err != iterator.Done {
|
|
return err
|
|
}
|
|
if err == iterator.Done {
|
|
break
|
|
}
|
|
|
|
if !strings.HasSuffix(obj.Name, "/") {
|
|
// Get the object destination path
|
|
objDst, err := filepath.Rel(object, obj.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
objDst = filepath.Join(dst, objDst)
|
|
// Download the matching object.
|
|
err = g.getObject(ctx, client, objDst, bucket, obj.Name)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (g *GCSGetter) GetFile(dst string, u *url.URL) error {
|
|
ctx := g.Context()
|
|
|
|
// Parse URL
|
|
bucket, object, err := g.parseURL(u)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
client, err := storage.NewClient(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return g.getObject(ctx, client, dst, bucket, object)
|
|
}
|
|
|
|
func (g *GCSGetter) getObject(ctx context.Context, client *storage.Client, dst, bucket, object string) error {
|
|
rc, err := client.Bucket(bucket).Object(object).NewReader(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rc.Close()
|
|
|
|
// Create all the parent directories
|
|
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
|
return err
|
|
}
|
|
|
|
f, err := os.Create(dst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
|
|
_, err = Copy(ctx, f, rc)
|
|
return err
|
|
}
|
|
|
|
func (g *GCSGetter) parseURL(u *url.URL) (bucket, path string, err error) {
|
|
if strings.Contains(u.Host, "googleapis.com") {
|
|
hostParts := strings.Split(u.Host, ".")
|
|
if len(hostParts) != 3 {
|
|
err = fmt.Errorf("URL is not a valid GCS URL")
|
|
return
|
|
}
|
|
|
|
pathParts := strings.SplitN(u.Path, "/", 5)
|
|
if len(pathParts) != 5 {
|
|
err = fmt.Errorf("URL is not a valid GCS URL")
|
|
return
|
|
}
|
|
bucket = pathParts[3]
|
|
path = pathParts[4]
|
|
}
|
|
return
|
|
}
|