Skip to content

Commit

Permalink
Merge pull request #33 from jpalermo/pr-add-auth-for-protected-download
Browse files Browse the repository at this point in the history 
Add support to download stemcells from s3/gcs with auth
  • Loading branch information
Rui Yang authored Dec 1, 2023
2 parents 04d25c2 + 04ef393 commit 74b110a
Show file tree
Hide file tree
Showing 7 changed files with 300 additions and 44 deletions.
5 changes: 5 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ would match `3262.1` and `3262.1.1`, but not `3262.2`.
* `force_regular`: *Optional.* Default `false`. By default, the resource will always download light stemcells for IaaSes that support light stemcells.
If `force_regular` is `true`, the resource will ignore light stemcells and always download regular stemcells.

* `auth`: *Optional.* These credentials are used when downloading stemcells stored in a protected bucket.
Has the following sub-properties:
* `access_key`: *Required.* The HMAC access key
* `secret_key`: *Required.* The HMAC secret key

## Behavior

### `check`: Check for new versions of the stemcell.
Expand Down
120 changes: 99 additions & 21 deletions boshio/boshio.go
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,23 @@
package boshio

import (
"context"
"crypto/sha1"
"crypto/sha256"
"encoding/json"
"fmt"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"golang.org/x/sync/errgroup"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"

"golang.org/x/sync/errgroup"
)

//go:generate counterfeiter -o ../fakes/bar.go --fake-name Bar . bar
Expand All @@ -25,6 +28,11 @@ type bar interface {
Finish()
}

type Auth struct {
AccessKey string
SecretKey string
}

//go:generate counterfeiter -o ../fakes/ranger.go --fake-name Ranger . ranger
type ranger interface {
BuildRange(contentLength int64) ([]string, error)
Expand Down Expand Up @@ -115,27 +123,41 @@ func (c *Client) WriteMetadata(stemcell Stemcell, metadataKey string, metadataFi
return nil
}

func (c *Client) DownloadStemcell(stemcell Stemcell, location string, preserveFileName bool) error {
req, err := http.NewRequest("HEAD", stemcell.Details().URL, nil)
if err != nil {
return fmt.Errorf("failed to construct HEAD request: %s", err)
}
func (c *Client) DownloadStemcell(stemcell Stemcell, location string, preserveFileName bool, auth Auth) error {
var contentLength int64
var err error
stemcellFileName := "stemcell.tgz"
stemcellUrl := stemcell.Details().URL

resp, err := c.httpClient.Do(req)
if err != nil {
return err
if preserveFileName {
stemcellUrlObject, err := url.Parse(stemcellUrl)
if err != nil {
return err
}
stemcellFileName = filepath.Base(stemcellUrlObject.Path)
}

stemcellURL := resp.Request.URL.String()
if auth.AccessKey != "" {
contentLength, err = c.contentLengthWithAuth(stemcellUrl, auth)
if err != nil {
return fmt.Errorf("failed to fetch object metadata: %s", err)
}
} else {
req, err := http.NewRequest("HEAD", stemcellUrl, nil)
if err != nil {
return fmt.Errorf("failed to construct HEAD request: %s", err)
}

ranges, err := c.Ranger.BuildRange(resp.ContentLength)
if err != nil {
return err
resp, err := c.httpClient.Do(req)
if err != nil {
return err
}
contentLength = resp.ContentLength
}

stemcellFileName := "stemcell.tgz"
if preserveFileName {
stemcellFileName = filepath.Base(resp.Request.URL.Path)
ranges, err := c.Ranger.BuildRange(contentLength)
if err != nil {
return err
}

stemcellData, err := os.Create(filepath.Join(location, stemcellFileName))
Expand All @@ -144,7 +166,7 @@ func (c *Client) DownloadStemcell(stemcell Stemcell, location string, preserveFi
}
defer stemcellData.Close()

c.Bar.SetTotal(int64(resp.ContentLength))
c.Bar.SetTotal(contentLength)
c.Bar.Kickoff()

var g errgroup.Group
Expand All @@ -153,13 +175,20 @@ func (c *Client) DownloadStemcell(stemcell Stemcell, location string, preserveFi
g.Go(func() error {

offset, err := strconv.Atoi(strings.Split(byteRange, "-")[0])
offsetEnd, err := strconv.Atoi(strings.Split(byteRange, "-")[1])
bytes := offsetEnd - offset + 1
if err != nil {
return err
}

respBytes, err := c.retryableRequest(stemcellURL, byteRange)
if err != nil {
return err
var respBytes []byte
if auth.AccessKey != "" {
respBytes, err = c.fetchWithAuth(stemcellUrl, bytes, offset, auth)
} else {
respBytes, err = c.retryableRequest(stemcellUrl, byteRange)
if err != nil {
return err
}
}

bytesWritten, err := stemcellData.WriteAt(respBytes, int64(offset))
Expand Down Expand Up @@ -245,3 +274,52 @@ func (c Client) retryableRequest(stemcellURL string, byteRange string) ([]byte,
return respBytes, nil
}
}

func (c Client) fetchWithAuth(urlString string, bytes int, offset int, auth Auth) ([]byte, error) {
reader, err := c.minioReaderForObject(urlString, auth)
if err != nil {
return nil, err
}

byteSegment := make([]byte, bytes)
_, err = reader.ReadAt(byteSegment, int64(offset))
if err != nil {
return nil, err
}

return byteSegment, nil
}

func (c Client) contentLengthWithAuth(urlString string, auth Auth) (int64, error) {
reader, err := c.minioReaderForObject(urlString, auth)
if err != nil {
return 0, err
}
objectInfo, err := reader.Stat()
if err != nil {
return 0, err
}
return objectInfo.Size, nil
}

func (c Client) minioReaderForObject(urlString string, auth Auth) (*minio.Object, error) {
parsedUrl, _ := url.Parse(urlString)
pieces := strings.SplitN(parsedUrl.Path, "/", 3)
bucket, object := pieces[1], pieces[2]

minioOptions := &minio.Options{
Creds: credentials.NewStaticV4(auth.AccessKey, auth.SecretKey, ""),
Secure: parsedUrl.Scheme == "https",
}

client, err := minio.New(parsedUrl.Host, minioOptions)
if err != nil {
return nil, err
}

reader, err := client.GetObject(context.Background(), bucket, object, minio.GetObjectOptions{})
if err != nil {
return nil, err
}
return reader, nil
}
56 changes: 47 additions & 9 deletions boshio/boshio_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ func (e EOFReader) Read(p []byte) (int, error) {

var _ = Describe("Boshio", func() {
var (
auth boshio.Auth
httpClient boshio.HTTPClient
client *boshio.Client
ranger *fakes.Ranger
Expand All @@ -34,6 +35,7 @@ var _ = Describe("Boshio", func() {
)

BeforeEach(func() {
auth = boshio.Auth{}
ranger = &fakes.Ranger{}
bar = &fakes.Bar{}
forceRegular = false
Expand Down Expand Up @@ -197,7 +199,7 @@ var _ = Describe("Boshio", func() {
location, err := ioutil.TempDir("", "")
Expect(err).NotTo(HaveOccurred())

err = client.DownloadStemcell(stubStemcell, location, false)
err = client.DownloadStemcell(stubStemcell, location, false, auth)
Expect(err).NotTo(HaveOccurred())

content, err := ioutil.ReadFile(filepath.Join(location, "stemcell.tgz"))
Expand All @@ -211,14 +213,50 @@ var _ = Describe("Boshio", func() {
location, err := ioutil.TempDir("", "")
Expect(err).NotTo(HaveOccurred())

err = client.DownloadStemcell(stubStemcell, location, true)
err = client.DownloadStemcell(stubStemcell, location, true, auth)
Expect(err).NotTo(HaveOccurred())

content, err := ioutil.ReadFile(filepath.Join(location, "light-different-stemcell.tgz"))
Expect(err).NotTo(HaveOccurred())

Expect(string(content)).To(Equal("this string is definitely not long enough to be 100 bytes but we get it there with a little bit of.."))
})

Context("when using auth", func() {
BeforeEach(func() {
auth = boshio.Auth{
AccessKey: "access key",
SecretKey: "secret key",
}
})

It("writes the stemcell to the provided location", func() {
stubStemcell.Regular.URL = serverPath("bucket_name/path/to/heavy-stemcell.tgz")
boshioServer.Start()
location, err := ioutil.TempDir("", "")
Expect(err).NotTo(HaveOccurred())

err = client.DownloadStemcell(stubStemcell, location, false, auth)
Expect(err).NotTo(HaveOccurred())

content, err := ioutil.ReadFile(filepath.Join(location, "stemcell.tgz"))
Expect(err).NotTo(HaveOccurred())

Expect(string(content)).To(Equal("this string is definitely not long enough to be 100 bytes but we get it there with a little bit of.."))
})

Context("when the metadata cannot be fetched", func() {
BeforeEach(func() {
stubStemcell.Regular.URL = serverPath("bucket_name/path/to/nothing.tgz")
boshioServer.Start()
})

It("returns an error", func() {
err := client.DownloadStemcell(stubStemcell, "", false, auth)
Expect(err).To(MatchError(ContainSubstring("failed to fetch object metadata:")))
})
})
})
})

Context("when an error occurs", func() {
Expand Down Expand Up @@ -275,7 +313,7 @@ var _ = Describe("Boshio", func() {
},
}

err = client.DownloadStemcell(stubStemcell, location, false)
err = client.DownloadStemcell(stubStemcell, location, false, auth)
Expect(err).NotTo(HaveOccurred())

content, err := ioutil.ReadFile(filepath.Join(location, "stemcell.tgz"))
Expand All @@ -298,7 +336,7 @@ var _ = Describe("Boshio", func() {
},
}

err := client.DownloadStemcell(stubStemcell, "", false)
err := client.DownloadStemcell(stubStemcell, "", false, auth)
Expect(err).To(MatchError(ContainSubstring("failed to construct HEAD request:")))
})
})
Expand All @@ -308,7 +346,7 @@ var _ = Describe("Boshio", func() {
ranger.BuildRangeReturns([]string{}, errors.New("failed to build a range"))
boshioServer.Start()

err := client.DownloadStemcell(stubStemcell, "", true)
err := client.DownloadStemcell(stubStemcell, "", true, auth)
Expect(err).To(MatchError("failed to build a range"))
})
})
Expand All @@ -324,7 +362,7 @@ var _ = Describe("Boshio", func() {
err = location.Close()
Expect(err).NotTo(HaveOccurred())

err = client.DownloadStemcell(stubStemcell, location.Name(), true)
err = client.DownloadStemcell(stubStemcell, location.Name(), true, auth)
Expect(err).To(MatchError(ContainSubstring("not a directory")))
})
})
Expand All @@ -335,7 +373,7 @@ var _ = Describe("Boshio", func() {
location, err := ioutil.TempDir("", "")
Expect(err).NotTo(HaveOccurred())

err = client.DownloadStemcell(stubStemcell, location, true)
err = client.DownloadStemcell(stubStemcell, location, true, auth)
Expect(err).To(MatchError("computed sha1 da39a3ee5e6b4b0d3255bfef95601890afd80709 did not match expected sha1 of 2222"))
})
})
Expand All @@ -347,7 +385,7 @@ var _ = Describe("Boshio", func() {
location, err := ioutil.TempDir("", "")
Expect(err).NotTo(HaveOccurred())

err = client.DownloadStemcell(stubStemcell, location, true)
err = client.DownloadStemcell(stubStemcell, location, true, auth)
Expect(err).To(MatchError("computed sha256 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 did not match expected sha256 of 4444"))
})
})
Expand All @@ -363,7 +401,7 @@ var _ = Describe("Boshio", func() {
location, err := ioutil.TempDir("", "")
Expect(err).NotTo(HaveOccurred())

err = client.DownloadStemcell(stubStemcell, location, true)
err = client.DownloadStemcell(stubStemcell, location, true, auth)
Expect(err).To(MatchError(ContainSubstring("failed to download stemcell - boshio returned 500")))
})
})
Expand Down
11 changes: 11 additions & 0 deletions boshio/init_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
package boshio_test

import (
"github.com/johannesboyne/gofakes3"
"github.com/johannesboyne/gofakes3/backend/s3mem"
"net/http"
"net/http/httptest"
"regexp"
"strconv"
"strings"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
Expand All @@ -28,6 +31,7 @@ type server struct {
LightAPIHandler http.HandlerFunc
HeavyAPIHandler http.HandlerFunc
HeavyAndLightAPIHandler http.HandlerFunc
S3Handler http.Handler
mux *http.ServeMux
s *httptest.Server
}
Expand All @@ -38,6 +42,7 @@ func (s *server) Start() {
s.mux.HandleFunc("/api/v1/stemcells/some-light-stemcell", boshioServer.LightAPIHandler)
s.mux.HandleFunc("/api/v1/stemcells/some-heavy-stemcell", boshioServer.HeavyAPIHandler)
s.mux.HandleFunc("/api/v1/stemcells/some-light-and-heavy-stemcell", boshioServer.HeavyAndLightAPIHandler)
s.mux.Handle("/bucket_name/", boshioServer.S3Handler)

s.s.Start()
}
Expand All @@ -53,12 +58,18 @@ func (s *server) URL() string {
var _ = BeforeEach(func() {
router := http.NewServeMux()
testServer := httptest.NewUnstartedServer(router)
s3Backend := s3mem.New()
fakeS3 := gofakes3.New(s3Backend)
s3Backend.CreateBucket("bucket_name")
stemcellContent := strings.NewReader("this string is definitely not long enough to be 100 bytes but we get it there with a little bit of..")
s3Backend.PutObject("bucket_name", "path/to/heavy-stemcell.tgz", map[string]string{"Last-Modified": "Mon, 2 Jan 2006 15:04:05 GMT"}, stemcellContent, 100)
boshioServer = &server{
mux: router,
TarballHandler: tarballHandler,
LightAPIHandler: lightAPIHandler,
HeavyAPIHandler: heavyAPIHandler,
HeavyAndLightAPIHandler: heavyAndLightAPIHandler,
S3Handler: fakeS3.Server(),
s: testServer,
}
})
Expand Down
Loading

0 comments on commit 74b110a

Please sign in to comment.