diff --git a/.github/scripts/dav/README.md b/.github/scripts/dav/README.md new file mode 100644 index 0000000..43ceb84 --- /dev/null +++ b/.github/scripts/dav/README.md @@ -0,0 +1,81 @@ +# DAV Integration Test Scripts + +Scripts for setting up and running WebDAV integration tests locally and in CI. + +## Scripts + +### `setup.sh` +Builds and starts a WebDAV test server using Docker. + +**Usage:** +```bash +./.github/scripts/dav/setup.sh +``` + +This will: +- Clean up any existing WebDAV containers +- Build the `webdav-test` Docker image from `dav/integration/testdata/` +- Start the server on `https://localhost:8443` +- Verify the server is ready + +### `run-int.sh` +Runs the DAV integration tests. + +**Required environment variables:** +- `DAV_ENDPOINT` - WebDAV server URL (e.g., `https://localhost:8443`) +- `DAV_USER` - Username for authentication (e.g., `testuser`) +- `DAV_PASSWORD` - Password for authentication (e.g., `testpass`) +- `DAV_CA_CERT` - CA certificate content (optional, for HTTPS with custom CA) +- `DAV_SECRET` - Secret for signed URLs (optional, for signed URL tests) + +**Usage:** +```bash +export DAV_ENDPOINT="https://localhost:8443" +export DAV_USER="testuser" +export DAV_PASSWORD="testpass" +export DAV_CA_CERT="$(cat dav/integration/testdata/certs/server.crt)" +export DAV_SECRET="test-secret-key" + +./.github/scripts/dav/run-int.sh +``` + +### `teardown.sh` +Cleans up the WebDAV test environment. + +**Usage:** +```bash +./.github/scripts/dav/teardown.sh +``` + +This will: +- Stop and remove the WebDAV container +- Remove the WebDAV test Docker image + +### `utils.sh` +Utility functions used by other scripts. Contains: +- `cleanup_webdav_container` - Stops and removes the WebDAV container +- `cleanup_webdav_image` - Removes the WebDAV test image + +## Running Locally + +To run the full integration test suite locally: + +```bash +# From the repository root +./.github/scripts/dav/setup.sh + +export DAV_ENDPOINT="https://localhost:8443" +export DAV_USER="testuser" +export DAV_PASSWORD="testpass" +export DAV_CA_CERT="$(cat dav/integration/testdata/certs/server.crt)" +export DAV_SECRET="test-secret-key" + +./.github/scripts/dav/run-int.sh + +# Cleanup +./.github/scripts/dav/teardown.sh +``` + +## CI Usage + +These scripts are used by the GitHub Actions workflow in `.github/workflows/dav-integration.yml`. diff --git a/.github/scripts/dav/run-int.sh b/.github/scripts/dav/run-int.sh new file mode 100755 index 0000000..b9275a9 --- /dev/null +++ b/.github/scripts/dav/run-int.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Get the directory where this script is located +script_dir="$( cd "$(dirname "${0}")" && pwd )" +repo_root="$(cd "${script_dir}/../../.." && pwd)" + +: "${DAV_ENDPOINT:?DAV_ENDPOINT environment variable must be set}" +: "${DAV_USER:?DAV_USER environment variable must be set}" +: "${DAV_PASSWORD:?DAV_PASSWORD environment variable must be set}" + +echo "Running DAV integration tests..." +echo " Endpoint: ${DAV_ENDPOINT}" +echo " User: ${DAV_USER}" + +pushd "${repo_root}/dav" > /dev/null + echo -e "\nRunning tests with $(go version)..." + ginkgo -v ./integration +popd > /dev/null diff --git a/.github/scripts/dav/setup.sh b/.github/scripts/dav/setup.sh new file mode 100755 index 0000000..867ef6e --- /dev/null +++ b/.github/scripts/dav/setup.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Get the directory where this script is located +script_dir="$( cd "$(dirname "${0}")" && pwd )" +repo_root="$(cd "${script_dir}/../../.." && pwd)" + +source "${script_dir}/utils.sh" + +# Cleanup any existing containers first +cleanup_webdav_container + +echo "Building WebDAV test server Docker image..." +cd "${repo_root}/dav/integration/testdata" +docker build -t webdav-test . + +echo "Starting WebDAV test server..." +docker run -d --name webdav -p 8443:443 webdav-test + +# Wait for Apache to be ready +echo "Waiting for Apache to start..." +sleep 5 + +# Verify htpasswd file in container +echo "Verifying htpasswd file in container..." +docker exec webdav cat /usr/local/apache2/htpasswd + +# Test connection +echo "Testing WebDAV server connection..." +if curl -k -u testuser:testpass -v https://localhost:8443/ 2>&1 | grep -q "200 OK\|301\|Authorization"; then + echo "✓ WebDAV server is ready" +else + echo "⚠ WebDAV server might not be fully ready yet" +fi diff --git a/.github/scripts/dav/teardown.sh b/.github/scripts/dav/teardown.sh new file mode 100755 index 0000000..9eeb5c3 --- /dev/null +++ b/.github/scripts/dav/teardown.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$( cd "$(dirname "${0}")" && pwd )" + +source "${script_dir}/utils.sh" + +echo "Tearing down WebDAV test environment..." +cleanup_webdav_container +cleanup_webdav_image + +echo "✓ Teardown complete" diff --git a/.github/scripts/dav/utils.sh b/.github/scripts/dav/utils.sh new file mode 100755 index 0000000..e7f6206 --- /dev/null +++ b/.github/scripts/dav/utils.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# Cleanup Docker container and image +function cleanup_webdav_container { + echo "Stopping and removing WebDAV container..." + docker stop webdav 2>/dev/null || true + docker rm webdav 2>/dev/null || true +} + +function cleanup_webdav_image { + echo "Removing WebDAV test image..." + docker rmi webdav-test 2>/dev/null || true +} diff --git a/.github/workflows/dav-integration.yml b/.github/workflows/dav-integration.yml new file mode 100644 index 0000000..4333eb0 --- /dev/null +++ b/.github/workflows/dav-integration.yml @@ -0,0 +1,49 @@ +name: DAV Integration Tests + +on: + workflow_dispatch: + pull_request: + paths: + - ".github/workflows/dav-integration.yml" + - "dav/**" + - "go.mod" + - "go.sum" + push: + branches: + - main + +concurrency: + group: dav-integration + cancel-in-progress: false + +jobs: + dav-integration: + name: DAV Integration Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Install Ginkgo + run: go install github.com/onsi/ginkgo/v2/ginkgo@latest + + - name: Setup WebDAV test server + run: ./.github/scripts/dav/setup.sh + + - name: Run Integration Tests + env: + DAV_ENDPOINT: "https://localhost:8443" + DAV_USER: "testuser" + DAV_PASSWORD: "testpass" + DAV_SECRET: "test-secret-key" + DAV_CA_CERT_FILE: "dav/integration/testdata/certs/server.crt" + run: | + export DAV_CA_CERT="$(cat ${DAV_CA_CERT_FILE})" + ./.github/scripts/dav/run-int.sh + diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 8369a8e..2fefdc0 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -42,7 +42,7 @@ jobs: run: | export CGO_ENABLED=0 go version - go test -v ./dav/... + go run github.com/onsi/ginkgo/v2/ginkgo --skip-package=integration ./dav/... - name: gcs unit tests run: | diff --git a/dav/README.md b/dav/README.md index 1641195..8fb561f 100644 --- a/dav/README.md +++ b/dav/README.md @@ -8,42 +8,129 @@ For general usage and build instructions, see the [main README](../README.md). ## DAV-Specific Configuration -The DAV client requires a JSON configuration file with WebDAV endpoint details and credentials. +The DAV client requires a JSON configuration file with the following structure: + +``` json +{ + "endpoint": " (required)", + "user": " (optional)", + "password": " (optional)", + "retry_attempts": (optional - default: 3), + "tls": { + "cert": { + "ca": " (optional - PEM-encoded CA certificate)" + } + }, + "secret": " (optional - required for pre-signed URLs)", + "signing_method": " (optional - 'sha256' (default) or 'md5')" +} +``` **Usage examples:** ```bash -# Upload an object -storage-cli -s dav -c dav-config.json put local-file.txt remote-object +# Upload a blob +storage-cli -s dav -c dav-config.json put local-file.txt remote-blob + +# Fetch a blob (destination file will be overwritten if exists) +storage-cli -s dav -c dav-config.json get remote-blob local-file.txt + +# Delete a blob +storage-cli -s dav -c dav-config.json delete remote-blob + +# Check if blob exists +storage-cli -s dav -c dav-config.json exists remote-blob -# Fetch an object -storage-cli -s dav -c dav-config.json get remote-object local-file.txt +# List all blobs +storage-cli -s dav -c dav-config.json list -# Delete an object -storage-cli -s dav -c dav-config.json delete remote-object +# List blobs with prefix +storage-cli -s dav -c dav-config.json list my-prefix -# Check if an object exists -storage-cli -s dav -c dav-config.json exists remote-object +# Copy a blob +storage-cli -s dav -c dav-config.json copy source-blob destination-blob -# Generate a signed URL (e.g., GET for 1 hour) -storage-cli -s dav -c dav-config.json sign remote-object get 60s +# Delete blobs by prefix +storage-cli -s dav -c dav-config.json delete-recursive my-prefix- + +# Get blob properties (outputs JSON with ContentLength, ETag, LastModified) +storage-cli -s dav -c dav-config.json properties remote-blob + +# Ensure storage exists (initialize WebDAV storage) +storage-cli -s dav -c dav-config.json ensure-storage-exists + +# Generate a pre-signed URL (e.g., GET for 3600 seconds) +storage-cli -s dav -c dav-config.json sign remote-blob get 3600s +``` + +### Using Signed URLs with curl + +```bash +# Downloading a blob: +curl -X GET + +# Uploading a blob: +curl -X PUT -T path/to/file ``` ## Pre-signed URLs The `sign` command generates a pre-signed URL for a specific object, action, and duration. -The request is signed using HMAC-SHA256 with a secret provided in the configuration. +The request is signed using the algorithm selected by `signing_method` configuration parameter with a secret provided in the configuration. + +**Supported signing methods:** +- **`sha256`** (default): HMAC-SHA256 signature +- **`md5`**: MD5-based signature + +The exact signature format depends on the selected signing method and signer implementation. + +The generated URL format varies based on signing method: +- **SHA256**: `/signed/{blob-path}?st={hmac-sha256}&ts={timestamp}&e={expires}` +- **MD5**: `/read/{blob-path}?md5={md5-hash}&expires={timestamp}` or `/write/{blob-path}?md5={md5-hash}&expires={timestamp}` -The HMAC format is: -`` +**Note:** Pre-signed URLs require the WebDAV server to have signature verification middleware. Standard WebDAV servers don't support this - it's a Cloud Foundry extension. -The generated URL format: -`https://blobstore.url/signed/object-id?st=HMACSignatureHash&ts=GenerationTimestamp&e=ExpirationTimestamp` +## Object Path Handling + +The DAV client treats object IDs as the final storage paths and uses them exactly as provided by the caller. The client does not apply any path transformations, partitioning, or prefixing - the caller is responsible for providing the complete object path including any directory structure. + +For example: +- Simple paths: `my-blob-id` +- Partitioned paths: `ab/cd/my-blob-id` +- Nested paths: `folder/subfolder/my-blob-id` + +All are stored exactly as specified. If your use case requires a specific directory layout (e.g., partitioning by hash prefix), implement this in the caller before invoking storage-cli. + +## Features + +### Automatic Retry Logic +All operations automatically retry on transient errors with 1-second delays between attempts. Default is 3 retry attempts, configurable via `retry_attempts` in config. + +### TLS/HTTPS Support +Supports HTTPS connections with custom CA certificates for internal or self-signed certificates. ## Testing ### Unit Tests Run unit tests from the repository root: + ```bash -ginkgo --cover -v -r ./dav/... +ginkgo --cover -v -r ./dav/client ``` + +Or using go test: +```bash +go test ./dav/client/... +``` + +### Integration Tests + +The DAV implementation includes Go-based integration tests that run against a real WebDAV server. These tests require a WebDAV server to be available and the following environment variables to be set: + +- `DAV_ENDPOINT` - WebDAV server URL +- `DAV_USER` - Username for authentication +- `DAV_PASSWORD` - Password for authentication +- `DAV_CA_CERT` - CA certificate (optional, for HTTPS with custom CA) +- `DAV_SECRET` - Secret for signed URLs (optional, for signed URL tests) + +If these environment variables are not set, the integration tests will be skipped. diff --git a/dav/app/app.go b/dav/app/app.go deleted file mode 100644 index dfbe1d8..0000000 --- a/dav/app/app.go +++ /dev/null @@ -1,80 +0,0 @@ -package app - -import ( - "errors" - "fmt" - "time" - - davcmd "github.com/cloudfoundry/storage-cli/dav/cmd" - davconfig "github.com/cloudfoundry/storage-cli/dav/config" -) - -type App struct { - runner davcmd.Runner - config davconfig.Config -} - -func New(r davcmd.Runner, c davconfig.Config) *App { - app := &App{runner: r, config: c} - return app -} - -func (app *App) run(args []string) (err error) { - - err = app.runner.SetConfig(app.config) - if err != nil { - err = fmt.Errorf("Invalid CA Certificate: %s", err.Error()) //nolint:staticcheck - return - } - - err = app.runner.Run(args) - return -} - -func (app *App) Put(sourceFilePath string, destinationObject string) error { - return app.run([]string{"put", sourceFilePath, destinationObject}) -} - -func (app *App) Get(sourceObject string, dest string) error { - return app.run([]string{"get", sourceObject, dest}) -} - -func (app *App) Delete(object string) error { - return app.run([]string{"delete", object}) -} - -func (app *App) Exists(object string) (bool, error) { - err := app.run([]string{"exists", object}) - if err != nil { - return false, err - } - return true, nil -} - -func (app *App) Sign(object string, action string, expiration time.Duration) (string, error) { - err := app.run([]string{"sign", object, action, expiration.String()}) - if err != nil { - return "", err - } - return "", nil -} - -func (app *App) List(prefix string) ([]string, error) { - return nil, errors.New("not implemented") -} - -func (app *App) Copy(srcBlob string, dstBlob string) error { - return errors.New("not implemented") -} - -func (app *App) Properties(dest string) error { - return errors.New("not implemented") -} - -func (app *App) EnsureStorageExists() error { - return errors.New("not implemented") -} - -func (app *App) DeleteRecursive(prefix string) error { - return errors.New("not implemented") -} diff --git a/dav/app/app_suite_test.go b/dav/app/app_suite_test.go deleted file mode 100644 index e4657e2..0000000 --- a/dav/app/app_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package app_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "testing" -) - -func TestApp(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Dav App Suite") -} diff --git a/dav/app/app_test.go b/dav/app/app_test.go deleted file mode 100644 index 71d00c2..0000000 --- a/dav/app/app_test.go +++ /dev/null @@ -1,166 +0,0 @@ -package app_test - -import ( - "errors" - "os" - "path/filepath" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - . "github.com/cloudfoundry/storage-cli/dav/app" - davconf "github.com/cloudfoundry/storage-cli/dav/config" -) - -type FakeRunner struct { - Config davconf.Config - SetConfigErr error - RunArgs []string - RunErr error -} - -func (r *FakeRunner) SetConfig(newConfig davconf.Config) (err error) { - r.Config = newConfig - return r.SetConfigErr -} - -func (r *FakeRunner) Run(cmdArgs []string) (err error) { - r.RunArgs = cmdArgs - return r.RunErr -} - -func pathToFixture(file string) string { - pwd, err := os.Getwd() - Expect(err).ToNot(HaveOccurred()) - - fixturePath := filepath.Join(pwd, "../test_assets", file) - - absPath, err := filepath.Abs(fixturePath) - Expect(err).ToNot(HaveOccurred()) - - return absPath -} - -var _ = Describe("App", func() { - - It("reads the CA cert from config", func() { - configFile, _ := os.Open(pathToFixture("dav-cli-config-with-ca.json")) //nolint:errcheck - defer configFile.Close() //nolint:errcheck - davConfig, _ := davconf.NewFromReader(configFile) //nolint:errcheck - - runner := &FakeRunner{} - app := New(runner, davConfig) - err := app.Put("localFile", "remoteFile") - Expect(err).ToNot(HaveOccurred()) - - expectedConfig := davconf.Config{ - User: "some user", - Password: "some pwd", - Endpoint: "https://example.com/some/endpoint", - Secret: "77D47E3A0B0F590B73CF3EBD9BB6761E244F90FA6F28BB39F941B0905789863FBE2861FDFD8195ADC81B72BB5310BC18969BEBBF4656366E7ACD3F0E4186FDDA", - TLS: davconf.TLS{ - Cert: davconf.Cert{ - CA: "ca-cert", - }, - }, - } - - Expect(runner.Config).To(Equal(expectedConfig)) - Expect(runner.Config.TLS.Cert.CA).ToNot(BeNil()) - }) - - It("returns error if CA Cert is invalid", func() { - configFile, _ := os.Open(pathToFixture("dav-cli-config-with-ca.json")) //nolint:errcheck - defer configFile.Close() //nolint:errcheck - davConfig, _ := davconf.NewFromReader(configFile) //nolint:errcheck - - runner := &FakeRunner{ - SetConfigErr: errors.New("invalid cert"), - } - - app := New(runner, davConfig) - err := app.Put("localFile", "remoteFile") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Invalid CA Certificate: invalid cert")) - - }) - - It("runs the put command", func() { - configFile, _ := os.Open(pathToFixture("dav-cli-config.json")) //nolint:errcheck - defer configFile.Close() //nolint:errcheck - davConfig, _ := davconf.NewFromReader(configFile) //nolint:errcheck - - runner := &FakeRunner{} - - app := New(runner, davConfig) - err := app.Put("localFile", "remoteFile") - Expect(err).ToNot(HaveOccurred()) - - expectedConfig := davconf.Config{ - User: "some user", - Password: "some pwd", - Endpoint: "http://example.com/some/endpoint", - Secret: "77D47E3A0B0F590B73CF3EBD9BB6761E244F90FA6F28BB39F941B0905789863FBE2861FDFD8195ADC81B72BB5310BC18969BEBBF4656366E7ACD3F0E4186FDDA", - } - - Expect(runner.Config).To(Equal(expectedConfig)) - Expect(runner.Config.TLS.Cert.CA).To(BeEmpty()) - Expect(runner.RunArgs).To(Equal([]string{"put", "localFile", "remoteFile"})) - }) - - It("returns error from the cmd runner", func() { - - configFile, _ := os.Open(pathToFixture("dav-cli-config.json")) //nolint:errcheck - defer configFile.Close() //nolint:errcheck - davConfig, _ := davconf.NewFromReader(configFile) //nolint:errcheck - - runner := &FakeRunner{ - RunErr: errors.New("fake-run-error"), - } - - app := New(runner, davConfig) - err := app.Put("localFile", "remoteFile") - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("fake-run-error")) - }) - - Context("Checking functionalities", func() { - // var app *App - var davConfig davconf.Config - BeforeEach(func() { - - configFile, _ := os.Open(pathToFixture("dav-cli-config.json")) //nolint:errcheck - defer configFile.Close() //nolint:errcheck - davConfig, _ = davconf.NewFromReader(configFile) //nolint:errcheck - }) - - It("Exists fails", func() { - - runner := &FakeRunner{ - RunErr: errors.New("object does not exist"), - } - app := New(runner, davConfig) - - exist, err := app.Exists("someObject") //nolint:errcheck - - Expect(err.Error()).To(ContainSubstring("object does not exist")) - Expect(exist).To(BeFalse()) - - }) - - It("Sign Fails", func() { - runner := &FakeRunner{ - RunErr: errors.New("can't sign"), - } - - app := New(runner, davConfig) - signedurl, err := app.Sign("someObject", "SomeObject", time.Second*100) - Expect(signedurl).To(BeEmpty()) - Expect(err.Error()).To(ContainSubstring("can't sign")) - - }) - - }) - -}) diff --git a/dav/client/client.go b/dav/client/client.go index cd43926..0d6725b 100644 --- a/dav/client/client.go +++ b/dav/client/client.go @@ -1,197 +1,189 @@ package client import ( - "crypto/sha1" "fmt" "io" - "net/http" - "net/url" - "path" - "strings" + "log/slog" + "os" "time" - URLsigner "github.com/cloudfoundry/storage-cli/dav/signer" - - bosherr "github.com/cloudfoundry/bosh-utils/errors" "github.com/cloudfoundry/bosh-utils/httpclient" boshlog "github.com/cloudfoundry/bosh-utils/logger" davconf "github.com/cloudfoundry/storage-cli/dav/config" ) -type Client interface { - Get(path string) (content io.ReadCloser, err error) - Put(path string, content io.ReadCloser, contentLength int64) (err error) - Exists(path string) (err error) - Delete(path string) (err error) - Sign(objectID, action string, duration time.Duration) (string, error) +// DavBlobstore implements the storage.Storager interface for WebDAV +type DavBlobstore struct { + storageClient StorageClient } -func NewClient(config davconf.Config, httpClient httpclient.Client, logger boshlog.Logger) (c Client) { +// New creates a new DavBlobstore instance +func New(config davconf.Config) (*DavBlobstore, error) { + logger := boshlog.NewLogger(boshlog.LevelNone) + + var httpClientBase httpclient.Client + var certPool, err = getCertPool(config) + if err != nil { + return nil, fmt.Errorf("failed to create certificate pool: %w", err) + } + + httpClientBase = httpclient.CreateDefaultClient(certPool) + if config.RetryAttempts == 0 { config.RetryAttempts = 3 } - // @todo should a logger now be passed in to this client? - duration := time.Duration(0) + // Retry with 1 second delay between attempts + duration := time.Duration(1) * time.Second retryClient := httpclient.NewRetryClient( - httpClient, + httpClientBase, config.RetryAttempts, duration, logger, ) - return client{ - config: config, - httpClient: retryClient, - } + storageClient := NewStorageClient(config, retryClient) + + return NewWithStorageClient(storageClient), nil } -type client struct { - config davconf.Config - httpClient httpclient.Client +// NewWithStorageClient creates a DavBlobstore with an injected StorageClient (for testing) +func NewWithStorageClient(storageClient StorageClient) *DavBlobstore { + return &DavBlobstore{storageClient: storageClient} } -func (c client) Get(path string) (io.ReadCloser, error) { - req, err := c.createReq("GET", path, nil) - if err != nil { - return nil, err - } +// Put uploads a file to the WebDAV server +func (d *DavBlobstore) Put(sourceFilePath string, dest string) error { + slog.Info("Uploading file to WebDAV", "source", sourceFilePath, "dest", dest) - resp, err := c.httpClient.Do(req) + source, err := os.Open(sourceFilePath) if err != nil { - return nil, bosherr.WrapErrorf(err, "Getting dav blob %s", path) + return fmt.Errorf("failed to open source file: %w", err) } + defer source.Close() //nolint:errcheck - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("Getting dav blob %s: Wrong response code: %d; body: %s", path, resp.StatusCode, c.readAndTruncateBody(resp)) //nolint:staticcheck - } - - return resp.Body, nil -} - -func (c client) Put(path string, content io.ReadCloser, contentLength int64) error { - req, err := c.createReq("PUT", path, content) + fileInfo, err := source.Stat() if err != nil { - return err + return fmt.Errorf("failed to stat source file: %w", err) } - defer content.Close() //nolint:errcheck - req.ContentLength = contentLength - resp, err := c.httpClient.Do(req) + err = d.storageClient.Put(dest, source, fileInfo.Size()) if err != nil { - return bosherr.WrapErrorf(err, "Putting dav blob %s", path) - } - - if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusNoContent { - return fmt.Errorf("Putting dav blob %s: Wrong response code: %d; body: %s", path, resp.StatusCode, c.readAndTruncateBody(resp)) //nolint:staticcheck + return fmt.Errorf("upload failure: %w", err) } + slog.Info("Successfully uploaded file", "dest", dest) return nil } -func (c client) Exists(path string) error { - req, err := c.createReq("HEAD", path, nil) - if err != nil { - return err - } +// Get downloads a file from the WebDAV server +func (d *DavBlobstore) Get(source string, dest string) error { + slog.Info("Downloading file from WebDAV", "source", source, "dest", dest) - resp, err := c.httpClient.Do(req) + destFile, err := os.Create(dest) if err != nil { - return bosherr.WrapErrorf(err, "Checking if dav blob %s exists", path) + return fmt.Errorf("failed to create destination file: %w", err) } + defer destFile.Close() //nolint:errcheck - if resp.StatusCode == http.StatusNotFound { - err := fmt.Errorf("%s not found", path) - return bosherr.WrapErrorf(err, "Checking if dav blob %s exists", path) + content, err := d.storageClient.Get(source) + if err != nil { + return fmt.Errorf("download failure: %w", err) } + defer content.Close() //nolint:errcheck - if resp.StatusCode != http.StatusOK { - err := fmt.Errorf("invalid status: %d", resp.StatusCode) - return bosherr.WrapErrorf(err, "Checking if dav blob %s exists", path) + _, err = io.Copy(destFile, content) + if err != nil { + return fmt.Errorf("failed to write to destination file: %w", err) } + slog.Info("Successfully downloaded file", "dest", dest) return nil } -func (c client) Delete(path string) error { - req, err := c.createReq("DELETE", path, nil) - if err != nil { - return bosherr.WrapErrorf(err, "Creating delete request for blob '%s'", path) - } +// Delete removes a file from the WebDAV server +func (d *DavBlobstore) Delete(dest string) error { + slog.Info("Deleting file from WebDAV", "dest", dest) + return d.storageClient.Delete(dest) +} + +// DeleteRecursive deletes all files matching a prefix +func (d *DavBlobstore) DeleteRecursive(prefix string) error { + slog.Info("Deleting files recursively from WebDAV", "prefix", prefix) - resp, err := c.httpClient.Do(req) + // List all blobs with the prefix + blobs, err := d.storageClient.List(prefix) if err != nil { - return bosherr.WrapErrorf(err, "Deleting blob '%s'", path) + return fmt.Errorf("failed to list blobs with prefix '%s': %w", prefix, err) } - if resp.StatusCode == http.StatusNotFound { - return nil - } + slog.Info("Found blobs to delete", "count", len(blobs), "prefix", prefix) - if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { - err := fmt.Errorf("invalid status: %d", resp.StatusCode) - return bosherr.WrapErrorf(err, "Deleting blob '%s'", path) + // Delete each blob + for _, blob := range blobs { + if err := d.storageClient.Delete(blob); err != nil { + return fmt.Errorf("failed to delete blob '%s': %w", blob, err) + } + slog.Info("Deleted blob", "blob", blob) } + slog.Info("Successfully deleted all blobs", "prefix", prefix) return nil } -func (c client) Sign(blobID, action string, duration time.Duration) (string, error) { - signer := URLsigner.NewSigner(c.config.Secret) - signTime := time.Now() - - prefixedBlob := fmt.Sprintf("%s/%s", getBlobPrefix(blobID), blobID) +// Exists checks if a file exists on the WebDAV server +func (d *DavBlobstore) Exists(dest string) (bool, error) { + slog.Info("Checking if file exists on WebDAV", "dest", dest) + return d.storageClient.Exists(dest) +} - signedURL, err := signer.GenerateSignedURL(c.config.Endpoint, prefixedBlob, action, signTime, duration) +// Sign generates a pre-signed URL for the blob +func (d *DavBlobstore) Sign(dest string, action string, expiration time.Duration) (string, error) { + slog.Info("Signing URL for WebDAV", "dest", dest, "action", action, "expiration", expiration) + signedURL, err := d.storageClient.Sign(dest, action, expiration) if err != nil { - return "", bosherr.WrapErrorf(err, "pre-signing the url") + return "", fmt.Errorf("failed to sign URL: %w", err) } - return signedURL, err + return signedURL, nil } -func (c client) createReq(method, blobID string, body io.Reader) (*http.Request, error) { - blobURL, err := url.Parse(c.config.Endpoint) +// List returns a list of blob paths that match the given prefix +func (d *DavBlobstore) List(prefix string) ([]string, error) { + slog.Info("Listing files on WebDAV", "prefix", prefix) + + blobs, err := d.storageClient.List(prefix) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to list blobs: %w", err) } - blobPrefix := getBlobPrefix(blobID) - - newPath := path.Join(blobURL.Path, blobPrefix, blobID) - if !strings.HasPrefix(newPath, "/") { - newPath = "/" + newPath - } + slog.Info("Found blobs", "count", len(blobs), "prefix", prefix) + return blobs, nil +} - blobURL.Path = newPath +// Copy copies a blob from source to destination +func (d *DavBlobstore) Copy(srcBlob string, dstBlob string) error { + slog.Info("Copying blob on WebDAV", "source", srcBlob, "dest", dstBlob) - req, err := http.NewRequest(method, blobURL.String(), body) + err := d.storageClient.Copy(srcBlob, dstBlob) if err != nil { - return req, err + return fmt.Errorf("copy failure: %w", err) } - if c.config.User != "" { - req.SetBasicAuth(c.config.User, c.config.Password) - } - return req, nil + slog.Info("Successfully copied blob", "source", srcBlob, "dest", dstBlob) + return nil } -func (c client) readAndTruncateBody(resp *http.Response) string { - body := "" - if resp.Body != nil { - buf := make([]byte, 1024) - n, err := resp.Body.Read(buf) - if err == io.EOF || err == nil { - body = string(buf[0:n]) - } - } - return body +// Properties retrieves metadata for a blob +func (d *DavBlobstore) Properties(dest string) error { + slog.Info("Getting properties for blob on WebDAV", "dest", dest) + return d.storageClient.Properties(dest) } -func getBlobPrefix(blobID string) string { - digester := sha1.New() - digester.Write([]byte(blobID)) - return fmt.Sprintf("%02x", digester.Sum(nil)[0]) +// EnsureStorageExists ensures the WebDAV directory structure exists +func (d *DavBlobstore) EnsureStorageExists() error { + slog.Info("Ensuring WebDAV storage exists") + return d.storageClient.EnsureStorageExists() } diff --git a/dav/client/client_suite_test.go b/dav/client/client_suite_test.go index 95b3f42..3409292 100644 --- a/dav/client/client_suite_test.go +++ b/dav/client/client_suite_test.go @@ -1,10 +1,10 @@ package client_test import ( + "testing" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - - "testing" ) func TestClient(t *testing.T) { diff --git a/dav/client/client_test.go b/dav/client/client_test.go index a26eab8..d53af70 100644 --- a/dav/client/client_test.go +++ b/dav/client/client_test.go @@ -2,297 +2,208 @@ package client_test import ( "io" - "net/http" + "os" "strings" + "time" + + "github.com/cloudfoundry/storage-cli/dav/client" + "github.com/cloudfoundry/storage-cli/dav/client/clientfakes" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/onsi/gomega/ghttp" - - "github.com/cloudfoundry/bosh-utils/httpclient" - boshlog "github.com/cloudfoundry/bosh-utils/logger" - - . "github.com/cloudfoundry/storage-cli/dav/client" - davconf "github.com/cloudfoundry/storage-cli/dav/config" ) var _ = Describe("Client", func() { - var ( - server *ghttp.Server - config davconf.Config - client Client - logger boshlog.Logger - ) - - BeforeEach(func() { - server = ghttp.NewServer() - config.Endpoint = server.URL() - config.User = "some_user" - config.Password = "some password" - logger = boshlog.NewLogger(boshlog.LevelNone) - client = NewClient(config, httpclient.DefaultClient, logger) - }) - disconnectingRequestHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - conn, _, err := w.(http.Hijacker).Hijack() - Expect(err).NotTo(HaveOccurred()) + Context("Put", func() { + It("uploads a file to a blob", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + fakeStorageClient.PutReturns(nil) - conn.Close() //nolint:errcheck - }) + davBlobstore := client.NewWithStorageClient(fakeStorageClient) - Describe("Exists", func() { - It("does not return an error if file exists", func() { - server.AppendHandlers(ghttp.RespondWith(200, "")) - err := client.Exists("/somefile") + file, err := os.CreateTemp("", "tmpfile") Expect(err).NotTo(HaveOccurred()) - }) + defer os.Remove(file.Name()) //nolint:errcheck - Context("the file does not exist", func() { - BeforeEach(func() { - server.AppendHandlers( - ghttp.RespondWith(404, ""), - ghttp.RespondWith(404, ""), - ghttp.RespondWith(404, ""), - ) - }) - - It("returns an error saying blob was not found", func() { - err := client.Exists("/somefile") - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(ContainSubstring("Checking if dav blob /somefile exists: /somefile not found"))) - }) + _, err = file.WriteString("test content") + Expect(err).NotTo(HaveOccurred()) + file.Close() //nolint:errcheck + + err = davBlobstore.Put(file.Name(), "target/blob") + + Expect(err).NotTo(HaveOccurred()) + Expect(fakeStorageClient.PutCallCount()).To(Equal(1)) + path, _, _ := fakeStorageClient.PutArgsForCall(0) + Expect(path).To(Equal("target/blob")) }) - Context("unexpected http status code returned", func() { - BeforeEach(func() { - server.AppendHandlers( - ghttp.RespondWith(601, ""), - ghttp.RespondWith(601, ""), - ghttp.RespondWith(601, ""), - ) - }) - - It("returns an error saying an unexpected error occurred", func() { - err := client.Exists("/somefile") - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(ContainSubstring("Checking if dav blob /somefile exists:"))) - }) + It("fails if the source file does not exist", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + + davBlobstore := client.NewWithStorageClient(fakeStorageClient) + err := davBlobstore.Put("nonexistent/path", "target/blob") + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to open source file")) + Expect(fakeStorageClient.PutCallCount()).To(Equal(0)) }) }) - Describe("Delete", func() { - Context("when the file does not exist", func() { - BeforeEach(func() { - server.AppendHandlers( - ghttp.RespondWith(404, ""), - ghttp.RespondWith(404, ""), - ghttp.RespondWith(404, ""), - ) - }) - - It("does not return an error if file does not exists", func() { - err := client.Delete("/somefile") - Expect(err).NotTo(HaveOccurred()) - }) - }) + Context("Get", func() { + It("downloads a blob to a file", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + content := io.NopCloser(strings.NewReader("test content")) + fakeStorageClient.GetReturns(content, nil) + + davBlobstore := client.NewWithStorageClient(fakeStorageClient) + + tmpFile, err := os.CreateTemp("", "download") + Expect(err).NotTo(HaveOccurred()) + tmpFile.Close() //nolint:errcheck + defer os.Remove(tmpFile.Name()) //nolint:errcheck + + err = davBlobstore.Get("source/blob", tmpFile.Name()) + + Expect(err).NotTo(HaveOccurred()) + Expect(fakeStorageClient.GetCallCount()).To(Equal(1)) - Context("when the file exists", func() { - BeforeEach(func() { - server.AppendHandlers(ghttp.RespondWith(204, "")) - }) - - It("does not return an error", func() { - err := client.Delete("/somefile") - Expect(err).ToNot(HaveOccurred()) - Expect(server.ReceivedRequests()).To(HaveLen(1)) - request := server.ReceivedRequests()[0] - Expect(request.URL.Path).To(Equal("/19/somefile")) - Expect(request.Method).To(Equal("DELETE")) - Expect(request.Header["Authorization"]).To(Equal([]string{"Basic c29tZV91c2VyOnNvbWUgcGFzc3dvcmQ="})) - Expect(request.Host).To(Equal(server.Addr())) - }) + // Verify content was written + downloaded, err := os.ReadFile(tmpFile.Name()) + Expect(err).NotTo(HaveOccurred()) + Expect(string(downloaded)).To(Equal("test content")) }) + }) + + Context("Delete", func() { + It("deletes a blob", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + fakeStorageClient.DeleteReturns(nil) - Context("when the status code is not in the 2xx range", func() { - It("returns an error saying an unexpected error occurred when the status code is greater than 299", func() { - server.AppendHandlers( - ghttp.RespondWith(300, ""), - ghttp.RespondWith(300, ""), - ghttp.RespondWith(300, ""), - ) - - err := client.Delete("/somefile") - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(Equal("Deleting blob '/somefile': invalid status: 300"))) - }) + davBlobstore := client.NewWithStorageClient(fakeStorageClient) + err := davBlobstore.Delete("blob/path") + + Expect(err).NotTo(HaveOccurred()) + Expect(fakeStorageClient.DeleteCallCount()).To(Equal(1)) + Expect(fakeStorageClient.DeleteArgsForCall(0)).To(Equal("blob/path")) }) }) - Describe("Get", func() { - It("returns the response body from the given path", func() { - server.AppendHandlers(ghttp.RespondWith(200, "response")) + Context("DeleteRecursive", func() { + It("lists and deletes all blobs with prefix", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + fakeStorageClient.ListReturns([]string{"blob1", "blob2", "blob3"}, nil) + fakeStorageClient.DeleteReturns(nil) + + davBlobstore := client.NewWithStorageClient(fakeStorageClient) + err := davBlobstore.DeleteRecursive("prefix/") - responseBody, err := client.Get("/") Expect(err).NotTo(HaveOccurred()) - buf := make([]byte, 1024) - n, _ := responseBody.Read(buf) //nolint:errcheck - Expect(string(buf[0:n])).To(Equal("response")) + Expect(fakeStorageClient.ListCallCount()).To(Equal(1)) + Expect(fakeStorageClient.DeleteCallCount()).To(Equal(3)) }) + }) + + Context("Exists", func() { + It("returns true when blob exists", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + fakeStorageClient.ExistsReturns(true, nil) - Context("when the http request fails", func() { - BeforeEach(func() { - server.Close() - }) - - It("returns err", func() { - responseBody, err := client.Get("/") - Expect(responseBody).To(BeNil()) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Getting dav blob /")) - }) + davBlobstore := client.NewWithStorageClient(fakeStorageClient) + exists, err := davBlobstore.Exists("somefile") + + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + Expect(fakeStorageClient.ExistsCallCount()).To(Equal(1)) + Expect(fakeStorageClient.ExistsArgsForCall(0)).To(Equal("somefile")) }) - Context("when the http response code is not 200", func() { - BeforeEach(func() { - server.AppendHandlers( - ghttp.RespondWith(300, "response"), - ghttp.RespondWith(300, "response"), - ghttp.RespondWith(300, "response"), - ) - }) - - It("returns err", func() { - responseBody, err := client.Get("/") - Expect(responseBody).To(BeNil()) - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(ContainSubstring("Getting dav blob /: Wrong response code: 300"))) - Expect(server.ReceivedRequests()).To(HaveLen(3)) - }) + It("returns false when blob does not exist", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + fakeStorageClient.ExistsReturns(false, nil) + + davBlobstore := client.NewWithStorageClient(fakeStorageClient) + exists, err := davBlobstore.Exists("somefile") + + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeFalse()) + Expect(fakeStorageClient.ExistsCallCount()).To(Equal(1)) }) }) - Describe("Put", func() { - Context("When the put request succeeds", func() { - itUploadsABlob := func() { - body := io.NopCloser(strings.NewReader("content")) - err := client.Put("/", body, int64(7)) - Expect(err).NotTo(HaveOccurred()) - - Expect(server.ReceivedRequests()).To(HaveLen(1)) - req := server.ReceivedRequests()[0] - Expect(req.ContentLength).To(Equal(int64(7))) - } - - It("uploads the given content if the blob does not exist", func() { - server.AppendHandlers( - ghttp.CombineHandlers( - ghttp.RespondWith(201, ""), - ghttp.VerifyBody([]byte("content")), - ), - ) - itUploadsABlob() - }) - - It("uploads the given content if the blob exists", func() { - server.AppendHandlers( - ghttp.CombineHandlers( - ghttp.RespondWith(204, ""), - ghttp.VerifyBody([]byte("content")), - ), - ) - itUploadsABlob() - }) - - It("adds an Authorizatin header to the request", func() { - server.AppendHandlers( - ghttp.CombineHandlers( - ghttp.RespondWith(204, ""), - ghttp.VerifyBody([]byte("content")), - ), - ) - itUploadsABlob() - req := server.ReceivedRequests()[0] - Expect(req.Header.Get("Authorization")).NotTo(BeEmpty()) - }) - - Context("when neither user nor password is provided in blobstore options", func() { - BeforeEach(func() { - config.User = "" - config.Password = "" - client = NewClient(config, httpclient.DefaultClient, logger) - }) - - It("sends a request with no Basic Auth header", func() { - server.AppendHandlers( - ghttp.CombineHandlers( - ghttp.RespondWith(204, ""), - ghttp.VerifyBody([]byte("content")), - ), - ) - itUploadsABlob() - req := server.ReceivedRequests()[0] - Expect(req.Header.Get("Authorization")).To(BeEmpty()) - }) - }) + Context("List", func() { + It("returns list of blobs", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + fakeStorageClient.ListReturns([]string{"blob1.txt", "blob2.txt"}, nil) + + davBlobstore := client.NewWithStorageClient(fakeStorageClient) + blobs, err := davBlobstore.List("prefix/") + + Expect(err).NotTo(HaveOccurred()) + Expect(blobs).To(Equal([]string{"blob1.txt", "blob2.txt"})) + Expect(fakeStorageClient.ListCallCount()).To(Equal(1)) + Expect(fakeStorageClient.ListArgsForCall(0)).To(Equal("prefix/")) }) + }) + + Context("Copy", func() { + It("copies a blob from source to destination", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + fakeStorageClient.CopyReturns(nil) + + davBlobstore := client.NewWithStorageClient(fakeStorageClient) + err := davBlobstore.Copy("source/blob", "dest/blob") - Context("when the http request fails", func() { - BeforeEach(func() { - server.AppendHandlers( - disconnectingRequestHandler, - disconnectingRequestHandler, - disconnectingRequestHandler, - ) - }) - - It("returns err", func() { - body := io.NopCloser(strings.NewReader("content")) - err := client.Put("/", body, int64(7)) - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(ContainSubstring("Putting dav blob /: Put \"%s/42\": EOF", server.URL()))) - Expect(server.ReceivedRequests()).To(HaveLen(3)) - }) + Expect(err).NotTo(HaveOccurred()) + Expect(fakeStorageClient.CopyCallCount()).To(Equal(1)) + src, dst := fakeStorageClient.CopyArgsForCall(0) + Expect(src).To(Equal("source/blob")) + Expect(dst).To(Equal("dest/blob")) }) + }) + + Context("Sign", func() { + It("generates a signed URL", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + fakeStorageClient.SignReturns("https://signed-url.com", nil) + + davBlobstore := client.NewWithStorageClient(fakeStorageClient) + signedURL, err := davBlobstore.Sign("blob/path", "get", 1*time.Hour) - Context("when the http response code is not 201 or 204", func() { - BeforeEach(func() { - server.AppendHandlers( - ghttp.RespondWith(300, "response"), - ghttp.RespondWith(300, "response"), - ghttp.RespondWith(300, "response"), - ) - }) - - It("returns err", func() { - body := io.NopCloser(strings.NewReader("content")) - err := client.Put("/", body, int64(7)) - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(ContainSubstring("Putting dav blob /: Wrong response code: 300"))) - }) + Expect(err).NotTo(HaveOccurred()) + Expect(signedURL).To(Equal("https://signed-url.com")) + Expect(fakeStorageClient.SignCallCount()).To(Equal(1)) + path, action, duration := fakeStorageClient.SignArgsForCall(0) + Expect(path).To(Equal("blob/path")) + Expect(action).To(Equal("get")) + Expect(duration).To(Equal(1 * time.Hour)) }) }) - Describe("retryable count is configurable", func() { - BeforeEach(func() { - server.AppendHandlers( - disconnectingRequestHandler, - disconnectingRequestHandler, - disconnectingRequestHandler, - disconnectingRequestHandler, - disconnectingRequestHandler, - disconnectingRequestHandler, - disconnectingRequestHandler, - ) - config = davconf.Config{RetryAttempts: 7, Endpoint: server.URL()} - client = NewClient(config, httpclient.DefaultClient, logger) + Context("Properties", func() { + It("retrieves blob properties", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + fakeStorageClient.PropertiesReturns(nil) + + davBlobstore := client.NewWithStorageClient(fakeStorageClient) + err := davBlobstore.Properties("blob/path") + + Expect(err).NotTo(HaveOccurred()) + Expect(fakeStorageClient.PropertiesCallCount()).To(Equal(1)) + Expect(fakeStorageClient.PropertiesArgsForCall(0)).To(Equal("blob/path")) }) + }) - It("tries the specified number of times", func() { - body := io.NopCloser(strings.NewReader("content")) - err := client.Put("/", body, int64(7)) - Expect(err).To(HaveOccurred()) - Expect(err).To(MatchError(ContainSubstring("Putting dav blob /: Put \"%s/42\": EOF", server.URL()))) - Expect(server.ReceivedRequests()).To(HaveLen(7)) + Context("EnsureStorageExists", func() { + It("ensures storage is initialized", func() { + fakeStorageClient := &clientfakes.FakeStorageClient{} + fakeStorageClient.EnsureStorageExistsReturns(nil) + + davBlobstore := client.NewWithStorageClient(fakeStorageClient) + err := davBlobstore.EnsureStorageExists() + + Expect(err).NotTo(HaveOccurred()) + Expect(fakeStorageClient.EnsureStorageExistsCallCount()).To(Equal(1)) }) }) }) diff --git a/dav/client/clientfakes/fake_storage_client.go b/dav/client/clientfakes/fake_storage_client.go new file mode 100644 index 0000000..e360ebf --- /dev/null +++ b/dav/client/clientfakes/fake_storage_client.go @@ -0,0 +1,708 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package clientfakes + +import ( + "io" + "sync" + "time" + + "github.com/cloudfoundry/storage-cli/dav/client" +) + +type FakeStorageClient struct { + CopyStub func(string, string) error + copyMutex sync.RWMutex + copyArgsForCall []struct { + arg1 string + arg2 string + } + copyReturns struct { + result1 error + } + copyReturnsOnCall map[int]struct { + result1 error + } + DeleteStub func(string) error + deleteMutex sync.RWMutex + deleteArgsForCall []struct { + arg1 string + } + deleteReturns struct { + result1 error + } + deleteReturnsOnCall map[int]struct { + result1 error + } + EnsureStorageExistsStub func() error + ensureStorageExistsMutex sync.RWMutex + ensureStorageExistsArgsForCall []struct { + } + ensureStorageExistsReturns struct { + result1 error + } + ensureStorageExistsReturnsOnCall map[int]struct { + result1 error + } + ExistsStub func(string) (bool, error) + existsMutex sync.RWMutex + existsArgsForCall []struct { + arg1 string + } + existsReturns struct { + result1 bool + result2 error + } + existsReturnsOnCall map[int]struct { + result1 bool + result2 error + } + GetStub func(string) (io.ReadCloser, error) + getMutex sync.RWMutex + getArgsForCall []struct { + arg1 string + } + getReturns struct { + result1 io.ReadCloser + result2 error + } + getReturnsOnCall map[int]struct { + result1 io.ReadCloser + result2 error + } + ListStub func(string) ([]string, error) + listMutex sync.RWMutex + listArgsForCall []struct { + arg1 string + } + listReturns struct { + result1 []string + result2 error + } + listReturnsOnCall map[int]struct { + result1 []string + result2 error + } + PropertiesStub func(string) error + propertiesMutex sync.RWMutex + propertiesArgsForCall []struct { + arg1 string + } + propertiesReturns struct { + result1 error + } + propertiesReturnsOnCall map[int]struct { + result1 error + } + PutStub func(string, io.ReadCloser, int64) error + putMutex sync.RWMutex + putArgsForCall []struct { + arg1 string + arg2 io.ReadCloser + arg3 int64 + } + putReturns struct { + result1 error + } + putReturnsOnCall map[int]struct { + result1 error + } + SignStub func(string, string, time.Duration) (string, error) + signMutex sync.RWMutex + signArgsForCall []struct { + arg1 string + arg2 string + arg3 time.Duration + } + signReturns struct { + result1 string + result2 error + } + signReturnsOnCall map[int]struct { + result1 string + result2 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeStorageClient) Copy(arg1 string, arg2 string) error { + fake.copyMutex.Lock() + ret, specificReturn := fake.copyReturnsOnCall[len(fake.copyArgsForCall)] + fake.copyArgsForCall = append(fake.copyArgsForCall, struct { + arg1 string + arg2 string + }{arg1, arg2}) + stub := fake.CopyStub + fakeReturns := fake.copyReturns + fake.recordInvocation("Copy", []interface{}{arg1, arg2}) + fake.copyMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) CopyCallCount() int { + fake.copyMutex.RLock() + defer fake.copyMutex.RUnlock() + return len(fake.copyArgsForCall) +} + +func (fake *FakeStorageClient) CopyCalls(stub func(string, string) error) { + fake.copyMutex.Lock() + defer fake.copyMutex.Unlock() + fake.CopyStub = stub +} + +func (fake *FakeStorageClient) CopyArgsForCall(i int) (string, string) { + fake.copyMutex.RLock() + defer fake.copyMutex.RUnlock() + argsForCall := fake.copyArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStorageClient) CopyReturns(result1 error) { + fake.copyMutex.Lock() + defer fake.copyMutex.Unlock() + fake.CopyStub = nil + fake.copyReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) CopyReturnsOnCall(i int, result1 error) { + fake.copyMutex.Lock() + defer fake.copyMutex.Unlock() + fake.CopyStub = nil + if fake.copyReturnsOnCall == nil { + fake.copyReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.copyReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) Delete(arg1 string) error { + fake.deleteMutex.Lock() + ret, specificReturn := fake.deleteReturnsOnCall[len(fake.deleteArgsForCall)] + fake.deleteArgsForCall = append(fake.deleteArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.DeleteStub + fakeReturns := fake.deleteReturns + fake.recordInvocation("Delete", []interface{}{arg1}) + fake.deleteMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) DeleteCallCount() int { + fake.deleteMutex.RLock() + defer fake.deleteMutex.RUnlock() + return len(fake.deleteArgsForCall) +} + +func (fake *FakeStorageClient) DeleteCalls(stub func(string) error) { + fake.deleteMutex.Lock() + defer fake.deleteMutex.Unlock() + fake.DeleteStub = stub +} + +func (fake *FakeStorageClient) DeleteArgsForCall(i int) string { + fake.deleteMutex.RLock() + defer fake.deleteMutex.RUnlock() + argsForCall := fake.deleteArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) DeleteReturns(result1 error) { + fake.deleteMutex.Lock() + defer fake.deleteMutex.Unlock() + fake.DeleteStub = nil + fake.deleteReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) DeleteReturnsOnCall(i int, result1 error) { + fake.deleteMutex.Lock() + defer fake.deleteMutex.Unlock() + fake.DeleteStub = nil + if fake.deleteReturnsOnCall == nil { + fake.deleteReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.deleteReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) EnsureStorageExists() error { + fake.ensureStorageExistsMutex.Lock() + ret, specificReturn := fake.ensureStorageExistsReturnsOnCall[len(fake.ensureStorageExistsArgsForCall)] + fake.ensureStorageExistsArgsForCall = append(fake.ensureStorageExistsArgsForCall, struct { + }{}) + stub := fake.EnsureStorageExistsStub + fakeReturns := fake.ensureStorageExistsReturns + fake.recordInvocation("EnsureStorageExists", []interface{}{}) + fake.ensureStorageExistsMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) EnsureStorageExistsCallCount() int { + fake.ensureStorageExistsMutex.RLock() + defer fake.ensureStorageExistsMutex.RUnlock() + return len(fake.ensureStorageExistsArgsForCall) +} + +func (fake *FakeStorageClient) EnsureStorageExistsCalls(stub func() error) { + fake.ensureStorageExistsMutex.Lock() + defer fake.ensureStorageExistsMutex.Unlock() + fake.EnsureStorageExistsStub = stub +} + +func (fake *FakeStorageClient) EnsureStorageExistsReturns(result1 error) { + fake.ensureStorageExistsMutex.Lock() + defer fake.ensureStorageExistsMutex.Unlock() + fake.EnsureStorageExistsStub = nil + fake.ensureStorageExistsReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) EnsureStorageExistsReturnsOnCall(i int, result1 error) { + fake.ensureStorageExistsMutex.Lock() + defer fake.ensureStorageExistsMutex.Unlock() + fake.EnsureStorageExistsStub = nil + if fake.ensureStorageExistsReturnsOnCall == nil { + fake.ensureStorageExistsReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.ensureStorageExistsReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) Exists(arg1 string) (bool, error) { + fake.existsMutex.Lock() + ret, specificReturn := fake.existsReturnsOnCall[len(fake.existsArgsForCall)] + fake.existsArgsForCall = append(fake.existsArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.ExistsStub + fakeReturns := fake.existsReturns + fake.recordInvocation("Exists", []interface{}{arg1}) + fake.existsMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) ExistsCallCount() int { + fake.existsMutex.RLock() + defer fake.existsMutex.RUnlock() + return len(fake.existsArgsForCall) +} + +func (fake *FakeStorageClient) ExistsCalls(stub func(string) (bool, error)) { + fake.existsMutex.Lock() + defer fake.existsMutex.Unlock() + fake.ExistsStub = stub +} + +func (fake *FakeStorageClient) ExistsArgsForCall(i int) string { + fake.existsMutex.RLock() + defer fake.existsMutex.RUnlock() + argsForCall := fake.existsArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) ExistsReturns(result1 bool, result2 error) { + fake.existsMutex.Lock() + defer fake.existsMutex.Unlock() + fake.ExistsStub = nil + fake.existsReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) ExistsReturnsOnCall(i int, result1 bool, result2 error) { + fake.existsMutex.Lock() + defer fake.existsMutex.Unlock() + fake.ExistsStub = nil + if fake.existsReturnsOnCall == nil { + fake.existsReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.existsReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) Get(arg1 string) (io.ReadCloser, error) { + fake.getMutex.Lock() + ret, specificReturn := fake.getReturnsOnCall[len(fake.getArgsForCall)] + fake.getArgsForCall = append(fake.getArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.GetStub + fakeReturns := fake.getReturns + fake.recordInvocation("Get", []interface{}{arg1}) + fake.getMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) GetCallCount() int { + fake.getMutex.RLock() + defer fake.getMutex.RUnlock() + return len(fake.getArgsForCall) +} + +func (fake *FakeStorageClient) GetCalls(stub func(string) (io.ReadCloser, error)) { + fake.getMutex.Lock() + defer fake.getMutex.Unlock() + fake.GetStub = stub +} + +func (fake *FakeStorageClient) GetArgsForCall(i int) string { + fake.getMutex.RLock() + defer fake.getMutex.RUnlock() + argsForCall := fake.getArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) GetReturns(result1 io.ReadCloser, result2 error) { + fake.getMutex.Lock() + defer fake.getMutex.Unlock() + fake.GetStub = nil + fake.getReturns = struct { + result1 io.ReadCloser + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) GetReturnsOnCall(i int, result1 io.ReadCloser, result2 error) { + fake.getMutex.Lock() + defer fake.getMutex.Unlock() + fake.GetStub = nil + if fake.getReturnsOnCall == nil { + fake.getReturnsOnCall = make(map[int]struct { + result1 io.ReadCloser + result2 error + }) + } + fake.getReturnsOnCall[i] = struct { + result1 io.ReadCloser + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) List(arg1 string) ([]string, error) { + fake.listMutex.Lock() + ret, specificReturn := fake.listReturnsOnCall[len(fake.listArgsForCall)] + fake.listArgsForCall = append(fake.listArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.ListStub + fakeReturns := fake.listReturns + fake.recordInvocation("List", []interface{}{arg1}) + fake.listMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) ListCallCount() int { + fake.listMutex.RLock() + defer fake.listMutex.RUnlock() + return len(fake.listArgsForCall) +} + +func (fake *FakeStorageClient) ListCalls(stub func(string) ([]string, error)) { + fake.listMutex.Lock() + defer fake.listMutex.Unlock() + fake.ListStub = stub +} + +func (fake *FakeStorageClient) ListArgsForCall(i int) string { + fake.listMutex.RLock() + defer fake.listMutex.RUnlock() + argsForCall := fake.listArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) ListReturns(result1 []string, result2 error) { + fake.listMutex.Lock() + defer fake.listMutex.Unlock() + fake.ListStub = nil + fake.listReturns = struct { + result1 []string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) ListReturnsOnCall(i int, result1 []string, result2 error) { + fake.listMutex.Lock() + defer fake.listMutex.Unlock() + fake.ListStub = nil + if fake.listReturnsOnCall == nil { + fake.listReturnsOnCall = make(map[int]struct { + result1 []string + result2 error + }) + } + fake.listReturnsOnCall[i] = struct { + result1 []string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) Properties(arg1 string) error { + fake.propertiesMutex.Lock() + ret, specificReturn := fake.propertiesReturnsOnCall[len(fake.propertiesArgsForCall)] + fake.propertiesArgsForCall = append(fake.propertiesArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.PropertiesStub + fakeReturns := fake.propertiesReturns + fake.recordInvocation("Properties", []interface{}{arg1}) + fake.propertiesMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) PropertiesCallCount() int { + fake.propertiesMutex.RLock() + defer fake.propertiesMutex.RUnlock() + return len(fake.propertiesArgsForCall) +} + +func (fake *FakeStorageClient) PropertiesCalls(stub func(string) error) { + fake.propertiesMutex.Lock() + defer fake.propertiesMutex.Unlock() + fake.PropertiesStub = stub +} + +func (fake *FakeStorageClient) PropertiesArgsForCall(i int) string { + fake.propertiesMutex.RLock() + defer fake.propertiesMutex.RUnlock() + argsForCall := fake.propertiesArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) PropertiesReturns(result1 error) { + fake.propertiesMutex.Lock() + defer fake.propertiesMutex.Unlock() + fake.PropertiesStub = nil + fake.propertiesReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) PropertiesReturnsOnCall(i int, result1 error) { + fake.propertiesMutex.Lock() + defer fake.propertiesMutex.Unlock() + fake.PropertiesStub = nil + if fake.propertiesReturnsOnCall == nil { + fake.propertiesReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.propertiesReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) Put(arg1 string, arg2 io.ReadCloser, arg3 int64) error { + fake.putMutex.Lock() + ret, specificReturn := fake.putReturnsOnCall[len(fake.putArgsForCall)] + fake.putArgsForCall = append(fake.putArgsForCall, struct { + arg1 string + arg2 io.ReadCloser + arg3 int64 + }{arg1, arg2, arg3}) + stub := fake.PutStub + fakeReturns := fake.putReturns + fake.recordInvocation("Put", []interface{}{arg1, arg2, arg3}) + fake.putMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) PutCallCount() int { + fake.putMutex.RLock() + defer fake.putMutex.RUnlock() + return len(fake.putArgsForCall) +} + +func (fake *FakeStorageClient) PutCalls(stub func(string, io.ReadCloser, int64) error) { + fake.putMutex.Lock() + defer fake.putMutex.Unlock() + fake.PutStub = stub +} + +func (fake *FakeStorageClient) PutArgsForCall(i int) (string, io.ReadCloser, int64) { + fake.putMutex.RLock() + defer fake.putMutex.RUnlock() + argsForCall := fake.putArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeStorageClient) PutReturns(result1 error) { + fake.putMutex.Lock() + defer fake.putMutex.Unlock() + fake.PutStub = nil + fake.putReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) PutReturnsOnCall(i int, result1 error) { + fake.putMutex.Lock() + defer fake.putMutex.Unlock() + fake.PutStub = nil + if fake.putReturnsOnCall == nil { + fake.putReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.putReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) Sign(arg1 string, arg2 string, arg3 time.Duration) (string, error) { + fake.signMutex.Lock() + ret, specificReturn := fake.signReturnsOnCall[len(fake.signArgsForCall)] + fake.signArgsForCall = append(fake.signArgsForCall, struct { + arg1 string + arg2 string + arg3 time.Duration + }{arg1, arg2, arg3}) + stub := fake.SignStub + fakeReturns := fake.signReturns + fake.recordInvocation("Sign", []interface{}{arg1, arg2, arg3}) + fake.signMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) SignCallCount() int { + fake.signMutex.RLock() + defer fake.signMutex.RUnlock() + return len(fake.signArgsForCall) +} + +func (fake *FakeStorageClient) SignCalls(stub func(string, string, time.Duration) (string, error)) { + fake.signMutex.Lock() + defer fake.signMutex.Unlock() + fake.SignStub = stub +} + +func (fake *FakeStorageClient) SignArgsForCall(i int) (string, string, time.Duration) { + fake.signMutex.RLock() + defer fake.signMutex.RUnlock() + argsForCall := fake.signArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeStorageClient) SignReturns(result1 string, result2 error) { + fake.signMutex.Lock() + defer fake.signMutex.Unlock() + fake.SignStub = nil + fake.signReturns = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) SignReturnsOnCall(i int, result1 string, result2 error) { + fake.signMutex.Lock() + defer fake.signMutex.Unlock() + fake.SignStub = nil + if fake.signReturnsOnCall == nil { + fake.signReturnsOnCall = make(map[int]struct { + result1 string + result2 error + }) + } + fake.signReturnsOnCall[i] = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeStorageClient) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ client.StorageClient = new(FakeStorageClient) diff --git a/dav/client/fakes/fake_client.go b/dav/client/fakes/fake_client.go deleted file mode 100644 index 9627637..0000000 --- a/dav/client/fakes/fake_client.go +++ /dev/null @@ -1,37 +0,0 @@ -package fakes - -import ( - "io" -) - -type FakeClient struct { - GetPath string - GetContents io.ReadCloser - GetErr error - - PutPath string - PutContents string - PutContentLength int64 - PutErr error -} - -func NewFakeClient() *FakeClient { - return &FakeClient{} -} - -func (c *FakeClient) Get(path string) (io.ReadCloser, error) { - c.GetPath = path - - return c.GetContents, c.GetErr -} - -func (c *FakeClient) Put(path string, content io.ReadCloser, contentLength int64) error { - c.PutPath = path - contentBytes := make([]byte, contentLength) - content.Read(contentBytes) //nolint:errcheck - defer content.Close() //nolint:errcheck - c.PutContents = string(contentBytes) - c.PutContentLength = contentLength - - return c.PutErr -} diff --git a/dav/client/helpers.go b/dav/client/helpers.go new file mode 100644 index 0000000..e090cc9 --- /dev/null +++ b/dav/client/helpers.go @@ -0,0 +1,120 @@ +package client + +import ( + "crypto/x509" + "errors" + "fmt" + "net/http" + "strings" + + boshcrypto "github.com/cloudfoundry/bosh-utils/crypto" + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +// getCertPool creates a certificate pool from the config +func getCertPool(config davconf.Config) (*x509.CertPool, error) { + if config.TLS.Cert.CA == "" { + return nil, nil + } + + certPool, err := boshcrypto.CertPoolFromPEM([]byte(config.TLS.Cert.CA)) + if err != nil { + return nil, err + } + + return certPool, nil +} + +// validateBlobID ensures blob IDs are valid and safe to use in path construction +func validateBlobID(blobID string) error { + if blobID == "" { + return fmt.Errorf("blob ID cannot be empty") + } + + // Reject leading or trailing slashes + if strings.HasPrefix(blobID, "/") || strings.HasSuffix(blobID, "/") { + return fmt.Errorf("blob ID cannot start or end with slash: %q", blobID) + } + + // Reject consecutive slashes (empty path segments) + if strings.Contains(blobID, "//") { + return fmt.Errorf("blob ID cannot contain empty path segments (//): %q", blobID) + } + + // Split into path segments and reject traversal-only segments + segments := strings.Split(blobID, "/") + for _, segment := range segments { + if segment == "." || segment == ".." { + return fmt.Errorf("blob ID cannot contain path traversal segments (. or ..): %q", blobID) + } + } + + // Reject control characters + for _, r := range blobID { + if r < 32 || r == 127 { + return fmt.Errorf("blob ID cannot contain control characters: %q", blobID) + } + } + + return nil +} + +// validatePrefix ensures list prefixes are safe (more lenient than validateBlobID) +// Allows trailing slashes for directory-style prefixes (e.g., "foo/") +func validatePrefix(prefix string) error { + if prefix == "" { + return fmt.Errorf("prefix cannot be empty") + } + + // Reject leading slash (but trailing slash is OK for prefixes) + if strings.HasPrefix(prefix, "/") { + return fmt.Errorf("prefix cannot start with slash: %q", prefix) + } + + // Reject consecutive slashes (empty path segments) + if strings.Contains(prefix, "//") { + return fmt.Errorf("prefix cannot contain empty path segments (//): %q", prefix) + } + + // Trim trailing slash for segment validation (but allow it in the original prefix) + prefixForValidation := strings.TrimSuffix(prefix, "/") + + // Split into path segments and reject traversal-only segments + segments := strings.Split(prefixForValidation, "/") + for _, segment := range segments { + if segment == "." || segment == ".." { + return fmt.Errorf("prefix cannot contain path traversal segments (. or ..): %q", prefix) + } + } + + // Reject control characters + for _, r := range prefix { + if r < 32 || r == 127 { + return fmt.Errorf("prefix cannot contain control characters: %q", prefix) + } + } + + return nil +} + +// shouldFallbackToCopyManual checks if we should fallback to GET+PUT for copy +// Only fallback for COPY-not-supported cases, not for auth/permission/not-found +func shouldFallbackToCopyManual(err error) bool { + var httpErr *davHTTPError + if errors.As(err, &httpErr) { + // 501 Not Implemented - server doesn't support COPY + // 405 Method Not Allowed - COPY disabled for this resource + return httpErr.StatusCode == http.StatusNotImplemented || httpErr.StatusCode == http.StatusMethodNotAllowed + } + return false +} + +// isMissingParentError checks if the error indicates missing parent collections +func isMissingParentError(err error) bool { + var httpErr *davHTTPError + if errors.As(err, &httpErr) { + // WebDAV uses 409 Conflict for missing intermediate collections + return httpErr.StatusCode == http.StatusConflict + } + return false +} diff --git a/dav/client/storage_client.go b/dav/client/storage_client.go new file mode 100644 index 0000000..5d1dde4 --- /dev/null +++ b/dav/client/storage_client.go @@ -0,0 +1,888 @@ +package client + +import ( + "encoding/json" + "encoding/xml" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/cloudfoundry/bosh-utils/httpclient" + davconf "github.com/cloudfoundry/storage-cli/dav/config" + URLsigner "github.com/cloudfoundry/storage-cli/dav/signer" +) + +// propfindRequest represents a WebDAV PROPFIND request body +type propfindRequest struct { + XMLName xml.Name `xml:"D:propfind"` + DAVNS string `xml:"xmlns:D,attr"` + Prop propfindReqProp `xml:"D:prop"` +} + +type propfindReqProp struct { + ResourceType struct{} `xml:"D:resourcetype"` +} + +// propfindBodyXML is the static PROPFIND request body, generated once at package initialization +var propfindBodyXML = func() string { + reqBody := propfindRequest{DAVNS: "DAV:"} + out, err := xml.MarshalIndent(reqBody, "", " ") + if err != nil { + // This should never happen with a static struct, but if it does, panic at init time + panic(fmt.Sprintf("failed to marshal PROPFIND request body: %v", err)) + } + return xml.Header + string(out) +}() + +// newPropfindBody returns a reader for the PROPFIND request body +func newPropfindBody() *strings.Reader { + return strings.NewReader(propfindBodyXML) +} + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . StorageClient + +// StorageClient handles low-level HTTP operations for WebDAV +type StorageClient interface { + Get(path string) (content io.ReadCloser, err error) + Put(path string, content io.ReadCloser, contentLength int64) (err error) + Exists(path string) (bool, error) + Delete(path string) (err error) + Sign(objectID, action string, duration time.Duration) (string, error) + Copy(srcBlob, dstBlob string) error + List(prefix string) ([]string, error) + Properties(path string) error + EnsureStorageExists() error +} + +// WebDAV XML response structures for PROPFIND +type multistatusResponse struct { + XMLName xml.Name `xml:"multistatus"` + Responses []davResponse `xml:"response"` +} + +type davResponse struct { + Href string `xml:"href"` + PropStats []davPropStat `xml:"propstat"` +} + +type davPropStat struct { + Prop davProp `xml:"prop"` +} + +type davProp struct { + ResourceType davResourceType `xml:"resourcetype"` +} + +type davResourceType struct { + Collection *struct{} `xml:"collection"` +} + +func (r davResponse) isCollection() bool { + for _, ps := range r.PropStats { + if ps.Prop.ResourceType.Collection != nil { + return true + } + } + return false +} + +// davHTTPError wraps an HTTP status code with context for better error handling +type davHTTPError struct { + Operation string // e.g., "COPY", "MKCOL", "PROPFIND" + StatusCode int + Body string +} + +func (e *davHTTPError) Error() string { + if e.Body != "" { + return fmt.Sprintf("%s request failed: status %d, body: %s", e.Operation, e.StatusCode, e.Body) + } + return fmt.Sprintf("%s request failed: status %d", e.Operation, e.StatusCode) +} + +// BlobProperties represents metadata for a blob +type BlobProperties struct { + ETag string `json:"etag,omitempty"` + LastModified time.Time `json:"last_modified,omitempty"` + ContentLength int64 `json:"content_length,omitempty"` +} + +type storageClient struct { + config davconf.Config + httpClient httpclient.Client + signer URLsigner.Signer +} + +// NewStorageClient creates a new HTTP client for WebDAV operations +func NewStorageClient(config davconf.Config, httpClientBase httpclient.Client) StorageClient { + var urlSigner URLsigner.Signer + if config.Secret != "" { + if config.SigningMethod != "" { + urlSigner = URLsigner.NewSignerWithMethod(config.Secret, config.SigningMethod) + } else { + urlSigner = URLsigner.NewSigner(config.Secret) + } + } + + return &storageClient{ + config: config, + httpClient: httpClientBase, + signer: urlSigner, + } +} + +func (c *storageClient) Get(path string) (io.ReadCloser, error) { + if err := validateBlobID(path); err != nil { + return nil, err + } + + req, err := c.createReq("GET", path, nil) + if err != nil { + return nil, err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("getting dav blob %q: %w", path, err) + } + + if resp.StatusCode != http.StatusOK { + defer resp.Body.Close() //nolint:errcheck + return nil, fmt.Errorf("getting dav blob %q: wrong response code: %d; body: %s", path, resp.StatusCode, c.readAndTruncateBody(resp)) + } + + return resp.Body, nil +} + +func (c *storageClient) Put(path string, content io.ReadCloser, contentLength int64) error { + if err := validateBlobID(path); err != nil { + return err + } + + // Ensure all parent collections exist when NOT using signed URLs. + // The caller provides the final blob path with any necessary directory structure. + // When using signed URLs, skip this step because MKCOL operations are not supported + // with nginx signed URLs - the nginx blobstore handles directory creation automatically. + if c.signer == nil { + if err := c.ensureObjectParentsExist(path); err != nil { + return fmt.Errorf("ensuring parent directories exist for blob %q: %w", path, err) + } + } + + req, err := c.createReq("PUT", path, content) + if err != nil { + return err + } + defer content.Close() //nolint:errcheck + + req.ContentLength = contentLength + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("putting dav blob %q: %w", path, err) + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("putting dav blob %q: wrong response code: %d; body: %s", path, resp.StatusCode, c.readAndTruncateBody(resp)) + } + + return nil +} + +func (c *storageClient) Exists(path string) (bool, error) { + if err := validateBlobID(path); err != nil { + return false, err + } + + req, err := c.createReq("HEAD", path, nil) + if err != nil { + return false, err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return false, fmt.Errorf("checking if dav blob %s exists: %w", path, err) + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode == http.StatusNotFound { + return false, nil + } + + if resp.StatusCode != http.StatusOK { + return false, fmt.Errorf("checking if dav blob %s exists: invalid status: %d", path, resp.StatusCode) + } + + return true, nil +} + +func (c *storageClient) Delete(path string) error { + if err := validateBlobID(path); err != nil { + return err + } + + req, err := c.createReq("DELETE", path, nil) + if err != nil { + return fmt.Errorf("creating delete request for blob %q: %w", path, err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("deleting blob %q: %w", path, err) + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode == http.StatusNotFound { + return nil + } + + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + return fmt.Errorf("deleting blob %q: invalid status %d", path, resp.StatusCode) + } + + return nil +} + +func (c *storageClient) Sign(blobID, action string, duration time.Duration) (string, error) { + if err := validateBlobID(blobID); err != nil { + return "", err + } + + // Normalize and validate action + action = strings.ToUpper(action) + if action != "GET" && action != "PUT" { + return "", fmt.Errorf("action not implemented: %s (only GET and PUT are supported)", action) + } + + if c.signer == nil { + return "", fmt.Errorf("signing is not configured (no secret provided)") + } + + signTime := time.Now() + signedURL, err := c.signer.GenerateSignedURL(c.config.Endpoint, blobID, action, signTime, duration) + if err != nil { + return "", fmt.Errorf("pre-signing the url: %w", err) + } + + return signedURL, nil +} + +// Copy copies a blob from source to destination using native WebDAV COPY method +func (c *storageClient) Copy(srcBlob, dstBlob string) error { + if err := validateBlobID(srcBlob); err != nil { + return fmt.Errorf("invalid source blob ID: %w", err) + } + if err := validateBlobID(dstBlob); err != nil { + return fmt.Errorf("invalid destination blob ID: %w", err) + } + + // Try native WebDAV COPY first + err := c.copyNative(srcBlob, dstBlob) + if err == nil { + return nil + } + + // If COPY failed, check if it's due to missing parent collections (409 Conflict) + if isMissingParentError(err) { + slog.Info("Native WebDAV COPY failed due to missing parents, creating them", "error", err.Error()) + + // Create parent collections for destination (skip for signed URLs - nginx handles it) + if c.signer == nil { + if err := c.ensureObjectParentsExist(dstBlob); err != nil { + return fmt.Errorf("ensuring parent directories exist for destination blob %q: %w", dstBlob, err) + } + } + + // Retry native COPY after creating parents + err = c.copyNative(srcBlob, dstBlob) + if err == nil { + return nil + } + } + + // Check if we should fallback to GET+PUT or return the error + // Only fallback for specific "COPY not supported" cases: + // - 501 Not Implemented (server doesn't support COPY) + // - 405 Method Not Allowed (COPY disabled for this resource) + if shouldFallbackToCopyManual(err) { + slog.Info("Native WebDAV COPY not supported, falling back to GET+PUT", "error", err.Error()) + + // Ensure parents exist for fallback PUT (skip for signed URLs - nginx handles it) + if c.signer == nil { + if err := c.ensureObjectParentsExist(dstBlob); err != nil { + return fmt.Errorf("ensuring parent directories exist for destination blob %q: %w", dstBlob, err) + } + } + + return c.copyFallback(srcBlob, dstBlob) + } + + // For other errors (auth, permission, source not found, etc.), return the native COPY error + return fmt.Errorf("native WebDAV COPY failed: %w", err) +} + +// copyNative performs a native WebDAV COPY operation +func (c *storageClient) copyNative(srcBlob, dstBlob string) error { + // Build source URL + srcURL, err := c.buildBlobURL(srcBlob) + if err != nil { + return fmt.Errorf("building source URL: %w", err) + } + + // Build destination URL for the Destination header + dstURL, err := c.buildBlobURL(dstBlob) + if err != nil { + return fmt.Errorf("building destination URL: %w", err) + } + + // Create COPY request to source URL + req, err := http.NewRequest("COPY", srcURL, nil) + if err != nil { + return fmt.Errorf("creating COPY request: %w", err) + } + + if c.config.User != "" { + req.SetBasicAuth(c.config.User, c.config.Password) + } + + // Set WebDAV COPY headers + req.Header.Set("Destination", dstURL) + req.Header.Set("Overwrite", "T") // Allow overwriting existing destination + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("performing COPY request: %w", err) + } + defer resp.Body.Close() //nolint:errcheck + + // Per RFC 4918 section 9.8, standard COPY success responses: + // 201 Created - destination resource was created + // 204 No Content - destination resource was overwritten + if resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusNoContent { + return nil + } + + // Read response body for error details + bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) //nolint:errcheck + bodyPreview := string(bodyBytes) + if len(bodyPreview) > 200 { + bodyPreview = bodyPreview[:200] + "..." + } + + // Return typed error with status code for proper error handling + return &davHTTPError{ + Operation: "COPY", + StatusCode: resp.StatusCode, + Body: bodyPreview, + } +} + +// copyFallback performs copy using GET + PUT as fallback +func (c *storageClient) copyFallback(srcBlob, dstBlob string) error { + srcReq, err := c.createReq("GET", srcBlob, nil) + if err != nil { + return fmt.Errorf("creating request for source blob %q: %w", srcBlob, err) + } + + // Get the source blob content + srcResp, err := c.httpClient.Do(srcReq) + if err != nil { + return fmt.Errorf("getting source blob %q: %w", srcBlob, err) + } + defer srcResp.Body.Close() //nolint:errcheck + + if srcResp.StatusCode != http.StatusOK { + return fmt.Errorf("getting source blob %q: wrong response code: %d; body: %s", srcBlob, srcResp.StatusCode, c.readAndTruncateBody(srcResp)) + } + + // Put the content to destination + dstReq, err := c.createReq("PUT", dstBlob, srcResp.Body) + if err != nil { + return fmt.Errorf("creating request for destination blob %q: %w", dstBlob, err) + } + + dstReq.ContentLength = srcResp.ContentLength + + dstResp, err := c.httpClient.Do(dstReq) + if err != nil { + return fmt.Errorf("putting destination blob %q: %w", dstBlob, err) + } + defer dstResp.Body.Close() //nolint:errcheck + + if dstResp.StatusCode != http.StatusOK && dstResp.StatusCode != http.StatusCreated && dstResp.StatusCode != http.StatusNoContent { + return fmt.Errorf("putting destination blob %q: wrong response code: %d; body: %s", dstBlob, dstResp.StatusCode, c.readAndTruncateBody(dstResp)) + } + + return nil +} + +// List returns a list of blob paths that match the given prefix +func (c *storageClient) List(prefix string) ([]string, error) { + // Validate non-empty prefix (empty prefix means list all) + if prefix != "" { + if err := validatePrefix(prefix); err != nil { + return nil, err + } + } + + blobURL, err := url.Parse(c.config.Endpoint) + if err != nil { + return nil, fmt.Errorf("parsing endpoint URL: %w", err) + } + + dirPath := blobURL.Path + if !strings.HasPrefix(dirPath, "/") { + dirPath = "/" + dirPath + } + blobURL.Path = dirPath + + // Perform recursive traversal of the WebDAV storage + // This works regardless of how the caller structures the paths + return c.listRecursive(blobURL.String(), blobURL.Path, prefix) +} + +// listRecursive performs recursive traversal of WebDAV collections +func (c *storageClient) listRecursive(dirURL string, endpointPath string, prefix string) ([]string, error) { + propfindBody := newPropfindBody() + + req, err := http.NewRequest("PROPFIND", dirURL, propfindBody) + if err != nil { + return nil, fmt.Errorf("creating PROPFIND request: %w", err) + } + + if c.config.User != "" { + req.SetBasicAuth(c.config.User, c.config.Password) + } + + req.Header.Set("Depth", "1") + req.Header.Set("Content-Type", "application/xml") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("performing PROPFIND request: %w", err) + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode != http.StatusMultiStatus && resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) //nolint:errcheck + return nil, fmt.Errorf("PROPFIND failed: %d, body: %s", resp.StatusCode, string(bodyBytes)) + } + + var propfindResp multistatusResponse + if err := xml.NewDecoder(resp.Body).Decode(&propfindResp); err != nil { + return nil, fmt.Errorf("decoding PROPFIND response: %w", err) + } + + reqURL, err := url.Parse(dirURL) + if err != nil { + return nil, fmt.Errorf("parsing request URL: %w", err) + } + requestPath := strings.TrimSuffix(reqURL.Path, "/") + + var allBlobs []string + for _, response := range propfindResp.Responses { + hrefURL, err := url.Parse(response.Href) + if err != nil { + continue + } + + hrefPath := strings.TrimSuffix(hrefURL.Path, "/") + + // Skip the directory itself + if hrefPath == requestPath { + continue + } + + if response.isCollection() { + // Recursively list subdirectory + subdirURL := hrefURL.String() + if !hrefURL.IsAbs() { + baseURL, err := url.Parse(dirURL) + if err != nil { + continue + } + subdirURL = baseURL.ResolveReference(hrefURL).String() + } + + subBlobs, err := c.listRecursive(subdirURL, endpointPath, prefix) + if err != nil { + return nil, err + } + allBlobs = append(allBlobs, subBlobs...) + } else { + // Extract blob ID relative to endpoint + blobID, err := c.extractBlobIDFromHref(response.Href, endpointPath) + if err != nil { + continue + } + + // Filter by prefix if specified + if prefix == "" || strings.HasPrefix(blobID, prefix) { + allBlobs = append(allBlobs, blobID) + } + } + } + + return allBlobs, nil +} + +// extractBlobIDFromHref extracts the blob ID from a WebDAV href +// Returns the path relative to the endpoint +func (c *storageClient) extractBlobIDFromHref(href, endpointPath string) (string, error) { + // URL decode the href + decoded, err := url.PathUnescape(href) + if err == nil { + href = decoded + } + + // Parse href as URL to handle both absolute URLs and paths + hrefURL, err := url.Parse(href) + if err != nil { + return "", fmt.Errorf("parsing href: %w", err) + } + + // Get the path component (works for both full URLs and path-only hrefs) + hrefPath := hrefURL.Path + + // Normalize: remove leading slash + hrefPath = strings.TrimPrefix(hrefPath, "/") + + // Strip endpoint path if present + endpointPathClean := strings.TrimPrefix(strings.TrimSuffix(endpointPath, "/"), "/") + if endpointPathClean != "" { + hrefPath = strings.TrimPrefix(hrefPath, endpointPathClean+"/") + } + + if hrefPath == "" { + return "", fmt.Errorf("no blob ID after stripping endpoint path") + } + + return hrefPath, nil +} + +// Properties retrieves metadata/properties for a blob using HEAD request +func (c *storageClient) Properties(path string) error { + if err := validateBlobID(path); err != nil { + return err + } + + req, err := c.createReq("HEAD", path, nil) + if err != nil { + return fmt.Errorf("creating HEAD request for blob %q: %w", path, err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("getting properties for blob %q: %w", path, err) + } + defer resp.Body.Close() //nolint:errcheck + + if resp.StatusCode == http.StatusNotFound { + fmt.Println(`{}`) + return nil + } + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("getting properties for blob %q: status %d", path, resp.StatusCode) + } + + // Extract properties from headers + properties := BlobProperties{ + ContentLength: resp.ContentLength, + } + + if etag := resp.Header.Get("ETag"); etag != "" { + properties.ETag = strings.Trim(etag, `"`) + } + + if lastModified := resp.Header.Get("Last-Modified"); lastModified != "" { + // Try multiple HTTP date formats per RFC 7231 + // RFC1123 is preferred, but some servers use RFC850 or ANSI C asctime + formats := []string{ + time.RFC1123, // "Mon, 02 Jan 2006 15:04:05 MST" + time.RFC1123Z, // "Mon, 02 Jan 2006 15:04:05 -0700" + time.RFC850, // "Monday, 02-Jan-06 15:04:05 MST" + time.ANSIC, // "Mon Jan _2 15:04:05 2006" + } + + var parsed bool + for _, format := range formats { + if t, err := time.Parse(format, lastModified); err == nil { + properties.LastModified = t + parsed = true + break + } + } + + if !parsed { + slog.Warn("Failed to parse Last-Modified header", "value", lastModified) + } + } + + output, err := json.MarshalIndent(properties, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal blob properties: %w", err) + } + + fmt.Println(string(output)) + return nil +} + +// EnsureStorageExists ensures the WebDAV directory structure exists +func (c *storageClient) EnsureStorageExists() error { + // When using signed URLs (secret present), the storage always exists. + // PROPFIND to the signed URL endpoint is not supported by nginx secure_link module. + // Skip the check in this case as the /read and /write paths are handled by nginx. + if c.config.Secret != "" { + return nil + } + + blobURL, err := url.Parse(c.config.Endpoint) + if err != nil { + return fmt.Errorf("parsing endpoint URL: %w", err) + } + + // Use PROPFIND (WebDAV-native method) instead of HEAD to check if collection exists + propfindBody := newPropfindBody() + + req, err := http.NewRequest("PROPFIND", blobURL.String(), propfindBody) + if err != nil { + return fmt.Errorf("creating PROPFIND request for root: %w", err) + } + + if c.config.User != "" { + req.SetBasicAuth(c.config.User, c.config.Password) + } + + req.Header.Set("Depth", "0") + req.Header.Set("Content-Type", "application/xml") + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("checking if root exists: %w", err) + } + defer resp.Body.Close() //nolint:errcheck + + // If the root exists, we're done + // PROPFIND returns 207 Multi-Status or 200 OK for existing collections + if resp.StatusCode == http.StatusMultiStatus || resp.StatusCode == http.StatusOK { + return nil + } + + // If not found, try to create it using MKCOL + if resp.StatusCode == http.StatusNotFound { + mkcolReq, err := http.NewRequest("MKCOL", blobURL.String(), nil) + if err != nil { + return fmt.Errorf("creating MKCOL request: %w", err) + } + + if c.config.User != "" { + mkcolReq.SetBasicAuth(c.config.User, c.config.Password) + } + + mkcolResp, err := c.httpClient.Do(mkcolReq) + if err != nil { + return fmt.Errorf("creating root directory: %w", err) + } + defer mkcolResp.Body.Close() //nolint:errcheck + + // Per RFC 4918, only accept standard MKCOL success responses: + // 201 Created - collection created successfully + // 405 Method Not Allowed - already exists (standard "already exists" case) + if mkcolResp.StatusCode == http.StatusCreated || mkcolResp.StatusCode == http.StatusMethodNotAllowed { + return nil + } + + // All other statuses are errors + bodyBytes, _ := io.ReadAll(io.LimitReader(mkcolResp.Body, 512)) //nolint:errcheck + bodyPreview := string(bodyBytes) + if len(bodyPreview) > 200 { + bodyPreview = bodyPreview[:200] + "..." + } + + return &davHTTPError{ + Operation: "MKCOL", + StatusCode: mkcolResp.StatusCode, + Body: bodyPreview, + } + } + + // Unexpected status - return error instead of silently succeeding + return &davHTTPError{ + Operation: "PROPFIND", + StatusCode: resp.StatusCode, + Body: "", + } +} + +// createReq creates an HTTP request for a blob operation +// IMPORTANT: blobID must be validated with validateBlobID before calling this function +func (c *storageClient) createReq(method, blobID string, body io.Reader) (*http.Request, error) { + // When using signed URLs, generate the signed URL with the signer + if c.signer != nil { + signedURL, err := c.signer.GenerateSignedURL( + c.config.Endpoint, + blobID, + method, + time.Now(), + 15*time.Minute, + ) + if err != nil { + return nil, fmt.Errorf("generating signed URL: %w", err) + } + + req, err := http.NewRequest(method, signedURL, body) + if err != nil { + return nil, err + } + return req, nil + } + + // Basic auth mode (no signer) + blobURL, err := url.Parse(c.config.Endpoint) + if err != nil { + return nil, err + } + + newPath := path.Join(blobURL.Path, blobID) + if !strings.HasPrefix(newPath, "/") { + newPath = "/" + newPath + } + + blobURL.Path = newPath + + req, err := http.NewRequest(method, blobURL.String(), body) + if err != nil { + return req, err + } + + if c.config.User != "" { + req.SetBasicAuth(c.config.User, c.config.Password) + } + return req, nil +} + +func (c *storageClient) readAndTruncateBody(resp *http.Response) string { + if resp.Body == nil { + return "" + } + bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 1024)) //nolint:errcheck + return string(bodyBytes) +} + +// ensureObjectParentsExist ensures all parent collections exist for the provided blob path. +// The caller provides the final object path, which may include nested directories. +// This function creates all intermediate collections required to store the blob. +// +// Examples: +// - "foo/bar.txt" creates /foo/ +// - "foo/bar/baz.txt" creates /foo/ and /foo/bar/ +// - "ab/cd/ef/object-id" creates /ab/, /ab/cd/, and /ab/cd/ef/ +func (c *storageClient) ensureObjectParentsExist(blobID string) error { + blobURL, err := url.Parse(c.config.Endpoint) + if err != nil { + return err + } + + basePath := blobURL.Path + + // If blob ID contains slashes, create all intermediate directories + if strings.Contains(blobID, "/") { + parts := strings.Split(blobID, "/") + // Skip the last part (the filename) + for i := 0; i < len(parts)-1; i++ { + dirPath := strings.Join(parts[0:i+1], "/") + if err := c.mkcolIfNeeded(basePath, dirPath); err != nil { + return err + } + } + } + + return nil +} + +// mkcolIfNeeded creates a WebDAV collection if it doesn't exist +func (c *storageClient) mkcolIfNeeded(basePath, collectionPath string) error { + blobURL, err := url.Parse(c.config.Endpoint) + if err != nil { + return err + } + + fullPath := path.Join(basePath, collectionPath) + if !strings.HasPrefix(fullPath, "/") { + fullPath = "/" + fullPath + } + // Add trailing slash for WebDAV collection + if !strings.HasSuffix(fullPath, "/") { + fullPath = fullPath + "/" + } + + blobURL.Path = fullPath + + req, err := http.NewRequest("MKCOL", blobURL.String(), nil) + if err != nil { + return err + } + + if c.config.User != "" { + req.SetBasicAuth(c.config.User, c.config.Password) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() //nolint:errcheck + + // Per RFC 4918, handle standard MKCOL response codes: + // 201 Created - collection created successfully + if resp.StatusCode == http.StatusCreated { + return nil + } + + // 405 Method Not Allowed - something already exists at that URI + // This is the standard "already exists" case + if resp.StatusCode == http.StatusMethodNotAllowed { + return nil + } + + // All other statuses are errors: + // 403 Forbidden - server policy forbids creating collection + // 409 Conflict - intermediate parent collection missing + // 401 Unauthorized - auth failure + // etc. + // Read response body for error details + bodyBytes, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) //nolint:errcheck + bodyPreview := string(bodyBytes) + if len(bodyPreview) > 200 { + bodyPreview = bodyPreview[:200] + "..." + } + + return &davHTTPError{ + Operation: "MKCOL", + StatusCode: resp.StatusCode, + Body: bodyPreview, + } +} + +// buildBlobURL constructs the full URL for a blob +// IMPORTANT: blobID must be validated with validateBlobID before calling this function +func (c *storageClient) buildBlobURL(blobID string) (string, error) { + blobURL, err := url.Parse(c.config.Endpoint) + if err != nil { + return "", err + } + + newPath := path.Join(blobURL.Path, blobID) + if !strings.HasPrefix(newPath, "/") { + newPath = "/" + newPath + } + blobURL.Path = newPath + + return blobURL.String(), nil +} diff --git a/dav/client/validation_test.go b/dav/client/validation_test.go new file mode 100644 index 0000000..1514596 --- /dev/null +++ b/dav/client/validation_test.go @@ -0,0 +1,118 @@ +package client + +import ( + "testing" +) + +func TestValidateBlobID(t *testing.T) { + tests := []struct { + name string + blobID string + wantError bool + }{ + // Valid blob IDs + {name: "simple blob", blobID: "file.txt", wantError: false}, + {name: "hierarchical blob", blobID: "foo/bar/baz.txt", wantError: false}, + {name: "deep hierarchy", blobID: "a/b/c/d/e/f.txt", wantError: false}, + {name: "with dashes", blobID: "my-file.txt", wantError: false}, + {name: "with underscores", blobID: "my_file.txt", wantError: false}, + {name: "with dots", blobID: "file.tar.gz", wantError: false}, + {name: "double dots in filename", blobID: "my..file.txt", wantError: false}, + {name: "version with dots", blobID: "version..1", wantError: false}, + {name: "nested with double dots", blobID: "foo/my..file.txt", wantError: false}, + {name: "uuid-like", blobID: "abc-123-def-456", wantError: false}, + {name: "nested with uuid", blobID: "backups/2024/abc-123.tar.gz", wantError: false}, + + // Invalid blob IDs - empty + {name: "empty string", blobID: "", wantError: true}, + + // Invalid blob IDs - leading/trailing slashes + {name: "leading slash", blobID: "/foo/bar.txt", wantError: true}, + {name: "trailing slash", blobID: "foo/bar/", wantError: true}, + {name: "both slashes", blobID: "/foo/bar/", wantError: true}, + + // Invalid blob IDs - path traversal + {name: "dot-dot segment", blobID: "foo/../bar.txt", wantError: true}, + {name: "dot-dot at start", blobID: "../bar.txt", wantError: true}, + {name: "dot-dot at end", blobID: "foo/..", wantError: true}, + {name: "multiple dot-dots", blobID: "foo/../../bar.txt", wantError: true}, + {name: "dot segment", blobID: "foo/./bar.txt", wantError: true}, + {name: "just dot-dot", blobID: "..", wantError: true}, + {name: "just dot", blobID: ".", wantError: true}, + + // Invalid blob IDs - empty segments + {name: "double slash", blobID: "foo//bar.txt", wantError: true}, + {name: "multiple double slashes", blobID: "foo///bar.txt", wantError: true}, + {name: "double slash at start", blobID: "//foo/bar.txt", wantError: true}, + + // Invalid blob IDs - control characters + {name: "null byte", blobID: "foo\x00bar.txt", wantError: true}, + {name: "newline", blobID: "foo\nbar.txt", wantError: true}, + {name: "tab", blobID: "foo\tbar.txt", wantError: true}, + {name: "carriage return", blobID: "foo\rbar.txt", wantError: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateBlobID(tt.blobID) + if tt.wantError && err == nil { + t.Errorf("validateBlobID(%q) expected error, got nil", tt.blobID) + } + if !tt.wantError && err != nil { + t.Errorf("validateBlobID(%q) unexpected error: %v", tt.blobID, err) + } + }) + } +} + +func TestValidatePrefix(t *testing.T) { + tests := []struct { + name string + prefix string + wantError bool + }{ + // Valid prefixes + {name: "simple prefix", prefix: "foo", wantError: false}, + {name: "hierarchical prefix", prefix: "foo/bar", wantError: false}, + {name: "prefix with trailing slash", prefix: "foo/", wantError: false}, + {name: "deep prefix with trailing slash", prefix: "foo/bar/baz/", wantError: false}, + {name: "prefix with dashes", prefix: "my-prefix", wantError: false}, + {name: "prefix with dots", prefix: "backup.2024", wantError: false}, + {name: "prefix with double dots in name", prefix: "my..prefix/", wantError: false}, + + // Invalid prefixes - empty + {name: "empty string", prefix: "", wantError: true}, + + // Invalid prefixes - leading slash + {name: "leading slash", prefix: "/foo", wantError: true}, + {name: "leading slash with trailing", prefix: "/foo/", wantError: true}, + + // Invalid prefixes - path traversal + {name: "dot-dot segment", prefix: "foo/../bar", wantError: true}, + {name: "dot-dot at start", prefix: "../foo", wantError: true}, + {name: "dot segment", prefix: "foo/./bar", wantError: true}, + {name: "just dot-dot", prefix: "..", wantError: true}, + {name: "dot-dot with trailing slash", prefix: "../", wantError: true}, + + // Invalid prefixes - empty segments + {name: "double slash", prefix: "foo//bar", wantError: true}, + {name: "double slash at end", prefix: "foo//", wantError: true}, + + // Invalid prefixes - control characters + {name: "null byte", prefix: "foo\x00bar", wantError: true}, + {name: "newline", prefix: "foo\nbar", wantError: true}, + {name: "tab", prefix: "foo\tbar", wantError: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validatePrefix(tt.prefix) + if tt.wantError && err == nil { + t.Errorf("validatePrefix(%q) expected error, got nil", tt.prefix) + } + if !tt.wantError && err != nil { + t.Errorf("validatePrefix(%q) unexpected error: %v", tt.prefix, err) + } + }) + } +} diff --git a/dav/cmd/cmd.go b/dav/cmd/cmd.go deleted file mode 100644 index 6f69763..0000000 --- a/dav/cmd/cmd.go +++ /dev/null @@ -1,5 +0,0 @@ -package cmd - -type Cmd interface { - Run(args []string) (err error) -} diff --git a/dav/cmd/cmd_suite_test.go b/dav/cmd/cmd_suite_test.go deleted file mode 100644 index 8d36bcd..0000000 --- a/dav/cmd/cmd_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package cmd_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "testing" -) - -func TestCmd(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Dav Cmd Suite") -} diff --git a/dav/cmd/delete.go b/dav/cmd/delete.go deleted file mode 100644 index f291828..0000000 --- a/dav/cmd/delete.go +++ /dev/null @@ -1,25 +0,0 @@ -package cmd - -import ( - "errors" - - davclient "github.com/cloudfoundry/storage-cli/dav/client" -) - -type DeleteCmd struct { - client davclient.Client -} - -func newDeleteCmd(client davclient.Client) (cmd DeleteCmd) { - cmd.client = client - return -} - -func (cmd DeleteCmd) Run(args []string) (err error) { - if len(args) != 1 { - err = errors.New("Incorrect usage, delete needs remote blob path") //nolint:staticcheck - return - } - err = cmd.client.Delete(args[0]) - return -} diff --git a/dav/cmd/delete_test.go b/dav/cmd/delete_test.go deleted file mode 100644 index 912c68b..0000000 --- a/dav/cmd/delete_test.go +++ /dev/null @@ -1,105 +0,0 @@ -package cmd_test - -import ( - "net/http" - "net/http/httptest" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - . "github.com/cloudfoundry/storage-cli/dav/cmd" - - boshlog "github.com/cloudfoundry/bosh-utils/logger" - testcmd "github.com/cloudfoundry/storage-cli/dav/cmd/testing" - davconf "github.com/cloudfoundry/storage-cli/dav/config" -) - -func runDelete(config davconf.Config, args []string) error { - logger := boshlog.NewLogger(boshlog.LevelNone) - factory := NewFactory(logger) - factory.SetConfig(config) //nolint:errcheck - - cmd, err := factory.Create("delete") - Expect(err).ToNot(HaveOccurred()) - - return cmd.Run(args) -} - -var _ = Describe("DeleteCmd", func() { - var ( - handler func(http.ResponseWriter, *http.Request) - requestedBlob string - ts *httptest.Server - config davconf.Config - ) - - BeforeEach(func() { - requestedBlob = "0ca907f2-dde8-4413-a304-9076c9d0978b" - - handler = func(w http.ResponseWriter, r *http.Request) { - req := testcmd.NewHTTPRequest(r) - - username, password, err := req.ExtractBasicAuth() - Expect(err).ToNot(HaveOccurred()) - Expect(req.URL.Path).To(Equal("/0d/" + requestedBlob)) - Expect(req.Method).To(Equal("DELETE")) - Expect(username).To(Equal("some user")) - Expect(password).To(Equal("some pwd")) - - w.WriteHeader(http.StatusOK) - } - }) - - AfterEach(func() { - ts.Close() - }) - - AssertDeleteBehavior := func() { - It("with valid args", func() { - err := runDelete(config, []string{requestedBlob}) - Expect(err).ToNot(HaveOccurred()) - }) - - It("returns err with incorrect arg count", func() { - err := runDelete(davconf.Config{}, []string{}) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Incorrect usage")) - }) - } - - Context("with http endpoint", func() { - BeforeEach(func() { - ts = httptest.NewServer(http.HandlerFunc(handler)) - config = davconf.Config{ - User: "some user", - Password: "some pwd", - Endpoint: ts.URL, - } - - }) - - AssertDeleteBehavior() - }) - - Context("with https endpoint", func() { - BeforeEach(func() { - ts = httptest.NewTLSServer(http.HandlerFunc(handler)) - - rootCa, err := testcmd.ExtractRootCa(ts) - Expect(err).ToNot(HaveOccurred()) - - config = davconf.Config{ - User: "some user", - Password: "some pwd", - Endpoint: ts.URL, - TLS: davconf.TLS{ - Cert: davconf.Cert{ - CA: rootCa, - }, - }, - } - }) - - AssertDeleteBehavior() - }) -}) diff --git a/dav/cmd/exists.go b/dav/cmd/exists.go deleted file mode 100644 index 220ccc6..0000000 --- a/dav/cmd/exists.go +++ /dev/null @@ -1,25 +0,0 @@ -package cmd - -import ( - "errors" - - davclient "github.com/cloudfoundry/storage-cli/dav/client" -) - -type ExistsCmd struct { - client davclient.Client -} - -func newExistsCmd(client davclient.Client) (cmd ExistsCmd) { - cmd.client = client - return -} - -func (cmd ExistsCmd) Run(args []string) (err error) { - if len(args) != 1 { - err = errors.New("Incorrect usage, exists needs remote blob path") //nolint:staticcheck - return - } - err = cmd.client.Exists(args[0]) - return -} diff --git a/dav/cmd/exists_test.go b/dav/cmd/exists_test.go deleted file mode 100644 index 0d01ce7..0000000 --- a/dav/cmd/exists_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package cmd_test - -import ( - "net/http" - "net/http/httptest" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - boshlog "github.com/cloudfoundry/bosh-utils/logger" - . "github.com/cloudfoundry/storage-cli/dav/cmd" - testcmd "github.com/cloudfoundry/storage-cli/dav/cmd/testing" - davconf "github.com/cloudfoundry/storage-cli/dav/config" -) - -func runExists(config davconf.Config, args []string) error { - logger := boshlog.NewLogger(boshlog.LevelNone) - factory := NewFactory(logger) - factory.SetConfig(config) //nolint:errcheck - - cmd, err := factory.Create("exists") - Expect(err).ToNot(HaveOccurred()) - - return cmd.Run(args) -} - -var _ = Describe("Exists", func() { - var ( - handler func(http.ResponseWriter, *http.Request) - requestedBlob string - ts *httptest.Server - config davconf.Config - ) - - BeforeEach(func() { - requestedBlob = "0ca907f2-dde8-4413-a304-9076c9d0978b" - - handler = func(w http.ResponseWriter, r *http.Request) { - req := testcmd.NewHTTPRequest(r) - - username, password, err := req.ExtractBasicAuth() - Expect(err).ToNot(HaveOccurred()) - Expect(req.URL.Path).To(Equal("/0d/" + requestedBlob)) - Expect(req.Method).To(Equal("HEAD")) - Expect(username).To(Equal("some user")) - Expect(password).To(Equal("some pwd")) - - w.WriteHeader(200) - } - }) - - AfterEach(func() { - ts.Close() - }) - - AssertExistsBehavior := func() { - It("with valid args", func() { - err := runExists(config, []string{requestedBlob}) - Expect(err).ToNot(HaveOccurred()) - }) - - It("with incorrect arg count", func() { - err := runExists(davconf.Config{}, []string{}) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Incorrect usage")) - }) - } - - Context("with http endpoint", func() { - BeforeEach(func() { - ts = httptest.NewServer(http.HandlerFunc(handler)) - config = davconf.Config{ - User: "some user", - Password: "some pwd", - Endpoint: ts.URL, - } - - }) - - AssertExistsBehavior() - }) - - Context("with https endpoint", func() { - BeforeEach(func() { - ts = httptest.NewTLSServer(http.HandlerFunc(handler)) - - rootCa, err := testcmd.ExtractRootCa(ts) - Expect(err).ToNot(HaveOccurred()) - - config = davconf.Config{ - User: "some user", - Password: "some pwd", - Endpoint: ts.URL, - TLS: davconf.TLS{ - Cert: davconf.Cert{ - CA: rootCa, - }, - }, - } - }) - - AssertExistsBehavior() - }) -}) diff --git a/dav/cmd/factory.go b/dav/cmd/factory.go deleted file mode 100644 index 6b68025..0000000 --- a/dav/cmd/factory.go +++ /dev/null @@ -1,62 +0,0 @@ -package cmd - -import ( - "crypto/x509" - "fmt" - - boshcrypto "github.com/cloudfoundry/bosh-utils/crypto" - boshhttpclient "github.com/cloudfoundry/bosh-utils/httpclient" - boshlog "github.com/cloudfoundry/bosh-utils/logger" - - davclient "github.com/cloudfoundry/storage-cli/dav/client" - davconf "github.com/cloudfoundry/storage-cli/dav/config" -) - -type Factory interface { - Create(name string) (cmd Cmd, err error) - SetConfig(config davconf.Config) (err error) -} - -func NewFactory(logger boshlog.Logger) Factory { - return &factory{ - cmds: make(map[string]Cmd), - logger: logger, - } -} - -type factory struct { - config davconf.Config //nolint:unused - cmds map[string]Cmd - logger boshlog.Logger -} - -func (f *factory) Create(name string) (cmd Cmd, err error) { - cmd, found := f.cmds[name] - if !found { - err = fmt.Errorf("Could not find command with name %s", name) //nolint:staticcheck - } - return -} - -func (f *factory) SetConfig(config davconf.Config) (err error) { - var httpClient boshhttpclient.Client - var certPool *x509.CertPool - - if len(config.TLS.Cert.CA) != 0 { - certPool, err = boshcrypto.CertPoolFromPEM([]byte(config.TLS.Cert.CA)) - } - - httpClient = boshhttpclient.CreateDefaultClient(certPool) - - client := davclient.NewClient(config, httpClient, f.logger) - - f.cmds = map[string]Cmd{ - "put": newPutCmd(client), - "get": newGetCmd(client), - "exists": newExistsCmd(client), - "delete": newDeleteCmd(client), - "sign": newSignCmd(client), - } - - return -} diff --git a/dav/cmd/factory_test.go b/dav/cmd/factory_test.go deleted file mode 100644 index 46378a6..0000000 --- a/dav/cmd/factory_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package cmd_test - -import ( - "reflect" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - boshlog "github.com/cloudfoundry/bosh-utils/logger" - . "github.com/cloudfoundry/storage-cli/dav/cmd" - davconf "github.com/cloudfoundry/storage-cli/dav/config" -) - -func buildFactory() (factory Factory) { - config := davconf.Config{User: "some user"} - logger := boshlog.NewLogger(boshlog.LevelNone) - factory = NewFactory(logger) - factory.SetConfig(config) //nolint:errcheck - return -} - -var _ = Describe("Factory", func() { - Describe("Create", func() { - It("factory create a put command", func() { - factory := buildFactory() - cmd, err := factory.Create("put") - - Expect(err).ToNot(HaveOccurred()) - Expect(reflect.TypeOf(cmd)).To(Equal(reflect.TypeOf(PutCmd{}))) - }) - - It("factory create a get command", func() { - factory := buildFactory() - cmd, err := factory.Create("get") - - Expect(err).ToNot(HaveOccurred()) - Expect(reflect.TypeOf(cmd)).To(Equal(reflect.TypeOf(GetCmd{}))) - }) - - It("factory create a delete command", func() { - factory := buildFactory() - cmd, err := factory.Create("delete") - - Expect(err).ToNot(HaveOccurred()) - Expect(reflect.TypeOf(cmd)).To(Equal(reflect.TypeOf(DeleteCmd{}))) - }) - - It("factory create when cmd is unknown", func() { - factory := buildFactory() - _, err := factory.Create("some unknown cmd") - - Expect(err).To(HaveOccurred()) - }) - }) - - Describe("SetConfig", func() { - It("returns an error if CaCert is given but invalid", func() { - factory := buildFactory() - config := davconf.Config{ - TLS: davconf.TLS{ - Cert: davconf.Cert{ - CA: "--- INVALID CERTIFICATE ---", - }, - }, - } - - err := factory.SetConfig(config) - Expect(err).To(HaveOccurred()) - }) - It("does not return an error if CaCert is valid", func() { - factory := buildFactory() - cert := `-----BEGIN CERTIFICATE----- -MIICEzCCAXygAwIBAgIQMIMChMLGrR+QvmQvpwAU6zANBgkqhkiG9w0BAQsFADAS -MRAwDgYDVQQKEwdBY21lIENvMCAXDTcwMDEwMTAwMDAwMFoYDzIwODQwMTI5MTYw -MDAwWjASMRAwDgYDVQQKEwdBY21lIENvMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB -iQKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9SjY1bIw4 -iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZBl2+XsDul -rKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQABo2gwZjAO -BgNVHQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUw -AwEB/zAuBgNVHREEJzAlggtleGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAAAAAAAAAA -AAAAATANBgkqhkiG9w0BAQsFAAOBgQCEcetwO59EWk7WiJsG4x8SY+UIAA+flUI9 -tyC4lNhbcF2Idq9greZwbYCqTTTr2XiRNSMLCOjKyI7ukPoPjo16ocHj+P3vZGfs -h1fIw3cSS2OolhloGw/XM6RWPWtPAlGykKLciQrBru5NAPvCMsb/I1DAceTiotQM -fblo6RBxUQ== ------END CERTIFICATE-----` - config := davconf.Config{ - TLS: davconf.TLS{ - Cert: davconf.Cert{ - CA: cert, - }, - }, - } - - err := factory.SetConfig(config) - Expect(err).ToNot(HaveOccurred()) - }) - It("does not return an error if CaCert is not provided", func() { - factory := buildFactory() - config := davconf.Config{ - TLS: davconf.TLS{ - Cert: davconf.Cert{ - CA: "", - }, - }, - } - - err := factory.SetConfig(config) - Expect(err).ToNot(HaveOccurred()) - }) - }) -}) diff --git a/dav/cmd/get.go b/dav/cmd/get.go deleted file mode 100644 index 3009585..0000000 --- a/dav/cmd/get.go +++ /dev/null @@ -1,39 +0,0 @@ -package cmd - -import ( - "errors" - "io" - "os" - - davclient "github.com/cloudfoundry/storage-cli/dav/client" -) - -type GetCmd struct { - client davclient.Client -} - -func newGetCmd(client davclient.Client) (cmd GetCmd) { - cmd.client = client - return -} - -func (cmd GetCmd) Run(args []string) (err error) { - if len(args) != 2 { - err = errors.New("Incorrect usage, get needs remote blob path and local file destination") //nolint:staticcheck - return - } - - readCloser, err := cmd.client.Get(args[0]) - if err != nil { - return - } - defer readCloser.Close() //nolint:errcheck - - targetFile, err := os.Create(args[1]) - if err != nil { - return - } - - _, err = io.Copy(targetFile, readCloser) - return -} diff --git a/dav/cmd/get_test.go b/dav/cmd/get_test.go deleted file mode 100644 index 0ab58a7..0000000 --- a/dav/cmd/get_test.go +++ /dev/null @@ -1,122 +0,0 @@ -package cmd_test - -import ( - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - boshlog "github.com/cloudfoundry/bosh-utils/logger" - . "github.com/cloudfoundry/storage-cli/dav/cmd" - testcmd "github.com/cloudfoundry/storage-cli/dav/cmd/testing" - davconf "github.com/cloudfoundry/storage-cli/dav/config" -) - -func runGet(config davconf.Config, args []string) error { - logger := boshlog.NewLogger(boshlog.LevelNone) - factory := NewFactory(logger) - factory.SetConfig(config) //nolint:errcheck - - cmd, err := factory.Create("get") - Expect(err).ToNot(HaveOccurred()) - - return cmd.Run(args) -} - -func getFileContent(path string) string { - file, err := os.Open(path) - Expect(err).ToNot(HaveOccurred()) - - fileBytes, err := io.ReadAll(file) - Expect(err).ToNot(HaveOccurred()) - - return string(fileBytes) -} - -var _ = Describe("GetCmd", func() { - var ( - handler func(http.ResponseWriter, *http.Request) - targetFilePath string - requestedBlob string - ts *httptest.Server - config davconf.Config - ) - - BeforeEach(func() { - requestedBlob = "0ca907f2-dde8-4413-a304-9076c9d0978b" - targetFilePath = filepath.Join(os.TempDir(), "testRunGetCommand.txt") - - handler = func(w http.ResponseWriter, r *http.Request) { - req := testcmd.NewHTTPRequest(r) - - username, password, err := req.ExtractBasicAuth() - Expect(err).ToNot(HaveOccurred()) - Expect(req.URL.Path).To(Equal("/0d/" + requestedBlob)) - Expect(req.Method).To(Equal("GET")) - Expect(username).To(Equal("some user")) - Expect(password).To(Equal("some pwd")) - - w.Write([]byte("this is your blob")) //nolint:errcheck - } - - }) - - AfterEach(func() { - os.RemoveAll(targetFilePath) //nolint:errcheck - ts.Close() - }) - - AssertGetBehavior := func() { - It("get run with valid args", func() { - err := runGet(config, []string{requestedBlob, targetFilePath}) - Expect(err).ToNot(HaveOccurred()) - Expect(getFileContent(targetFilePath)).To(Equal("this is your blob")) - }) - - It("get run with incorrect arg count", func() { - err := runGet(davconf.Config{}, []string{}) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Incorrect usage")) - }) - } - - Context("with http endpoint", func() { - BeforeEach(func() { - ts = httptest.NewServer(http.HandlerFunc(handler)) - - config = davconf.Config{ - User: "some user", - Password: "some pwd", - Endpoint: ts.URL, - } - }) - - AssertGetBehavior() - }) - - Context("with https endpoint", func() { - BeforeEach(func() { - ts = httptest.NewTLSServer(http.HandlerFunc(handler)) - - rootCa, err := testcmd.ExtractRootCa(ts) - Expect(err).ToNot(HaveOccurred()) - - config = davconf.Config{ - User: "some user", - Password: "some pwd", - Endpoint: ts.URL, - TLS: davconf.TLS{ - Cert: davconf.Cert{ - CA: rootCa, - }, - }, - } - }) - - AssertGetBehavior() - }) -}) diff --git a/dav/cmd/put.go b/dav/cmd/put.go deleted file mode 100644 index 44f6d84..0000000 --- a/dav/cmd/put.go +++ /dev/null @@ -1,35 +0,0 @@ -package cmd - -import ( - "errors" - "os" - - davclient "github.com/cloudfoundry/storage-cli/dav/client" -) - -type PutCmd struct { - client davclient.Client -} - -func newPutCmd(client davclient.Client) (cmd PutCmd) { - cmd.client = client - return -} - -func (cmd PutCmd) Run(args []string) error { - if len(args) != 2 { - return errors.New("Incorrect usage, put needs local file and remote blob destination") //nolint:staticcheck - } - - file, err := os.OpenFile(args[0], os.O_RDWR, os.ModeExclusive) - if err != nil { - return err - } - - fileInfo, err := file.Stat() - if err != nil { - return err - } - - return cmd.client.Put(args[1], file, fileInfo.Size()) -} diff --git a/dav/cmd/put_test.go b/dav/cmd/put_test.go deleted file mode 100644 index f7af661..0000000 --- a/dav/cmd/put_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package cmd_test - -import ( - "io" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - boshlog "github.com/cloudfoundry/bosh-utils/logger" - - . "github.com/cloudfoundry/storage-cli/dav/cmd" - testcmd "github.com/cloudfoundry/storage-cli/dav/cmd/testing" - davconf "github.com/cloudfoundry/storage-cli/dav/config" -) - -func runPut(config davconf.Config, args []string) error { - logger := boshlog.NewLogger(boshlog.LevelNone) - factory := NewFactory(logger) - factory.SetConfig(config) //nolint:errcheck - - cmd, err := factory.Create("put") - Expect(err).ToNot(HaveOccurred()) - - return cmd.Run(args) -} - -func fileBytes(path string) []byte { - file, err := os.Open(path) - Expect(err).ToNot(HaveOccurred()) - - content, err := io.ReadAll(file) - Expect(err).ToNot(HaveOccurred()) - - return content -} - -var _ = Describe("PutCmd", func() { - Describe("Run", func() { - var ( - handler func(http.ResponseWriter, *http.Request) - config davconf.Config - ts *httptest.Server - sourceFilePath string - targetBlob string - serverWasHit bool - ) - BeforeEach(func() { - pwd, err := os.Getwd() - Expect(err).ToNot(HaveOccurred()) - - sourceFilePath = filepath.Join(pwd, "../test_assets/cat.jpg") - targetBlob = "some-other-awesome-guid" - serverWasHit = false - - handler = func(w http.ResponseWriter, r *http.Request) { - defer GinkgoRecover() - serverWasHit = true - req := testcmd.NewHTTPRequest(r) - - username, password, err := req.ExtractBasicAuth() - Expect(err).ToNot(HaveOccurred()) - Expect(req.URL.Path).To(Equal("/d1/" + targetBlob)) - Expect(req.Method).To(Equal("PUT")) - Expect(req.ContentLength).To(Equal(int64(1718186))) - Expect(username).To(Equal("some user")) - Expect(password).To(Equal("some pwd")) - - expectedBytes := fileBytes(sourceFilePath) - actualBytes, _ := io.ReadAll(r.Body) //nolint:errcheck - Expect(expectedBytes).To(Equal(actualBytes)) - - w.WriteHeader(201) - } - }) - - AfterEach(func() { - defer ts.Close() - }) - - AssertPutBehavior := func() { - It("uploads the blob with valid args", func() { - err := runPut(config, []string{sourceFilePath, targetBlob}) - Expect(err).ToNot(HaveOccurred()) - Expect(serverWasHit).To(BeTrue()) - }) - - It("returns err with incorrect arg count", func() { - err := runPut(davconf.Config{}, []string{}) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Incorrect usage")) - }) - } - - Context("with http endpoint", func() { - BeforeEach(func() { - ts = httptest.NewServer(http.HandlerFunc(handler)) - config = davconf.Config{ - User: "some user", - Password: "some pwd", - Endpoint: ts.URL, - } - - }) - - AssertPutBehavior() - }) - - Context("with https endpoint", func() { - BeforeEach(func() { - ts = httptest.NewTLSServer(http.HandlerFunc(handler)) - - rootCa, err := testcmd.ExtractRootCa(ts) - Expect(err).ToNot(HaveOccurred()) - - config = davconf.Config{ - User: "some user", - Password: "some pwd", - Endpoint: ts.URL, - TLS: davconf.TLS{ - Cert: davconf.Cert{ - CA: rootCa, - }, - }, - } - }) - - AssertPutBehavior() - }) - }) -}) diff --git a/dav/cmd/runner.go b/dav/cmd/runner.go deleted file mode 100644 index 0fbf423..0000000 --- a/dav/cmd/runner.go +++ /dev/null @@ -1,40 +0,0 @@ -package cmd - -import ( - "errors" - - davconf "github.com/cloudfoundry/storage-cli/dav/config" -) - -type Runner interface { - SetConfig(newConfig davconf.Config) (err error) - Run(cmdArgs []string) (err error) -} - -func NewRunner(factory Factory) Runner { - return runner{ - factory: factory, - } -} - -type runner struct { - factory Factory -} - -func (r runner) Run(cmdArgs []string) (err error) { - if len(cmdArgs) == 0 { - err = errors.New("Missing command name") //nolint:staticcheck - return - } - - cmd, err := r.factory.Create(cmdArgs[0]) - if err != nil { - return - } - - return cmd.Run(cmdArgs[1:]) -} - -func (r runner) SetConfig(newConfig davconf.Config) (err error) { - return r.factory.SetConfig(newConfig) -} diff --git a/dav/cmd/runner_test.go b/dav/cmd/runner_test.go deleted file mode 100644 index 2087b1a..0000000 --- a/dav/cmd/runner_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package cmd_test - -import ( - "errors" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - . "github.com/cloudfoundry/storage-cli/dav/cmd" - davconf "github.com/cloudfoundry/storage-cli/dav/config" -) - -type FakeFactory struct { - CreateName string - CreateCmd *FakeCmd - CreateErr error - - Config davconf.Config - SetConfigErr error -} - -func (f *FakeFactory) Create(name string) (cmd Cmd, err error) { - f.CreateName = name - cmd = f.CreateCmd - err = f.CreateErr - return -} - -func (f *FakeFactory) SetConfig(config davconf.Config) (err error) { - f.Config = config - return f.SetConfigErr -} - -type FakeCmd struct { - RunArgs []string - RunErr error -} - -func (cmd *FakeCmd) Run(args []string) (err error) { - cmd.RunArgs = args - err = cmd.RunErr - return -} - -var _ = Describe("Runner", func() { - Describe("Run", func() { - It("run can run a command and return its error", func() { - factory := &FakeFactory{ - CreateCmd: &FakeCmd{ - RunErr: errors.New("fake-run-error"), - }, - } - cmdRunner := NewRunner(factory) - - err := cmdRunner.Run([]string{"put", "foo", "bar"}) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(Equal("fake-run-error")) - - Expect(factory.CreateName).To(Equal("put")) - Expect(factory.CreateCmd.RunArgs).To(Equal([]string{"foo", "bar"})) - }) - - It("run expects at least one argument", func() { - factory := &FakeFactory{ - CreateCmd: &FakeCmd{}, - } - cmdRunner := NewRunner(factory) - - err := cmdRunner.Run([]string{}) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(Equal("Missing command name")) - }) - - It("accepts exactly one argument", func() { - factory := &FakeFactory{ - CreateCmd: &FakeCmd{}, - } - cmdRunner := NewRunner(factory) - - err := cmdRunner.Run([]string{"put"}) - Expect(err).ToNot(HaveOccurred()) - - Expect(factory.CreateName).To(Equal("put")) - Expect(factory.CreateCmd.RunArgs).To(Equal([]string{})) - }) - }) - - Describe("SetConfig", func() { - It("delegates to factory", func() { - factory := &FakeFactory{} - cmdRunner := NewRunner(factory) - conf := davconf.Config{User: "foo"} - - err := cmdRunner.SetConfig(conf) - - Expect(factory.Config).To(Equal(conf)) - Expect(err).ToNot(HaveOccurred()) - }) - It("propagates errors", func() { - setConfigErr := errors.New("some error") - factory := &FakeFactory{ - SetConfigErr: setConfigErr, - } - cmdRunner := NewRunner(factory) - conf := davconf.Config{User: "foo"} - - err := cmdRunner.SetConfig(conf) - Expect(err).To(HaveOccurred()) - }) - }) -}) diff --git a/dav/cmd/sign.go b/dav/cmd/sign.go deleted file mode 100644 index 27b9ac6..0000000 --- a/dav/cmd/sign.go +++ /dev/null @@ -1,41 +0,0 @@ -package cmd - -import ( - "errors" - "fmt" - "time" - - davclient "github.com/cloudfoundry/storage-cli/dav/client" -) - -type SignCmd struct { - client davclient.Client -} - -func newSignCmd(client davclient.Client) (cmd SignCmd) { - cmd.client = client - return -} - -func (cmd SignCmd) Run(args []string) (err error) { - if len(args) != 3 { - err = errors.New("incorrect usage, sign requires: ") - return - } - - objectID, action := args[0], args[1] - - expiration, err := time.ParseDuration(args[2]) - if err != nil { - err = fmt.Errorf("expiration should be a duration value eg: 45s or 1h43m. Got: %s", args[2]) - return - } - - signedURL, err := cmd.client.Sign(objectID, action, expiration) - if err != nil { - return err - } - - fmt.Print(signedURL) - return -} diff --git a/dav/cmd/sign_test.go b/dav/cmd/sign_test.go deleted file mode 100644 index 09a570d..0000000 --- a/dav/cmd/sign_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package cmd_test - -import ( - "bytes" - "io" - "os" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - . "github.com/cloudfoundry/storage-cli/dav/cmd" - - boshlog "github.com/cloudfoundry/bosh-utils/logger" - - davconf "github.com/cloudfoundry/storage-cli/dav/config" -) - -func runSign(config davconf.Config, args []string) error { - logger := boshlog.NewLogger(boshlog.LevelNone) - factory := NewFactory(logger) - factory.SetConfig(config) //nolint:errcheck - - cmd, err := factory.Create("sign") - Expect(err).ToNot(HaveOccurred()) - - return cmd.Run(args) -} - -var _ = Describe("SignCmd", func() { - var ( - objectID = "0ca907f2-dde8-4413-a304-9076c9d0978b" - config davconf.Config - ) - - It("with valid args", func() { - old := os.Stdout // keep backup of the real stdout - r, w, _ := os.Pipe() //nolint:errcheck - os.Stdout = w - - err := runSign(config, []string{objectID, "get", "15m"}) - - outC := make(chan string) - // copy the output in a separate goroutine so printing can't block indefinitely - go func() { - var buf bytes.Buffer - io.Copy(&buf, r) //nolint:errcheck - outC <- buf.String() - }() - - // back to normal state - w.Close() //nolint:errcheck - os.Stdout = old // restoring the real stdout - out := <-outC - - Expect(err).ToNot(HaveOccurred()) - Expect(out).To(HavePrefix("signed/")) - Expect(out).To(ContainSubstring(objectID)) - Expect(out).To(ContainSubstring("?e=")) - Expect(out).To(ContainSubstring("&st=")) - Expect(out).To(ContainSubstring("&ts=")) - }) - - It("returns err with incorrect arg count", func() { - err := runSign(davconf.Config{}, []string{}) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("incorrect usage")) - }) - - It("returns err with non-implemented action", func() { - err := runSign(davconf.Config{}, []string{objectID, "delete", "15m"}) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("action not implemented")) - }) - - It("returns err with incorrect duration", func() { - err := runSign(davconf.Config{}, []string{objectID, "put", "15"}) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("expiration should be a duration value")) - }) -}) diff --git a/dav/cmd/testing/http_request.go b/dav/cmd/testing/http_request.go deleted file mode 100644 index 912d363..0000000 --- a/dav/cmd/testing/http_request.go +++ /dev/null @@ -1,47 +0,0 @@ -package testing - -import ( - "encoding/base64" - "errors" - "net/http" - "strings" -) - -type HTTPRequest struct { - *http.Request -} - -func NewHTTPRequest(req *http.Request) (testReq HTTPRequest) { - return HTTPRequest{req} -} - -func (req HTTPRequest) ExtractBasicAuth() (username, password string, err error) { - authHeader := req.Header["Authorization"] - if len(authHeader) != 1 { - err = errors.New("Missing basic auth header") //nolint:staticcheck - return - } - - encodedAuth := authHeader[0] - encodedAuthParts := strings.Split(encodedAuth, " ") - if len(encodedAuthParts) != 2 { - err = errors.New("Invalid basic auth header format") //nolint:staticcheck - return - } - - clearAuth, err := base64.StdEncoding.DecodeString(encodedAuthParts[1]) - if len(encodedAuthParts) != 2 { - err = errors.New("Invalid basic auth header encoding") //nolint:staticcheck - return - } - - clearAuthParts := strings.Split(string(clearAuth), ":") - if len(clearAuthParts) != 2 { - err = errors.New("Invalid basic auth header encoded username and pwd") //nolint:staticcheck - return - } - - username = clearAuthParts[0] - password = clearAuthParts[1] - return -} diff --git a/dav/cmd/testing/testing_suite_test.go b/dav/cmd/testing/testing_suite_test.go deleted file mode 100644 index e1ac225..0000000 --- a/dav/cmd/testing/testing_suite_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package testing_test - -import ( - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "testing" -) - -func TestTesting(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Dav Testing Suite") -} diff --git a/dav/cmd/testing/tls_server.go b/dav/cmd/testing/tls_server.go deleted file mode 100644 index 6bdeb96..0000000 --- a/dav/cmd/testing/tls_server.go +++ /dev/null @@ -1,31 +0,0 @@ -package testing - -import ( - "bytes" - "crypto/x509" - "encoding/pem" - "net/http/httptest" -) - -func ExtractRootCa(server *httptest.Server) (rootCaStr string, err error) { - rootCa := new(bytes.Buffer) - - cert, err := x509.ParseCertificate(server.TLS.Certificates[0].Certificate[0]) - if err != nil { - panic(err.Error()) - } - // TODO: Replace above with following on Go 1.9 - //cert := server.Certificate() - - block := &pem.Block{ - Type: "CERTIFICATE", - Bytes: cert.Raw, - } - - err = pem.Encode(rootCa, block) - if err != nil { - return "", err - } - - return rootCa.String(), nil -} diff --git a/dav/config/config.go b/dav/config/config.go index 40711ac..c3dd568 100644 --- a/dav/config/config.go +++ b/dav/config/config.go @@ -12,6 +12,7 @@ type Config struct { RetryAttempts uint TLS TLS Secret string + SigningMethod string `json:"signing_method"` // "sha256" (default) or "md5" } type TLS struct { diff --git a/dav/integration/assertions.go b/dav/integration/assertions.go new file mode 100644 index 0000000..3ce4e6d --- /dev/null +++ b/dav/integration/assertions.go @@ -0,0 +1,235 @@ +package integration + +import ( + "fmt" + "os" + + "github.com/cloudfoundry/storage-cli/dav/config" + + . "github.com/onsi/gomega" //nolint:staticcheck +) + +// AssertLifecycleWorks tests the main blobstore object lifecycle from creation to deletion +func AssertLifecycleWorks(cliPath string, cfg *config.Config) { + storageType := "dav" + expectedString := GenerateRandomString() + blobName := GenerateRandomString() + + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + contentFile := MakeContentFile(expectedString) + defer os.Remove(contentFile) //nolint:errcheck + + // Test PUT + session, err := RunCli(cliPath, configPath, storageType, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + // Test EXISTS + session, err = RunCli(cliPath, configPath, storageType, "exists", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + // Test GET + tmpLocalFile, err := os.CreateTemp("", "davcli-download") + Expect(err).ToNot(HaveOccurred()) + err = tmpLocalFile.Close() + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tmpLocalFile.Name()) //nolint:errcheck + + session, err = RunCli(cliPath, configPath, storageType, "get", blobName, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + gottenBytes, err := os.ReadFile(tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(string(gottenBytes)).To(Equal(expectedString)) + + // Test PROPERTIES + session, err = RunCli(cliPath, configPath, storageType, "properties", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + Expect(session.Out.Contents()).To(ContainSubstring(fmt.Sprintf("\"content_length\": %d", len(expectedString)))) + Expect(session.Out.Contents()).To(ContainSubstring("\"etag\":")) + Expect(session.Out.Contents()).To(ContainSubstring("\"last_modified\":")) + + // Test COPY + session, err = RunCli(cliPath, configPath, storageType, "copy", blobName, blobName+"_copy") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunCli(cliPath, configPath, storageType, "exists", blobName+"_copy") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + tmpCopiedFile, err := os.CreateTemp("", "davcli-download-copy") + Expect(err).ToNot(HaveOccurred()) + err = tmpCopiedFile.Close() + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tmpCopiedFile.Name()) //nolint:errcheck + + session, err = RunCli(cliPath, configPath, storageType, "get", blobName+"_copy", tmpCopiedFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + copiedBytes, err := os.ReadFile(tmpCopiedFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(string(copiedBytes)).To(Equal(expectedString)) + + // Test DELETE (copied blob) + session, err = RunCli(cliPath, configPath, storageType, "delete", blobName+"_copy") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + // Test DELETE (original blob) + session, err = RunCli(cliPath, configPath, storageType, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + // Verify blob no longer exists + session, err = RunCli(cliPath, configPath, storageType, "exists", blobName) + Expect(err).ToNot(HaveOccurred()) + // Exit code should be non-zero (blob doesn't exist) + Expect(session.ExitCode()).ToNot(BeZero()) + + // Properties should return empty for non-existent blob + session, err = RunCli(cliPath, configPath, storageType, "properties", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + Expect(session.Out.Contents()).To(ContainSubstring("{}")) +} + +// AssertGetNonexistentFails tests that getting a non-existent blob fails +func AssertGetNonexistentFails(cliPath string, cfg *config.Config) { + storageType := "dav" + blobName := GenerateRandomString() + + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + tmpLocalFile, err := os.CreateTemp("", "davcli-download") + Expect(err).ToNot(HaveOccurred()) + err = tmpLocalFile.Close() + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tmpLocalFile.Name()) //nolint:errcheck + + session, err := RunCli(cliPath, configPath, storageType, "get", blobName, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).ToNot(BeZero()) +} + +// AssertDeleteNonexistentWorks tests that deleting a non-existent blob succeeds +func AssertDeleteNonexistentWorks(cliPath string, cfg *config.Config) { + storageType := "dav" + blobName := GenerateRandomString() + + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + session, err := RunCli(cliPath, configPath, storageType, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) +} + +// AssertOnListDeleteLifecycle tests list and delete-recursive functionality +func AssertOnListDeleteLifecycle(cliPath string, cfg *config.Config) { + storageType := "dav" + prefix := GenerateRandomString() + + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + // Create multiple blobs with the same prefix + for i := 0; i < 3; i++ { + content := GenerateRandomString() + contentFile := MakeContentFile(content) + defer os.Remove(contentFile) //nolint:errcheck + + blobName := fmt.Sprintf("%s-%d", prefix, i) + session, err := RunCli(cliPath, configPath, storageType, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + } + + // Test LIST + session, err := RunCli(cliPath, configPath, storageType, "list", prefix) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + for i := 0; i < 3; i++ { + Expect(session.Out.Contents()).To(ContainSubstring(fmt.Sprintf("%s-%d", prefix, i))) + } + + // Test DELETE-RECURSIVE + session, err = RunCli(cliPath, configPath, storageType, "delete-recursive", prefix) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + // Verify all blobs are deleted + for i := 0; i < 3; i++ { + blobName := fmt.Sprintf("%s-%d", prefix, i) + session, err := RunCli(cliPath, configPath, storageType, "exists", blobName) + Expect(err).ToNot(HaveOccurred()) + // Exit code should be non-zero (blob doesn't exist) + // DAV returns 3 for NotExistsError, but may return 1 for other "not found" scenarios + Expect(session.ExitCode()).ToNot(BeZero()) + } +} + +// AssertOnSignedURLs tests signed URL generation +// Note: This test only validates that signed URLs are generated with correct format. +// It does not test actual signed URL usage since that requires nginx with secure_link module, +// which is not available in the Apache WebDAV test environment. +func AssertOnSignedURLs(cliPath string, cfg *config.Config) { + storageType := "dav" + blobName := GenerateRandomString() + + // Create config with secret for signing + configWithSecret := MakeConfigFile(cfg) + defer os.Remove(configWithSecret) //nolint:errcheck + + // Generate signed PUT URL + session, err := RunCli(cliPath, configWithSecret, storageType, "sign", blobName, "put", "3600s") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + signedPutURL := string(session.Out.Contents()) + Expect(signedPutURL).To(ContainSubstring("http")) + Expect(signedPutURL).To(ContainSubstring("st=")) + Expect(signedPutURL).To(ContainSubstring("ts=")) + Expect(signedPutURL).To(ContainSubstring("e=")) + + // Verify PUT URL contains /signed/ path prefix for BOSH compatibility + Expect(signedPutURL).To(ContainSubstring("/signed/")) + + // Generate signed GET URL + session, err = RunCli(cliPath, configWithSecret, storageType, "sign", blobName, "get", "3600s") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + signedGetURL := string(session.Out.Contents()) + Expect(signedGetURL).To(ContainSubstring("http")) + Expect(signedGetURL).To(ContainSubstring("st=")) + Expect(signedGetURL).To(ContainSubstring("ts=")) + Expect(signedGetURL).To(ContainSubstring("e=")) + + // Verify GET URL contains /signed/ path prefix for BOSH compatibility + Expect(signedGetURL).To(ContainSubstring("/signed/")) +} + +// AssertEnsureStorageExists tests ensure-storage-exists command +func AssertEnsureStorageExists(cliPath string, cfg *config.Config) { + storageType := "dav" + + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + session, err := RunCli(cliPath, configPath, storageType, "ensure-storage-exists") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + // Should be idempotent - run again + session, err = RunCli(cliPath, configPath, storageType, "ensure-storage-exists") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) +} diff --git a/dav/integration/general_dav_test.go b/dav/integration/general_dav_test.go new file mode 100644 index 0000000..8899c0c --- /dev/null +++ b/dav/integration/general_dav_test.go @@ -0,0 +1,203 @@ +package integration_test + +import ( + "os" + + "github.com/cloudfoundry/storage-cli/dav/config" + "github.com/cloudfoundry/storage-cli/dav/integration" + + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("General testing for DAV", func() { + Context("with DAV configurations", func() { + var ( + endpoint string + user string + password string + ca string + secret string + ) + + BeforeEach(func() { + endpoint = os.Getenv("DAV_ENDPOINT") + user = os.Getenv("DAV_USER") + password = os.Getenv("DAV_PASSWORD") + ca = os.Getenv("DAV_CA_CERT") + secret = os.Getenv("DAV_SECRET") + + // Skip tests if environment variables are not set + if endpoint == "" || user == "" || password == "" { + Skip("Skipping DAV integration tests - environment variables not set (DAV_ENDPOINT, DAV_USER, DAV_PASSWORD required)") + } + }) + + It("Blobstore lifecycle works with basic config", func() { + cfg := &config.Config{ + Endpoint: endpoint, + User: user, + Password: password, + TLS: config.TLS{ + Cert: config.Cert{ + CA: ca, + }, + }, + } + integration.AssertLifecycleWorks(cliPath, cfg) + }) + + It("Blobstore lifecycle works with custom retry attempts", func() { + cfg := &config.Config{ + Endpoint: endpoint, + User: user, + Password: password, + RetryAttempts: 5, + TLS: config.TLS{ + Cert: config.Cert{ + CA: ca, + }, + }, + } + integration.AssertLifecycleWorks(cliPath, cfg) + }) + + It("Invoking `get` on a non-existent-key fails with basic config", func() { + cfg := &config.Config{ + Endpoint: endpoint, + User: user, + Password: password, + TLS: config.TLS{ + Cert: config.Cert{ + CA: ca, + }, + }, + } + integration.AssertGetNonexistentFails(cliPath, cfg) + }) + + It("Invoking `get` on a non-existent-key fails with custom retry attempts", func() { + cfg := &config.Config{ + Endpoint: endpoint, + User: user, + Password: password, + RetryAttempts: 5, + TLS: config.TLS{ + Cert: config.Cert{ + CA: ca, + }, + }, + } + integration.AssertGetNonexistentFails(cliPath, cfg) + }) + + It("Invoking `delete` on a non-existent-key does not fail with basic config", func() { + cfg := &config.Config{ + Endpoint: endpoint, + User: user, + Password: password, + TLS: config.TLS{ + Cert: config.Cert{ + CA: ca, + }, + }, + } + integration.AssertDeleteNonexistentWorks(cliPath, cfg) + }) + + It("Invoking `delete` on a non-existent-key does not fail with custom retry attempts", func() { + cfg := &config.Config{ + Endpoint: endpoint, + User: user, + Password: password, + RetryAttempts: 5, + TLS: config.TLS{ + Cert: config.Cert{ + CA: ca, + }, + }, + } + integration.AssertDeleteNonexistentWorks(cliPath, cfg) + }) + + It("Blobstore list and delete-recursive lifecycle works with basic config", func() { + cfg := &config.Config{ + Endpoint: endpoint, + User: user, + Password: password, + TLS: config.TLS{ + Cert: config.Cert{ + CA: ca, + }, + }, + } + integration.AssertOnListDeleteLifecycle(cliPath, cfg) + }) + + It("Blobstore list and delete-recursive lifecycle works with custom retry attempts", func() { + cfg := &config.Config{ + Endpoint: endpoint, + User: user, + Password: password, + RetryAttempts: 5, + TLS: config.TLS{ + Cert: config.Cert{ + CA: ca, + }, + }, + } + integration.AssertOnListDeleteLifecycle(cliPath, cfg) + }) + + It("Invoking `ensure-storage-exists` works with basic config", func() { + cfg := &config.Config{ + Endpoint: endpoint, + User: user, + Password: password, + TLS: config.TLS{ + Cert: config.Cert{ + CA: ca, + }, + }, + } + integration.AssertEnsureStorageExists(cliPath, cfg) + }) + + It("Invoking `ensure-storage-exists` works with custom retry attempts", func() { + cfg := &config.Config{ + Endpoint: endpoint, + User: user, + Password: password, + RetryAttempts: 5, + TLS: config.TLS{ + Cert: config.Cert{ + CA: ca, + }, + }, + } + integration.AssertEnsureStorageExists(cliPath, cfg) + }) + + Context("with signed URL support", func() { + BeforeEach(func() { + if secret == "" { + Skip("DAV_SECRET not set - skipping signed URL tests") + } + }) + + It("Invoking `sign` returns a signed URL with secret for signed URLs", func() { + cfg := &config.Config{ + Endpoint: endpoint, + User: user, + Password: password, + Secret: secret, + TLS: config.TLS{ + Cert: config.Cert{ + CA: ca, + }, + }, + } + integration.AssertOnSignedURLs(cliPath, cfg) + }) + }) + }) +}) diff --git a/dav/integration/integration_suite_test.go b/dav/integration/integration_suite_test.go new file mode 100644 index 0000000..8664bc7 --- /dev/null +++ b/dav/integration/integration_suite_test.go @@ -0,0 +1,31 @@ +package integration_test + +import ( + "io" + "log" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" +) + +func TestIntegration(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DAV Integration Suite") +} + +var cliPath string + +var _ = BeforeSuite(func() { + // Suppress logs during integration tests + log.SetOutput(io.Discard) + + var err error + cliPath, err = gexec.Build("github.com/cloudfoundry/storage-cli") + Expect(err).ShouldNot(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + gexec.CleanupBuildArtifacts() +}) diff --git a/dav/integration/testdata/Dockerfile b/dav/integration/testdata/Dockerfile new file mode 100644 index 0000000..684c242 --- /dev/null +++ b/dav/integration/testdata/Dockerfile @@ -0,0 +1,19 @@ +FROM httpd:2.4 + +# Create required directories +RUN mkdir -p /usr/local/apache2/certs \ + && mkdir -p /usr/local/apache2/webdav \ + && mkdir -p /usr/local/apache2/var \ + && chmod 777 /usr/local/apache2/webdav \ + && chmod 777 /usr/local/apache2/var + +# Copy configuration files +COPY httpd.conf /usr/local/apache2/conf/httpd.conf +COPY htpasswd /usr/local/apache2/htpasswd +COPY certs/server.crt /usr/local/apache2/certs/server.crt +COPY certs/server.key /usr/local/apache2/certs/server.key + +# Ensure htpasswd file has correct permissions +RUN chmod 644 /usr/local/apache2/htpasswd + +EXPOSE 443 diff --git a/dav/integration/testdata/certs/server.crt b/dav/integration/testdata/certs/server.crt new file mode 100644 index 0000000..1a2d48f --- /dev/null +++ b/dav/integration/testdata/certs/server.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDmzCCAoOgAwIBAgIUSYI1aoMpFE3lFX1E9vEE5/3gj3swDQYJKoZIhvcNAQEL +BQAwTjELMAkGA1UEBhMCVVMxDTALBgNVBAgMBFRlc3QxDTALBgNVBAcMBFRlc3Qx +DTALBgNVBAoMBFRlc3QxEjAQBgNVBAMMCWxvY2FsaG9zdDAgFw0yNjAzMTYxNDQz +MjhaGA8yMTI2MDIyMDE0NDMyOFowTjELMAkGA1UEBhMCVVMxDTALBgNVBAgMBFRl +c3QxDTALBgNVBAcMBFRlc3QxDTALBgNVBAoMBFRlc3QxEjAQBgNVBAMMCWxvY2Fs +aG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMBpU1j421YuJLdR +GcdmAcAxPq5s9XYygen3BAUQ4nSkwTY6ietiSZww8L9zsPdrfGK7IAXlwaNjXZ5R +0hZmKZx9qqFCtFI6f0gKcZj7ftApj1RsHz/sxW377/4RlBZJoPzrQgKEZ4wPZjPw +TYXSXf3ilkXNpKZWYf7fCF3Pu5CK/FwxA+BqaJPi39myijbkNrfg/fEA2dWa70Zr +4M/UL4qVYWGFbtHAVgJTd5Hr6YpzgKFuBvG4QCEhMfjxJ/mWpVmfIiERRx7iw74u +zArCWvW6VZeynFpVy7oznX/FpuFZPIsFWS5Z9MSidB8qL+nH+l7hf2XQfCahyj3Q +Uhkp8NsCAwEAAaNvMG0wHQYDVR0OBBYEFLCh++1VKISYQP2OglSoNUKnmCDtMB8G +A1UdIwQYMBaAFLCh++1VKISYQP2OglSoNUKnmCDtMA8GA1UdEwEB/wQFMAMBAf8w +GgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3DQEBCwUAA4IBAQCk +g5SNgqZzC+IRlLIcrOW5QbEmWPkkvrPtQzAJKyduwTM4mJBrOOvJSA1QQdO9wvx1 +5TgZWAel6sOG1SSh7DoXFlig4tr+b7rvEy043km4mHDaHHfFfk8yoZxvvrouQ0OB +n0O8e8+6TFRM1Qk2WAPSPbEEx0pDgag7+NHEKSqmkTlGmCBTLydWnEk4lQsisX0Y +MgpPaECVsWZEeSh0+G+Xq8NZUXE6U2KTXJla1VuKuFkhMZSMHqCeIkSGnJtA1rfv +jKIVMdIah/i52PNmH8amAgWzRakvdehWMA1xWJD7pJalwbmmu3LjQ0OpGTuGooL3 +fzJRnA5FdaBHQ/QUbNpg +-----END CERTIFICATE----- diff --git a/dav/integration/testdata/certs/server.key b/dav/integration/testdata/certs/server.key new file mode 100644 index 0000000..2a157ed --- /dev/null +++ b/dav/integration/testdata/certs/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDAaVNY+NtWLiS3 +URnHZgHAMT6ubPV2MoHp9wQFEOJ0pME2OonrYkmcMPC/c7D3a3xiuyAF5cGjY12e +UdIWZimcfaqhQrRSOn9ICnGY+37QKY9UbB8/7MVt++/+EZQWSaD860IChGeMD2Yz +8E2F0l394pZFzaSmVmH+3whdz7uQivxcMQPgamiT4t/Zsoo25Da34P3xANnVmu9G +a+DP1C+KlWFhhW7RwFYCU3eR6+mKc4ChbgbxuEAhITH48Sf5lqVZnyIhEUce4sO+ +LswKwlr1ulWXspxaVcu6M51/xabhWTyLBVkuWfTEonQfKi/px/pe4X9l0Hwmoco9 +0FIZKfDbAgMBAAECggEAXnmY3h/a+pbPml8s7DZO98J2R4jigXXNSkbqZ15iAun+ +oJTfsX7iK9nv6+FvbB0Pxx6gW6Tzjjk0946vPCZPmjIt/N5W0eU9J+9Q1c/u9WDi +qo4oTegBDL7emP6imsruTCFrmPbQLPpGsYv1VJb1ZbrDFGUjyjSyC0YRwpZEP3TU +lFGwXNrvHe1dlwYWgl3Fr6vCttWwF//kNCkGaemRF27BQ2HwZEsH1AKfXNrPrSls +hclvlZpYsXn809B3OuNUH7T9UkWJ1RuRhv1Bl2WFaOzp0PwnkF2PfbzgCMVTM71T +xgNLQIwfAPKFkiDL2SSudzf23NQmzyjNlK0t7xWz6QKBgQDsHYBnj6wLlGZsDrVm +DBuK4+4OhTBlENEuNemZi/sotxIe09xjaZQK2VPyQRDlp9k+PTDBBNUPxJ+zA3t2 +wZTngZtpY/0mVG2RvyTJVVi3QwoDGQRu73X2qXI8DUZW263gVo5cJXQzrW+2B5QP +8wV7fTAEp5RgDAKG6q2vUX+ZhQKBgQDQnZjLXCIuXzX4bo5zwh58m8eTH+pmSsxx +Bhfkk9s69WeFhYtK9B9vwU9+kuB36pTFxmL8ufsTsO6ghk+mEf+6jtKzGqDiR4Pa +K8pes2o4w7wdRftVW59m3xghFQZ6CuZet1cVsrRi0UHal22p00ISuC2VqIARi2kK +VfACmhE+3wKBgQDEWuBevz8/PgFTGYRHQghhn51oW+DcG3kp6dHDTILo4B3knyFn +VvSzdPp3ux53Lffe53o0+nTJMSXx9BJntyLCx6jbozhx+MJJ82B/QkeN1+VqoBJs +wx0hrNaAFDYLo5Lcvn6TKN6S30fIZFMAVISZpokZRdeBbFtpoZ4g7zCjFQKBgFB6 +7A5QHfOr0YNlC1nHIsHJy0WMA36xDovv4NnS3LmzINvW+DTTVyli90sKWMSKYBio +f1mmWiFvma+eAS49NV4AaXKlLDn/gvNw/2Jnbuw1PuZAMETu0uD54jIpDVWZzOPv +cQ4y4fpZZkFxN+JTWOMl4Jgi6D1cfgp5ut0WGN8bAoGANorszuao8a7tGkfuq45A +LG1eKiWMLdzvwiDzpYeVbRDxFF+o3KdqAnzagVOcz6cLniauPyaYUAWk4t4Yxi+B +fJTe2H/mjY1Y3Wqn3cLs/+2oPrEjOxI3rWLx8mVn/kwO6Ubh9S2/pkGcRphi+lDF +heiUQERU1zMG/8MWBW+IJIE= +-----END PRIVATE KEY----- diff --git a/dav/integration/testdata/htpasswd b/dav/integration/testdata/htpasswd new file mode 100644 index 0000000..07d9c74 --- /dev/null +++ b/dav/integration/testdata/htpasswd @@ -0,0 +1 @@ +testuser:$apr1$8tlNa1sl$bo3IwQf9K1Wzk89IiKt/Z0 diff --git a/dav/integration/testdata/httpd.conf b/dav/integration/testdata/httpd.conf new file mode 100644 index 0000000..dd36905 --- /dev/null +++ b/dav/integration/testdata/httpd.conf @@ -0,0 +1,43 @@ +ServerRoot "/usr/local/apache2" +Listen 443 + +LoadModule mpm_event_module modules/mod_mpm_event.so +LoadModule authn_file_module modules/mod_authn_file.so +LoadModule authn_core_module modules/mod_authn_core.so +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule authz_user_module modules/mod_authz_user.so +LoadModule authz_core_module modules/mod_authz_core.so +LoadModule auth_basic_module modules/mod_auth_basic.so +LoadModule dav_module modules/mod_dav.so +LoadModule dav_fs_module modules/mod_dav_fs.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule ssl_module modules/mod_ssl.so +LoadModule unixd_module modules/mod_unixd.so +LoadModule dir_module modules/mod_dir.so + +User daemon +Group daemon + +DAVLockDB /usr/local/apache2/var/DavLock + + + SSLRandomSeed startup builtin + SSLRandomSeed connect builtin + + + + SSLEngine on + SSLCertificateFile /usr/local/apache2/certs/server.crt + SSLCertificateKeyFile /usr/local/apache2/certs/server.key + + DocumentRoot "/usr/local/apache2/webdav" + + + Dav On + Options +Indexes + AuthType Basic + AuthName "WebDAV" + AuthUserFile /usr/local/apache2/htpasswd + Require valid-user + + diff --git a/dav/integration/utils.go b/dav/integration/utils.go new file mode 100644 index 0000000..2d4d22d --- /dev/null +++ b/dav/integration/utils.go @@ -0,0 +1,74 @@ +package integration + +import ( + "encoding/json" + "math/rand" + "os" + "os/exec" + "time" + + "github.com/cloudfoundry/storage-cli/dav/config" + + . "github.com/onsi/ginkgo/v2" //nolint:staticcheck + "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" +) + +const alphaNum = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +// GenerateRandomString generates a random string of desired length (default: 25) +func GenerateRandomString(params ...int) string { + size := 25 + if len(params) == 1 { + size = params[0] + } + + randBytes := make([]byte, size) + for i := range randBytes { + randBytes[i] = alphaNum[rand.Intn(len(alphaNum))] + } + return string(randBytes) +} + +// MakeConfigFile creates a config file from a DAV config struct +func MakeConfigFile(cfg *config.Config) string { + cfgBytes, err := json.Marshal(cfg) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + tmpFile, err := os.CreateTemp("", "davcli-test") + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = tmpFile.Write(cfgBytes) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + err = tmpFile.Close() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + return tmpFile.Name() +} + +// MakeContentFile creates a temporary file with content to upload to WebDAV +func MakeContentFile(content string) string { + tmpFile, err := os.CreateTemp("", "davcli-test-content") + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = tmpFile.Write([]byte(content)) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + err = tmpFile.Close() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + return tmpFile.Name() +} + +// RunCli runs the storage-cli and outputs the session after waiting for it to finish +func RunCli(cliPath string, configPath string, storageType string, subcommand string, args ...string) (*gexec.Session, error) { + cmdArgs := []string{ + "-c", + configPath, + "-s", + storageType, + subcommand, + } + cmdArgs = append(cmdArgs, args...) + command := exec.Command(cliPath, cmdArgs...) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + if err != nil { + return nil, err + } + session.Wait(1 * time.Minute) + return session, nil +} diff --git a/dav/signer/signer.go b/dav/signer/signer.go index 55f203e..9be8372 100644 --- a/dav/signer/signer.go +++ b/dav/signer/signer.go @@ -2,6 +2,7 @@ package signer import ( "crypto/hmac" + "crypto/md5" "crypto/sha256" "encoding/base64" "fmt" @@ -17,47 +18,141 @@ type Signer interface { } type signer struct { - secret string + secret string + signingMethod string // "sha256" (default) or "md5" } func NewSigner(secret string) Signer { return &signer{ - secret: secret, + secret: secret, + signingMethod: "sha256", // Default to BOSH } } -func (s *signer) generateSignature(prefixedBlobID, verb string, timeStamp time.Time, expires int) string { - verb = strings.ToUpper(verb) - signature := fmt.Sprintf("%s%s%d%d", verb, prefixedBlobID, timeStamp.Unix(), expires) - hmac := hmac.New(sha256.New, []byte(s.secret)) - hmac.Write([]byte(signature)) - sigBytes := hmac.Sum(nil) - return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(sigBytes) +func NewSignerWithMethod(secret string, signingMethod string) Signer { + if signingMethod == "" { + signingMethod = "sha256" // Default to BOSH + } + return &signer{ + secret: secret, + signingMethod: strings.ToLower(signingMethod), + } } +// GenerateSignedURL generates nginx secure_link or secure_link_hmac compatible signed URLs +// Supports both BOSH (SHA256 HMAC) and CAPI (MD5) formats func (s *signer) GenerateSignedURL(endpoint, prefixedBlobID, verb string, timeStamp time.Time, expiresAfter time.Duration) (string, error) { verb = strings.ToUpper(verb) - if verb != "GET" && verb != "PUT" { - return "", fmt.Errorf("action not implemented: %s. Available actions are 'GET' and 'PUT'", verb) + if verb != "GET" && verb != "PUT" && verb != "HEAD" { + return "", fmt.Errorf("action not implemented: %s. Available actions are 'GET', 'PUT', and 'HEAD'", verb) + } + + if s.signingMethod == "md5" { + return s.generateMD5SignedURL(endpoint, prefixedBlobID, verb, timeStamp, expiresAfter) } + return s.generateSHA256SignedURL(endpoint, prefixedBlobID, verb, timeStamp, expiresAfter) +} +// generateSHA256SignedURL generates BOSH-compatible SHA256 HMAC signed URLs +// Uses nginx secure_link_hmac module format +func (s *signer) generateSHA256SignedURL(endpoint, prefixedBlobID, verb string, timeStamp time.Time, expiresAfter time.Duration) (string, error) { endpoint = strings.TrimSuffix(endpoint, "/") + timestamp := timeStamp.Unix() expiresAfterSeconds := int(expiresAfter.Seconds()) - signature := s.generateSignature(prefixedBlobID, verb, timeStamp, expiresAfterSeconds) + // Parse the endpoint to extract any existing path blobURL, err := url.Parse(endpoint) if err != nil { return "", err } - blobURL.Path = path.Join(blobURL.Path, "signed", prefixedBlobID) + + // Normalize base path from endpoint + basePath := strings.TrimSuffix(blobURL.Path, "/") + + // Build the full path: /signed/basePath/blobID + // The /signed prefix must come FIRST for nginx secure_link_hmac + fullPath := path.Join("/signed", basePath, prefixedBlobID) + + // Generate HMAC-SHA256 signature using BOSH secure_link_hmac format: + // hmac_sha256("{verb}{blobID}{timestamp}{duration}", secret) + // Note: Uses duration in seconds, not absolute expiration timestamp + signatureInput := fmt.Sprintf("%s%s%d%d", verb, prefixedBlobID, timestamp, expiresAfterSeconds) + h := hmac.New(sha256.New, []byte(s.secret)) + h.Write([]byte(signatureInput)) + hmacSum := h.Sum(nil) + signature := sanitizeBase64(base64.StdEncoding.EncodeToString(hmacSum)) + + blobURL.Path = fullPath + req, err := http.NewRequest(verb, blobURL.String(), nil) if err != nil { return "", err } + + // Add query parameters for nginx secure_link_hmac q := req.URL.Query() q.Add("st", signature) - q.Add("ts", fmt.Sprintf("%d", timeStamp.Unix())) + q.Add("ts", fmt.Sprintf("%d", timestamp)) q.Add("e", fmt.Sprintf("%d", expiresAfterSeconds)) req.URL.RawQuery = q.Encode() + + return req.URL.String(), nil +} + +// generateMD5SignedURL generates CAPI-compatible MD5 signed URLs +// Uses nginx secure_link module format +func (s *signer) generateMD5SignedURL(endpoint, prefixedBlobID, verb string, timeStamp time.Time, expiresAfter time.Duration) (string, error) { + endpoint = strings.TrimSuffix(endpoint, "/") + expires := timeStamp.Unix() + int64(expiresAfter.Seconds()) + + // Determine the path prefix based on verb + var pathPrefix string + if verb == "GET" || verb == "HEAD" { + pathPrefix = "/read" + } else { + pathPrefix = "/write" + } + + // Parse the endpoint to extract any existing path (e.g., /cc-droplets) + blobURL, err := url.Parse(endpoint) + if err != nil { + return "", err + } + + // Normalize base path from endpoint + basePath := strings.TrimSuffix(blobURL.Path, "/") + + // Build complete path: /write/cc-droplets/08/d3/... or /read/cc-droplets/08/d3/... + // The path prefix (/write or /read) must come FIRST for nginx secure_link + completePath := path.Join(pathPrefix, basePath, prefixedBlobID) + + // Generate MD5 signature using CAPI blobstore_url_signer format: + // md5("{expires}{path} {secret}") + signatureInput := fmt.Sprintf("%d%s %s", expires, completePath, s.secret) + md5sum := md5.Sum([]byte(signatureInput)) + signature := sanitizeBase64(base64.StdEncoding.EncodeToString(md5sum[:])) + + blobURL.Path = completePath + + req, err := http.NewRequest(verb, blobURL.String(), nil) + if err != nil { + return "", err + } + + // Add query parameters for nginx secure_link + q := req.URL.Query() + q.Add("md5", signature) + q.Add("expires", fmt.Sprintf("%d", expires)) + req.URL.RawQuery = q.Encode() + return req.URL.String(), nil } + +// sanitizeBase64 converts base64 to URL-safe format for nginx secure_link_hmac +// Matches BOSH format: / -> _, + -> -, remove = +func sanitizeBase64(input string) string { + str := strings.ReplaceAll(input, "/", "_") + str = strings.ReplaceAll(str, "+", "-") + str = strings.ReplaceAll(str, "=", "") + return str +} diff --git a/dav/signer/signer_test.go b/dav/signer/signer_test.go index 197a55d..41bbfd6 100644 --- a/dav/signer/signer_test.go +++ b/dav/signer/signer_test.go @@ -12,13 +12,18 @@ var _ = Describe("Signer", func() { secret := "mefq0umpmwevpv034m890j34m0j0-9!fijm434j99j034mjrwjmv9m304mj90;2ef32buf32gbu2i3" objectID := "fake-object-id" verb := "get" - signer := signer.NewSigner(secret) duration := time.Duration(15 * time.Minute) timeStamp := time.Date(2019, 8, 26, 11, 11, 0, 0, time.UTC) path := "https://api.example.com/" - Context("HMAC Signed URL", func() { + Context("SHA256 HMAC Signed URL (BOSH format - default)", func() { + signer := signer.NewSigner(secret) + // Expected signature for: HMAC-SHA256("GETfake-object-id1566817860900", secret) + // timestamp: 1566817860 (2019-08-26 11:11:00 UTC) + // duration: 900 seconds (15 minutes) + // Signature matches BOSH secure_link_hmac format: $request_method$object_id$arg_ts$arg_e + // where arg_e is the DURATION in seconds, not absolute expiration expected := "https://api.example.com/signed/fake-object-id?e=900&st=BxLKZK_dTSLyBis1pAjdwq4aYVrJvXX6vvLpdCClGYo&ts=1566817860" It("Generates a properly formed URL", func() { @@ -27,4 +32,30 @@ var _ = Describe("Signer", func() { Expect(actual).To(Equal(expected)) }) }) + + Context("SHA256 HMAC Signed URL (BOSH format - explicit)", func() { + signer := signer.NewSignerWithMethod(secret, "sha256") + + expected := "https://api.example.com/signed/fake-object-id?e=900&st=BxLKZK_dTSLyBis1pAjdwq4aYVrJvXX6vvLpdCClGYo&ts=1566817860" + + It("Generates a properly formed URL", func() { + actual, err := signer.GenerateSignedURL(path, objectID, verb, timeStamp, duration) + Expect(err).To(BeNil()) + Expect(actual).To(Equal(expected)) + }) + }) + + Context("MD5 Signed URL (CAPI format)", func() { + signer := signer.NewSignerWithMethod(secret, "md5") + + // Expected signature for: md5("1566818760/read/fake-object-id {secret}") + // expires: 1566818760 (timestamp + 900 seconds) + expected := "https://api.example.com/read/fake-object-id?expires=1566818760&md5=WQ0QdFWpV_nxXrlyHPOu6g" + + It("Generates a properly formed URL", func() { + actual, err := signer.GenerateSignedURL(path, objectID, verb, timeStamp, duration) + Expect(err).To(BeNil()) + Expect(actual).To(Equal(expected)) + }) + }) }) diff --git a/storage/factory.go b/storage/factory.go index a64634d..ee1a6f5 100644 --- a/storage/factory.go +++ b/storage/factory.go @@ -5,13 +5,11 @@ import ( "fmt" "os" - boshlog "github.com/cloudfoundry/bosh-utils/logger" alioss "github.com/cloudfoundry/storage-cli/alioss/client" aliossconfig "github.com/cloudfoundry/storage-cli/alioss/config" azurebs "github.com/cloudfoundry/storage-cli/azurebs/client" azureconfigbs "github.com/cloudfoundry/storage-cli/azurebs/config" - davapp "github.com/cloudfoundry/storage-cli/dav/app" - davcmd "github.com/cloudfoundry/storage-cli/dav/cmd" + davclient "github.com/cloudfoundry/storage-cli/dav/client" davconfig "github.com/cloudfoundry/storage-cli/dav/config" gcs "github.com/cloudfoundry/storage-cli/gcs/client" gcsconfig "github.com/cloudfoundry/storage-cli/gcs/config" @@ -92,12 +90,12 @@ var newDavClient = func(configFile *os.File) (Storager, error) { return nil, err } - logger := boshlog.NewLogger(boshlog.LevelNone) - cmdFactory := davcmd.NewFactory(logger) - - cmdRunner := davcmd.NewRunner(cmdFactory) + davClient, err := davclient.New(davConfig) + if err != nil { + return nil, err + } - return davapp.New(cmdRunner, davConfig), nil + return davClient, nil } func NewStorageClient(storageType string, configFile *os.File) (Storager, error) {