Merge pull request #12102 from kobergj/TestBlobstoreCli

Add CLI command to test Blobstore

changelog/unreleased/enhancement-storage-users-blobstore-cli.md

@@ -0,0 +1,24 @@
+Enhancement: Add blobstore CLI commands to storage-users service
+
+Added two new CLI commands under `ocis storage-users blobstore` to help
+operators verify and inspect the configured blobstore without needing
+direct access to the underlying storage system.
+
+`blobstore check` performs a full upload/download/delete round-trip using
+a random payload. The payload size is configurable via `--blob-size` and
+accepts human-readable values such as `64`, `1KB` or `4MiB` (default: 64 bytes).
+
+`blobstore get` downloads a specific blob by its ID to verify it is
+readable. The blob can be identified either with `--blob-id` and
+`--space-id`, or by passing the raw blob path from a log line directly
+via `--path`. Both the s3ng key format
+(`<spaceID>/<pathified_blobID>`) and the ocis filesystem path format
+(`…/spaces/<pathified_spaceID>/blobs/<pathified_blobID>`) are accepted.
+When using the s3ng driver without a known blob size, an automatic retry
+with the actual size is performed on a size mismatch.
+
+Both commands read the existing service configuration, so they always
+target the same blobstore as the running service. Only the `ocis` and
+`s3ng` storage drivers are supported.
+
+https://github.com/owncloud/ocis/pull/12102

services/storage-users/README.md

@@ -184,6 +184,92 @@ ocis storage-users uploads delete-stale-nodes --dry-run=false --verbose
 ```
 
 
+### Check and Inspect the Blobstore
+
+The `blobstore` command group provides tools to verify connectivity to the configured blobstore and to inspect individual blobs. Both the `ocis` and `s3ng` storage drivers are supported.
+
+```bash
+ocis storage-users blobstore <command>
+```
+
+```plaintext
+COMMANDS:
+   check  check blobstore connectivity via an upload/download/delete round-trip
+   get    get a blob from the blobstore by ID
+```
+
+#### Check
+
+Verifies that the blobstore is reachable and fully operational by uploading a random blob, downloading and verifying it, then deleting it again. All three steps must succeed.
+
+```bash
+ocis storage-users blobstore check [command options]
+```
+
+```
+OPTIONS:
+   --blob-size value  size of the random blob to upload, e.g. 64, 1KB, 1MB, 4MiB (default: "64")
+   --help, -h         show help
+```
+
+**Examples:**
+
+```bash
+# Basic connectivity check using the default 64-byte payload
+ocis storage-users blobstore check
+
+# Use a larger payload to also stress-test throughput
+ocis storage-users blobstore check --blob-size=4MiB
+```
+
+```plaintext
+Uploading test blob: spaceID=a5c9bd5c-7348-4e9e-a462-d4d9e5287d01 blobID=3f1e5a82-b7e3-4c91-a110-1d5e2f3a4b6c
+Upload: OK
+Download and verify: OK
+Delete: OK
+Blobstore check successful.
+```
+
+#### Get
+
+Downloads a single blob by its ID to verify it exists and is readable. Useful when investigating errors in log lines that contain a blob path such as:
+
+```
+decomposedfs: error download blob '04ba5496-...': blob path: b19ec764-.../61/03/ab/c3/-b08a-...: The specified key does not exist.
+```
+
+The blob can be identified either by passing the raw path from the log line with `--path`, or by supplying `--blob-id` and `--space-id` individually.
+
+```bash
+ocis storage-users blobstore get [command options]
+```
+
+```
+OPTIONS:
+   --path value      blobstore path as it appears in log lines; spaceID and blobID are extracted automatically.
+                     Supports both s3ng format ("<spaceID>/<pathified_blobID>") and ocis format
+                     ("…/spaces/<pathified_spaceID>/blobs/<pathified_blobID>").
+   --blob-id value   blob ID to download (required when --path is not set)
+   --space-id value  space ID the blob belongs to (required when --path is not set)
+   --blob-size value expected blob size in bytes; only needed for the s3ng driver when the size is known
+                     upfront. If omitted or wrong, a size mismatch triggers one automatic retry with the
+                     actual size returned by s3ng. (default: 0)
+   --help, -h        show help
+```
+
+**Examples:**
+
+```bash
+# Identify a blob using the path from a log line (s3ng)
+ocis storage-users blobstore get --path="b19ec764-5398-458a-8ff1-1925bd906999/61/03/ab/c3/-b08a-4556-9937-2bf3065c1202"
+
+# Identify a blob using the full filesystem path from a log line (ocis driver)
+ocis storage-users blobstore get --path="/var/lib/ocis/storage/users/spaces/b1/9ec764-5398-458a-8ff1-1925bd906999/blobs/61/03/ab/c3/-b08a-4556-9937-2bf3065c1202"
+
+# Identify a blob using explicit IDs
+ocis storage-users blobstore get --space-id=b19ec764-5398-458a-8ff1-1925bd906999 --blob-id=6103abc3-b08a-4556-9937-2bf3065c1202
+```
+
 ### Manage Spaces
 
 This command set provides commands to manage spaces, including purging disabled spaces that exceed the retention period.

services/storage-users/pkg/command/blobstore.go

@@ -0,0 +1,286 @@
+package command
+
+import (
+	"bytes"
+	"crypto/rand"
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
+	"github.com/owncloud/ocis/v2/services/storage-users/pkg/config"
+	"github.com/owncloud/ocis/v2/services/storage-users/pkg/config/parser"
+	"github.com/owncloud/reva/v2/pkg/bytesize"
+	ocisbs "github.com/owncloud/reva/v2/pkg/storage/fs/ocis/blobstore"
+	s3bs "github.com/owncloud/reva/v2/pkg/storage/fs/s3ng/blobstore"
+	"github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/node"
+	"github.com/owncloud/reva/v2/pkg/storage/utils/decomposedfs/tree"
+	"github.com/urfave/cli/v2"
+)
+
+// Blobstore is the entry point for the blobstore command group.
+func Blobstore(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "blobstore",
+		Usage: "manage the blobstore",
+		Subcommands: []*cli.Command{
+			BlobstoreCheck(cfg),
+			BlobstoreGet(cfg),
+		},
+	}
+}
+
+// BlobstoreCheck uploads random bytes to a random path, downloads and
+// verifies them, then deletes them again.  All three steps must succeed.
+func BlobstoreCheck(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "check",
+		Usage: "check blobstore connectivity via an upload/download/delete round-trip",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "blob-size",
+				Usage: "size of the random blob to upload, e.g. 64, 1KB, 1MB, 4MiB",
+				Value: "64",
+			},
+		},
+		Before: func(c *cli.Context) error {
+			return configlog.ReturnFatal(parser.ParseConfig(cfg))
+		},
+		Action: func(c *cli.Context) error {
+			bs, err := initBlobstore(cfg)
+			if err != nil {
+				return err
+			}
+			size, err := bytesize.Parse(c.String("blob-size"))
+			if err != nil {
+				return fmt.Errorf("invalid --blob-size %q: %w", c.String("blob-size"), err)
+			}
+			return runBlobstoreRoundTrip(bs, int(size))
+		},
+	}
+}
+
+// BlobstoreGet downloads a specific blob by its ID to verify it is readable.
+func BlobstoreGet(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "get",
+		Usage: "get a blob from the blobstore by ID",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "path",
+				Usage: "blobstore path as it appears in log lines (e.g. \"<spaceID>/<pathified_blobID>\" for s3ng or \"…/spaces/<pathified_spaceID>/blobs/<pathified_blobID>\" for ocis); extracts --blob-id and --space-id automatically",
+			},
+			&cli.StringFlag{
+				Name:  "blob-id",
+				Usage: "blob ID to download (required when --path is not set)",
+			},
+			&cli.StringFlag{
+				Name:  "space-id",
+				Usage: "space ID the blob belongs to (required when --path is not set)",
+			},
+			&cli.Int64Flag{
+				Name: "blob-size",
+				Usage: "expected blob size in bytes; only needed for the s3ng driver when the size is known upfront. " +
+					"If omitted or wrong, a size mismatch will trigger one automatic retry with the actual size returned by s3ng.",
+			},
+		},
+		Before: func(c *cli.Context) error {
+			return configlog.ReturnFatal(parser.ParseConfig(cfg))
+		},
+		Action: func(c *cli.Context) error {
+			bs, err := initBlobstore(cfg)
+			if err != nil {
+				return err
+			}
+			blobID, spaceID := c.String("blob-id"), c.String("space-id")
+			if c.IsSet("path") {
+				spaceID, blobID, err = parseBlobPath(c.String("path"))
+				if err != nil {
+					return err
+				}
+			}
+			if blobID == "" || spaceID == "" {
+				return fmt.Errorf("either --path or both --blob-id and --space-id must be set")
+			}
+			return downloadBlob(bs, blobID, spaceID, c.Int64("blob-size"))
+		},
+	}
+}
+
+// parseBlobPath extracts spaceID and blobID from a blobstore path as it
+// appears in log lines.  Two formats are supported:
+//
+//   - S3NG:  "<spaceID>/<pathified_blobID>"
+//     e.g.  "b19ec764-5398-458a-8ff1-1925bd906999/61/03/ab/c3/-b08a-4556-9937-2bf3065c1202"
+//
+//   - OCIS:  any path containing "/spaces/<pathified_spaceID>/blobs/<pathified_blobID>"
+//     e.g.  "/var/lib/ocis/storage/users/spaces/b1/9ec764-.../blobs/61/03/ab/c3/-b08a-..."
+func parseBlobPath(path string) (spaceID, blobID string, err error) {
+	// OCIS paths contain both /spaces/ and /blobs/ as structural markers.
+	if _, rest, ok := strings.Cut(path, "/spaces/"); ok {
+		pathifiedSpaceID, pathifiedBlobID, ok := strings.Cut(rest, "/blobs/")
+		if !ok {
+			return "", "", fmt.Errorf("cannot parse ocis blob path %q: missing /blobs/ segment", path)
+		}
+		spaceID = depathify(pathifiedSpaceID, 1)
+		blobID = depathify(pathifiedBlobID, 4)
+		return
+	}
+
+	// S3NG paths: <spaceID>/<pathified_blobID>
+	var pathifiedBlobID string
+	spaceID, pathifiedBlobID, _ = strings.Cut(path, "/")
+	if pathifiedBlobID == "" {
+		return "", "", fmt.Errorf("cannot parse s3ng blob path %q: expected <spaceID>/<pathified_blobID>", path)
+	}
+	blobID = depathify(pathifiedBlobID, 4)
+	return
+}
+
+// depathify reverses the effect of lookup.Pathify(id, depth, 2):
+// it strips the directory separators that were inserted every two characters
+// up to the given depth.
+// e.g. depathify("61/03/ab/c3/-b08a-4556-9937-2bf3065c1202", 4)
+//
+//	→ "6103abc3-b08a-4556-9937-2bf3065c1202"
+func depathify(path string, depth int) string {
+	parts := strings.SplitN(path, "/", depth+1)
+	return strings.Join(parts, "")
+}
+
+// initBlobstore builds a tree.Blobstore from the service configuration.
+// Only the "ocis" and "s3ng" drivers are supported.
+func initBlobstore(cfg *config.Config) (tree.Blobstore, error) {
+	switch cfg.Driver {
+	case "ocis":
+		return ocisbs.New(cfg.Drivers.OCIS.Root)
+	case "s3ng":
+		return s3bs.New(
+			cfg.Drivers.S3NG.Endpoint,
+			cfg.Drivers.S3NG.Region,
+			cfg.Drivers.S3NG.Bucket,
+			cfg.Drivers.S3NG.AccessKey,
+			cfg.Drivers.S3NG.SecretKey,
+			s3bs.Options{
+				DisableContentSha256:  cfg.Drivers.S3NG.DisableContentSha256,
+				DisableMultipart:      cfg.Drivers.S3NG.DisableMultipart,
+				SendContentMd5:        cfg.Drivers.S3NG.SendContentMd5,
+				ConcurrentStreamParts: cfg.Drivers.S3NG.ConcurrentStreamParts,
+				NumThreads:            cfg.Drivers.S3NG.NumThreads,
+				PartSize:              cfg.Drivers.S3NG.PartSize,
+			},
+		)
+	default:
+		return nil, fmt.Errorf("blobstore operations are not supported for driver '%s'", cfg.Driver)
+	}
+}
+
+// blobSizeMismatchRe matches the error produced by the s3ng blobstore when the
+// retrieved object size does not match node.Blobsize, and captures the actual size.
+var blobSizeMismatchRe = regexp.MustCompile(`blob has unexpected size\. \d+ bytes expected, got (\d+) bytes`)
+
+// downloadBlob downloads a single blob identified by blobID, drains the reader
+// to surface any streaming errors, then closes it.
+// If the s3ng blobstore rejects the download due to a size mismatch, the actual
+// size is extracted from the error and the download is retried with the correct value.
+func downloadBlob(bs tree.Blobstore, blobID, spaceID string, blobSize int64) error {
+	n := &node.Node{
+		BlobID:   blobID,
+		SpaceID:  spaceID,
+		Blobsize: blobSize,
+	}
+	rc, err := bs.Download(n)
+	if err != nil {
+		if m := blobSizeMismatchRe.FindStringSubmatch(err.Error()); m != nil {
+			fmt.Printf("blob size mismatch, retrying with actual size %s bytes\n", m[1])
+			if _, err := fmt.Sscan(m[1], &n.Blobsize); err != nil {
+				return fmt.Errorf("download failed: could not parse actual blob size %q: %w", m[1], err)
+			}
+			rc, err = bs.Download(n)
+		}
+		if err != nil {
+			return fmt.Errorf("download failed: %w", err)
+		}
+	}
+	defer func() {
+		if cerr := rc.Close(); cerr != nil {
+			fmt.Fprintf(os.Stderr, "warning: failed to close blob reader for blob %s in space %s: %v\n", blobID, spaceID, cerr)
+		}
+	}()
+	if _, err := io.Copy(io.Discard, rc); err != nil {
+		return fmt.Errorf("download failed while reading: %w", err)
+	}
+	fmt.Println("Download: OK")
+	return nil
+}
+
+// runBlobstoreRoundTrip uploads random data, verifies it can be retrieved, and
+// then removes it again.  All three steps must succeed for the check to pass.
+func runBlobstoreRoundTrip(bs tree.Blobstore, blobSize int) error {
+	// 1. Generate random bytes that serve as the test payload.
+	data := make([]byte, blobSize)
+	if _, err := rand.Read(data); err != nil {
+		return fmt.Errorf("failed to generate random data: %w", err)
+	}
+
+	// Write the payload to a temporary file.  The OCIS blobstore may rename
+	// (move) this file into the blobstore, so os.Remove in the defer is a
+	// no-op for that driver – which is fine.
+	tmpFile, err := os.CreateTemp("", "ocis-blobstore-check-*")
+	if err != nil {
+		return fmt.Errorf("failed to create temp file: %w", err)
+	}
+	tmpPath := tmpFile.Name()
+	defer os.Remove(tmpPath) // best-effort cleanup
+
+	if _, err := tmpFile.Write(data); err != nil {
+		tmpFile.Close()
+		return fmt.Errorf("failed to write temp file: %w", err)
+	}
+	if err := tmpFile.Close(); err != nil {
+		return fmt.Errorf("failed to close temp file: %w", err)
+	}
+
+	// 2. Build a test node with random identifiers.
+	testNode := &node.Node{
+		SpaceID:  uuid.New().String(),
+		BlobID:   uuid.New().String(),
+		Blobsize: int64(len(data)),
+	}
+	fmt.Printf("Uploading test blob: spaceID=%s blobID=%s\n", testNode.SpaceID, testNode.BlobID)
+
+	// 3. Upload.
+	if err := bs.Upload(testNode, tmpPath); err != nil {
+		return fmt.Errorf("upload failed: %w", err)
+	}
+	fmt.Println("Upload: OK")
+
+	// 4. Download and verify.
+	rc, err := bs.Download(testNode)
+	if err != nil {
+		_ = bs.Delete(testNode)
+		return fmt.Errorf("download failed: %w", err)
+	}
+	downloaded, err := io.ReadAll(rc)
+	rc.Close()
+	if err != nil {
+		_ = bs.Delete(testNode)
+		return fmt.Errorf("failed to read downloaded data: %w", err)
+	}
+	if !bytes.Equal(data, downloaded) {
+		_ = bs.Delete(testNode)
+		return fmt.Errorf("data integrity check failed: downloaded content does not match uploaded content")
+	}
+	fmt.Println("Download and verify: OK")
+
+	// 5. Delete.
+	if err := bs.Delete(testNode); err != nil {
+		return fmt.Errorf("delete failed: %w", err)
+	}
+	fmt.Println("Delete: OK")
+	fmt.Println("Blobstore check successful.")
+	return nil
+}

services/storage-users/pkg/command/blobstore_test.go

@@ -0,0 +1,104 @@
+package command
+
+import (
+	"testing"
+)
+
+func Test_depathify(t *testing.T) {
+	tests := []struct {
+		name  string
+		path  string
+		depth int
+		want  string
+	}{
+		{
+			name:  "blob id depth 4",
+			path:  "61/03/ab/c3/-b08a-4556-9937-2bf3065c1202",
+			depth: 4,
+			want:  "6103abc3-b08a-4556-9937-2bf3065c1202",
+		},
+		{
+			name:  "space id depth 1",
+			path:  "b1/9ec764-5398-458a-8ff1-1925bd906999",
+			depth: 1,
+			want:  "b19ec764-5398-458a-8ff1-1925bd906999",
+		},
+		{
+			name:  "depth 0 is a no-op",
+			path:  "abcd-1234",
+			depth: 0,
+			want:  "abcd-1234",
+		},
+		{
+			name:  "fewer segments than depth leaves remainder intact",
+			path:  "ab/cd",
+			depth: 4,
+			want:  "abcd",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := depathify(tt.path, tt.depth)
+			if got != tt.want {
+				t.Errorf("depathify(%q, %d) = %q, want %q", tt.path, tt.depth, got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_parseBlobPath(t *testing.T) {
+	tests := []struct {
+		name        string
+		path        string
+		wantSpaceID string
+		wantBlobID  string
+		wantErr     bool
+	}{
+		{
+			// s3ng: <spaceID>/<pathified_blobID>
+			name:        "s3ng format",
+			path:        "b19ec764-5398-458a-8ff1-1925bd906999/61/03/ab/c3/-b08a-4556-9937-2bf3065c1202",
+			wantSpaceID: "b19ec764-5398-458a-8ff1-1925bd906999",
+			wantBlobID:  "6103abc3-b08a-4556-9937-2bf3065c1202",
+		},
+		{
+			// ocis: …/spaces/<pathified_spaceID>/blobs/<pathified_blobID>
+			name:        "ocis filesystem format",
+			path:        "/var/lib/ocis/storage/users/spaces/b1/9ec764-5398-458a-8ff1-1925bd906999/blobs/61/03/ab/c3/-b08a-4556-9937-2bf3065c1202",
+			wantSpaceID: "b19ec764-5398-458a-8ff1-1925bd906999",
+			wantBlobID:  "6103abc3-b08a-4556-9937-2bf3065c1202",
+		},
+		{
+			name:    "ocis format missing /blobs/ segment",
+			path:    "/var/lib/ocis/storage/users/spaces/b1/9ec764-5398-458a-8ff1-1925bd906999/noblobs/61/03/ab/c3",
+			wantErr: true,
+		},
+		{
+			name:    "s3ng format missing blob id (no slash after spaceID)",
+			path:    "b19ec764-5398-458a-8ff1-1925bd906999",
+			wantErr: true,
+		},
+		{
+			name:    "empty path",
+			path:    "",
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotSpaceID, gotBlobID, err := parseBlobPath(tt.path)
+			if (err != nil) != tt.wantErr {
+				t.Fatalf("parseBlobPath(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr)
+			}
+			if err != nil {
+				return
+			}
+			if gotSpaceID != tt.wantSpaceID {
+				t.Errorf("parseBlobPath(%q) spaceID = %q, want %q", tt.path, gotSpaceID, tt.wantSpaceID)
+			}
+			if gotBlobID != tt.wantBlobID {
+				t.Errorf("parseBlobPath(%q) blobID = %q, want %q", tt.path, gotBlobID, tt.wantBlobID)
+			}
+		})
+	}
+}

services/storage-users/pkg/command/root.go

@@ -18,6 +18,7 @@ func GetCommands(cfg *config.Config) cli.Commands {
 		Uploads(cfg),
 		TrashBin(cfg),
 		Spaces(cfg),
+		Blobstore(cfg),
 
 		// infos about this service
 		Health(cfg),