Merge branch 'main-v1' into merge-v1

2026-04-25 12:44:54 +02:00 · 2026-01-24 09:47:11 +01:00
parent 68876f0b08 c685a2cbaf
commit cd641a9ed2
17 changed files with 115 additions and 50 deletions
--- a/.woodpecker/debug.yaml
+++ b/.woodpecker/debug.yaml
@@ -12,32 +12,32 @@ when:

 steps:
  - name: check formatting
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    commands:
-      - nix-shell --attr devShell --run "cargo fmt -- --check"
+      - nix-build -j4 --attr flakePackages.fmt

  - name: build
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    commands:
      - nix-build -j4 --attr flakePackages.dev

  - name: unit + func tests (lmdb)
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    commands:
      - nix-build -j4 --attr flakePackages.tests-lmdb

  - name: unit + func tests (sqlite)
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    commands:
      - nix-build -j4 --attr flakePackages.tests-sqlite

  - name: unit + func tests (fjall)
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    commands:
      - nix-build -j4 --attr flakePackages.tests-fjall

  - name: integration tests
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    commands:
      - nix-build -j4 --attr flakePackages.dev
      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
--- a/.woodpecker/publish.yaml
+++ b/.woodpecker/publish.yaml
@@ -11,7 +11,7 @@ depends_on:

 steps:
  - name: refresh-index
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    environment:
      AWS_ACCESS_KEY_ID:
        from_secret: garagehq_aws_access_key_id
@@ -22,7 +22,7 @@ steps:
      - nix-shell --attr ci --run "refresh_index"

  - name: multiarch-docker
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    environment:
      DOCKER_AUTH:
        from_secret: docker_auth
--- a/.woodpecker/release.yaml
+++ b/.woodpecker/release.yaml
@@ -19,17 +19,17 @@ matrix:

 steps:
  - name: build
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    commands:
      - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}

  - name: check is static binary
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    commands:
      - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"

  - name: integration tests
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    commands:
      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
    when:
@@ -39,7 +39,7 @@ steps:
          ARCH: i386

  - name: upgrade tests from v1.0.0
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    commands:
      - nix-shell --attr ci --run "./script/test-upgrade.sh v1.0.0 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
    when:
@@ -47,7 +47,7 @@ steps:
          ARCH: amd64

  - name: upgrade tests from v0.8.4
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    commands:
      - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
    when:
@@ -55,7 +55,7 @@ steps:
          ARCH: amd64

  - name: push static binary
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    environment:
      TARGET: "${TARGET}"
      AWS_ACCESS_KEY_ID:
@@ -66,7 +66,7 @@ steps:
      - nix-shell --attr ci --run "to_s3"

  - name: docker build and publish
-    image: nixpkgs/nix:nixos-22.05
+    image: nixpkgs/nix:nixos-24.05
    environment:
      DOCKER_PLATFORM: "linux/${ARCH}"
      CONTAINER_NAME: "dxflrs/${ARCH}_garage"
--- a/doc/book/cookbook/binary-packages.md
+++ b/doc/book/cookbook/binary-packages.md
@@ -15,9 +15,10 @@ Alpine Linux repositories (available since v3.17):
 apk add garage
 ```

-The default configuration file is installed to `/etc/garage.toml`. You can run
-Garage using: `rc-service garage start`. If you don't specify `rpc_secret`, it
-will be automatically replaced with a random string on the first start.
+The default configuration file is installed to `/etc/garage/garage.toml`. You can run
+Garage using: `rc-service garage start`.
+
+If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start.

 Please note that this package is built without Consul discovery, Kubernetes
 discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once
--- a/doc/book/reference-manual/configuration.md
+++ b/doc/book/reference-manual/configuration.md
@@ -25,7 +25,7 @@ db_engine = "lmdb"
 block_size = "1M"
 block_ram_buffer_max = "256MiB"
 block_max_concurrent_reads = 16
-
+block_max_concurrent_writes_per_request =10
 lmdb_map_size = "1T"

 compression_level = 1
@@ -103,6 +103,7 @@ Top-level configuration options, in alphabetical order:
 [`allow_world_readable_secrets`](#allow_world_readable_secrets),
 [`block_max_concurrent_reads`](#block_max_concurrent_reads),
 [`block_ram_buffer_max`](#block_ram_buffer_max),
+[`block_max_concurrent_writes_per_request`](#block_max_concurrent_writes_per_request),
 [`block_size`](#block_size),
 [`bootstrap_peers`](#bootstrap_peers),
 [`compression_level`](#compression_level),
@@ -559,6 +560,14 @@ metric in Prometheus: a non-zero number of such events indicates an I/O
 bottleneck on HDD read speed.


+#### `block_max_concurrent_writes_per_request` (since  `v2.1.0`) {#block_max_concurrent_writes_per_request}
+
+This parameter is designed to adapt to the concurrent write performance of 
+different storage media.Maximum number of parallel block writes per put request 
+Higher values improve throughput but increase memory usage.
+
+Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD
+
 #### `lmdb_map_size` {#lmdb_map_size}

 This parameters can be used to set the map size used by LMDB,
--- a/doc/book/reference-manual/s3-compatibility.md
+++ b/doc/book/reference-manual/s3-compatibility.md
@@ -27,7 +27,7 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the

 | Feature                      | Garage                           | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) |
 |------------------------------|----------------------------------|-----------------|---------------|---------|-----|
-| [signature v2](https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) (deprecated) | ❌ Missing | ✅ |  ✅ | ✅ | ✅ |
+| [signature v2](https://docs.aws.amazon.com/AmazonS3/latest/API/Appendix-Sigv2.html) (deprecated) | ❌ Missing | ✅ |  ✅ | ✅ | ✅ |
 | [signature v4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) |  ✅ Implemented |  ✅ |  ✅ | ❌ | ✅ |
 | [URL path-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access) (eg. `host.tld/bucket/key`) |  ✅ Implemented | ✅ |  ✅ | ❓| ✅ |
 | [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) |  ✅ Implemented | ❌| ✅| ✅ | ✅ |
--- a/flake.lock
+++ b/flake.lock
@@ -2,11 +2,11 @@
  "nodes": {
    "crane": {
      "locked": {
-        "lastModified": 1737689766,
-        "narHash": "sha256-ivVXYaYlShxYoKfSo5+y5930qMKKJ8CLcAoIBPQfJ6s=",
+        "lastModified": 1768873933,
+        "narHash": "sha256-CfyzdaeLNGkyAHp3kT5vjvXhA1pVVK7nyDziYxCPsNk=",
        "owner": "ipetkov",
        "repo": "crane",
-        "rev": "6fe74265bbb6d016d663b1091f015e2976c4a527",
+        "rev": "0bda7e7d005ccb5522a76d11ccfbf562b71953ca",
        "type": "github"
      },
      "original": {
@@ -17,11 +17,11 @@
    },
    "flake-compat": {
      "locked": {
-        "lastModified": 1717312683,
-        "narHash": "sha256-FrlieJH50AuvagamEvWMIE6D2OAnERuDboFDYAED/dE=",
+        "lastModified": 1761640442,
+        "narHash": "sha256-AtrEP6Jmdvrqiv4x2xa5mrtaIp3OEe8uBYCDZDS+hu8=",
        "owner": "nix-community",
        "repo": "flake-compat",
-        "rev": "38fd3954cf65ce6faf3d0d45cd26059e059f07ea",
+        "rev": "4a56054d8ffc173222d09dad23adf4ba946c8884",
        "type": "github"
      },
      "original": {
@@ -50,17 +50,17 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1747825515,
-        "narHash": "sha256-BWpMQymVI73QoKZdcVCxUCCK3GNvr/xa2Dc4DM1o2BE=",
+        "lastModified": 1763977559,
+        "narHash": "sha256-g4MKqsIRy5yJwEsI+fYODqLUnAqIY4kZai0nldAP6EM=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6",
+        "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6",
+        "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632",
        "type": "github"
      }
    },
@@ -80,17 +80,17 @@
        ]
      },
      "locked": {
-        "lastModified": 1738549608,
-        "narHash": "sha256-GdyT9QEUSx5k/n8kILuNy83vxxdyUfJ8jL5mMpQZWfw=",
+        "lastModified": 1763952169,
+        "narHash": "sha256-+PeDBD8P+NKauH+w7eO/QWCIp8Cx4mCfWnh9sJmy9CM=",
        "owner": "oxalica",
        "repo": "rust-overlay",
-        "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
+        "rev": "ab726555a9a72e6dc80649809147823a813fa95b",
        "type": "github"
      },
      "original": {
        "owner": "oxalica",
        "repo": "rust-overlay",
-        "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d",
+        "rev": "ab726555a9a72e6dc80649809147823a813fa95b",
        "type": "github"
      }
    },
--- a/flake.nix
+++ b/flake.nix
@@ -2,13 +2,13 @@
  description =
    "Garage, an S3-compatible distributed object store for self-hosted deployments";

-  # Nixpkgs 25.05 as of 2025-05-22
+  # Nixpkgs 25.05 as of 2025-11-24
  inputs.nixpkgs.url =
-    "github:NixOS/nixpkgs/cd2812de55cf87df88a9e09bf3be1ce63d50c1a6";
+    "github:NixOS/nixpkgs/cfe2c7d5b5d3032862254e68c37a6576b633d632";

-  # Rust overlay as of 2025-02-03
+  # Rust overlay as of 2025-11-24
  inputs.rust-overlay.url =
-  "github:oxalica/rust-overlay/35c6f8c4352f995ecd53896200769f80a3e8f22d";
+  "github:oxalica/rust-overlay/ab726555a9a72e6dc80649809147823a813fa95b";
  inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs";

  inputs.crane.url = "github:ipetkov/crane";
@@ -30,6 +30,10 @@
          inherit system nixpkgs crane rust-overlay extraTestEnv;
          release = false;
        }).garage-test;
+        lints = (compile {
+          inherit system nixpkgs crane rust-overlay;
+          release = false;
+        });
      in
      {
        packages = {
@@ -56,6 +60,10 @@
          tests-fjall = testWith {
            GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall";
          };
+
+          # lints (fmt, clippy)
+          fmt = lints.garage-cargo-fmt;
+          clippy = lints.garage-cargo-clippy;
        };

        # ---- developpment shell, for making native builds only ----
--- a/nix/compile.nix
+++ b/nix/compile.nix
@@ -48,7 +48,7 @@ let

  inherit (pkgs) lib stdenv;

-  toolchainFn = (p: p.rust-bin.stable."1.82.0".default.override {
+  toolchainFn = (p: p.rust-bin.stable."1.91.0".default.override {
    targets = lib.optionals (target != null) [ rustTarget ];
    extensions = [
      "rust-src"
@@ -190,4 +190,15 @@ in rec {
      pkgs.cacert
    ];
  } // extraTestEnv);
+
+  # ---- source code linting ----
+
+  garage-cargo-fmt = craneLib.cargoFmt (commonArgs // {
+    cargoExtraArgs = "";
+  });
+
+  garage-cargo-clippy = craneLib.cargoClippy (commonArgs // {
+    cargoArtifacts = garage-deps;
+    cargoClippyExtraArgs = "--all-targets -- -D warnings";
+  });
 }
--- a/script/dev-env-aws.sh
+++ b/script/dev-env-aws.sh
@@ -1,6 +1,7 @@
 export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1`
 export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2`
 export AWS_DEFAULT_REGION='garage'
+export AWS_REQUEST_CHECKSUM_CALCULATION='when_required'
 # FUTUREWORK: set AWS_ENDPOINT_URL instead, once nixpkgs bumps awscli to >=2.13.0.
 function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; }

--- a/shell.nix
+++ b/shell.nix
@@ -36,6 +36,8 @@ in
      jq
    ];
    shellHook = ''
+      export AWS_REQUEST_CHECKSUM_CALCULATION='when_required'
+
      function to_s3 {
        AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED AWS_RESPONSE_CHECKSUM_VALIDATION=WHEN_REQUIRED \
        aws \
--- a/src/api/admin/error.rs
+++ b/src/api/admin/error.rs
@@ -26,7 +26,7 @@ pub enum Error {
 	NoSuchAdminToken(String),

 	/// The API access key does not exist
-	#[error("Access key not found: {00}")]
+	#[error("Access key not found: {0}")]
 	NoSuchAccessKey(String),

 	/// The requested block does not exist
--- a/src/api/common/signature/payload.rs
+++ b/src/api/common/signature/payload.rs
@@ -105,7 +105,7 @@ fn check_standard_signature(
 	// Verify that all necessary request headers are included in signed_headers
 	// The following must be included for all signatures:
 	// - the Host header (mandatory)
-	// - all x-amz-* headers used in the request
+	// - all x-amz-* headers used in the request (except x-amz-content-sha256)
 	// AWS also indicates that the Content-Type header should be signed if
 	// it is used, but Minio client doesn't sign it so we don't check it for compatibility.
 	let signed_headers = split_signed_headers(&authorization)?;
@@ -152,7 +152,7 @@ fn check_presigned_signature(
 	// Verify that all necessary request headers are included in signed_headers
 	// For AWSv4 pre-signed URLs, the following must be included:
 	// - the Host header (mandatory)
-	// - all x-amz-* headers used in the request
+	// - all x-amz-* headers used in the request (except x-amz-content-sha256)
 	let signed_headers = split_signed_headers(&authorization)?;
 	verify_signed_headers(request.headers(), &signed_headers)?;

@@ -269,7 +269,9 @@ fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) ->
 		return Err(Error::bad_request("Header `Host` should be signed"));
 	}
 	for (name, _) in headers.iter() {
-		if name.as_str().starts_with("x-amz-") {
+		// Enforce signature of all x-amz-* headers, except x-amz-content-sh256
+		// because it is included in the canonical request in all cases
+		if name.as_str().starts_with("x-amz-") && name != X_AMZ_CONTENT_SHA256 {
 			if !signed_headers.contains(name) {
 				return Err(Error::bad_request(format!(
 					"Header `{}` should be signed",
@@ -475,8 +477,7 @@ impl Authorization {

 		let date = headers
 			.get(X_AMZ_DATE)
-			.ok_or_bad_request("Missing X-Amz-Date field")
-			.map_err(Error::from)?
+			.ok_or_bad_request("Missing X-Amz-Date field")?
 			.to_str()?;
 		let date = parse_date(date)?;

--- a/src/api/s3/get.rs
+++ b/src/api/s3/get.rs
@@ -853,7 +853,9 @@ impl PreconditionHeaders {
 	}

 	fn check(&self, v: &ObjectVersion, etag: &str) -> Result<Option<StatusCode>, Error> {
-		let v_date = UNIX_EPOCH + Duration::from_millis(v.timestamp);
+		// we store date with ms precision, but headers are precise to the second: truncate
+		// the timestamp to handle the same-second edge case
+		let v_date = UNIX_EPOCH + Duration::from_secs(v.timestamp / 1000);

 		// Implemented from https://datatracker.ietf.org/doc/html/rfc7232#section-6

--- a/src/api/s3/put.rs
+++ b/src/api/s3/put.rs
@@ -39,8 +39,6 @@ use crate::encryption::{EncryptionParams, OekDerivationInfo};
 use crate::error::*;
 use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;

-const PUT_BLOCKS_MAX_PARALLEL: usize = 3;
-
 pub(crate) struct SaveStreamResult {
 	pub(crate) version_uuid: Uuid,
 	pub(crate) version_timestamp: u64,
@@ -507,7 +505,7 @@ pub(crate) async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> +
 			};
 			let recv_next = async {
 				// If more than a maximum number of writes are in progress, don't add more for now
-				if currently_running >= PUT_BLOCKS_MAX_PARALLEL {
+				if currently_running >= ctx.garage.config.block_max_concurrent_writes_per_request {
 					futures::future::pending().await
 				} else {
 					block_rx3.recv().await
--- a/src/garage/tests/s3/objects.rs
+++ b/src/garage/tests/s3/objects.rs
@@ -198,6 +198,7 @@ async fn test_precondition() {
 		);
 	}
 	let older_date = DateTime::from_secs_f64(last_modified.as_secs_f64() - 10.0);
+	let same_date = DateTime::from_secs_f64(last_modified.as_secs_f64());
 	let newer_date = DateTime::from_secs_f64(last_modified.as_secs_f64() + 10.0);
 	{
 		let err = ctx
@@ -212,6 +213,18 @@ async fn test_precondition() {
 			matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304)
 		);

+		let err = ctx
+			.client
+			.get_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.if_modified_since(same_date)
+			.send()
+			.await;
+		assert!(
+			matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304)
+		);
+
 		let o = ctx
 			.client
 			.get_object()
@@ -236,6 +249,17 @@ async fn test_precondition() {
 			matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 412)
 		);

+		let o = ctx
+			.client
+			.get_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.if_unmodified_since(same_date)
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag);
+
 		let o = ctx
 			.client
 			.get_object()
--- a/src/util/config.rs
+++ b/src/util/config.rs
@@ -45,6 +45,11 @@ pub struct Config {
 	)]
 	pub block_size: usize,

+	/// Maximum number of parallel block writes per PUT request
+	/// Higher values improve throughput but increase memory usage
+	/// Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD
+	#[serde(default = "default_block_max_concurrent_writes_per_request")]
+	pub block_max_concurrent_writes_per_request: usize,
 	/// Number of replicas. Can be any positive integer, but uneven numbers are more favorable.
 	/// - 1 for single-node clusters, or to disable replication
 	/// - 3 is the recommended and supported setting.
@@ -272,6 +277,9 @@ pub struct KubernetesDiscoveryConfig {
 	pub skip_crd: bool,
 }

+pub fn default_block_max_concurrent_writes_per_request() -> usize {
+	3
+}
 /// Read and parse configuration
 pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
 	let config = std::fs::read_to_string(config_file)?;