diff --git a/.woodpecker/debug.yaml b/.woodpecker/debug.yaml index 4c729672..4dc7d3c9 100644 --- a/.woodpecker/debug.yaml +++ b/.woodpecker/debug.yaml @@ -12,32 +12,32 @@ when: steps: - name: check formatting - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - - nix-shell --attr devShell --run "cargo fmt -- --check" + - nix-build -j4 --attr flakePackages.fmt - name: build - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build -j4 --attr flakePackages.dev - name: unit + func tests (lmdb) - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build -j4 --attr flakePackages.tests-lmdb - name: unit + func tests (sqlite) - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build -j4 --attr flakePackages.tests-sqlite - name: unit + func tests (fjall) - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build -j4 --attr flakePackages.tests-fjall - name: integration tests - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build -j4 --attr flakePackages.dev - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) diff --git a/.woodpecker/publish.yaml b/.woodpecker/publish.yaml index 24a84463..8f3b482f 100644 --- a/.woodpecker/publish.yaml +++ b/.woodpecker/publish.yaml @@ -11,7 +11,7 @@ depends_on: steps: - name: refresh-index - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 environment: AWS_ACCESS_KEY_ID: from_secret: garagehq_aws_access_key_id @@ -22,7 +22,7 @@ steps: - nix-shell --attr ci --run "refresh_index" - name: multiarch-docker - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 environment: DOCKER_AUTH: from_secret: docker_auth diff --git a/.woodpecker/release.yaml b/.woodpecker/release.yaml index c1514cd1..4133b92d 100644 --- a/.woodpecker/release.yaml +++ b/.woodpecker/release.yaml @@ -19,17 +19,17 @@ matrix: steps: - name: build - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} - name: check is static binary - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage" - name: integration tests - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) when: @@ -39,7 +39,7 @@ steps: ARCH: i386 - name: upgrade tests from v1.0.0 - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-shell --attr ci --run "./script/test-upgrade.sh v1.0.0 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false) when: @@ -47,7 +47,7 @@ steps: ARCH: amd64 - name: upgrade tests from v0.8.4 - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false) when: @@ -55,7 +55,7 @@ steps: ARCH: amd64 - name: push static binary - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 environment: TARGET: "${TARGET}" AWS_ACCESS_KEY_ID: @@ -66,7 +66,7 @@ steps: - nix-shell --attr ci --run "to_s3" - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 environment: DOCKER_PLATFORM: "linux/${ARCH}" CONTAINER_NAME: "dxflrs/${ARCH}_garage" diff --git a/doc/book/cookbook/binary-packages.md b/doc/book/cookbook/binary-packages.md index b7e78267..1e399764 100644 --- a/doc/book/cookbook/binary-packages.md +++ b/doc/book/cookbook/binary-packages.md @@ -15,9 +15,10 @@ Alpine Linux repositories (available since v3.17): apk add garage ``` -The default configuration file is installed to `/etc/garage.toml`. You can run -Garage using: `rc-service garage start`. If you don't specify `rpc_secret`, it -will be automatically replaced with a random string on the first start. +The default configuration file is installed to `/etc/garage/garage.toml`. You can run +Garage using: `rc-service garage start`. + +If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start. Please note that this package is built without Consul discovery, Kubernetes discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index 9b2d57b6..909c5715 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -25,7 +25,7 @@ db_engine = "lmdb" block_size = "1M" block_ram_buffer_max = "256MiB" block_max_concurrent_reads = 16 - +block_max_concurrent_writes_per_request =10 lmdb_map_size = "1T" compression_level = 1 @@ -103,6 +103,7 @@ Top-level configuration options, in alphabetical order: [`allow_world_readable_secrets`](#allow_world_readable_secrets), [`block_max_concurrent_reads`](#block_max_concurrent_reads), [`block_ram_buffer_max`](#block_ram_buffer_max), +[`block_max_concurrent_writes_per_request`](#block_max_concurrent_writes_per_request), [`block_size`](#block_size), [`bootstrap_peers`](#bootstrap_peers), [`compression_level`](#compression_level), @@ -559,6 +560,14 @@ metric in Prometheus: a non-zero number of such events indicates an I/O bottleneck on HDD read speed. +#### `block_max_concurrent_writes_per_request` (since `v2.1.0`) {#block_max_concurrent_writes_per_request} + +This parameter is designed to adapt to the concurrent write performance of +different storage media.Maximum number of parallel block writes per put request +Higher values improve throughput but increase memory usage. + +Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD + #### `lmdb_map_size` {#lmdb_map_size} This parameters can be used to set the map size used by LMDB, diff --git a/doc/book/reference-manual/s3-compatibility.md b/doc/book/reference-manual/s3-compatibility.md index edf8de0d..b869b6f4 100644 --- a/doc/book/reference-manual/s3-compatibility.md +++ b/doc/book/reference-manual/s3-compatibility.md @@ -27,7 +27,7 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the | Feature | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) | |------------------------------|----------------------------------|-----------------|---------------|---------|-----| -| [signature v2](https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) (deprecated) | ❌ Missing | ✅ | ✅ | ✅ | ✅ | +| [signature v2](https://docs.aws.amazon.com/AmazonS3/latest/API/Appendix-Sigv2.html) (deprecated) | ❌ Missing | ✅ | ✅ | ✅ | ✅ | | [signature v4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) | ✅ Implemented | ✅ | ✅ | ❌ | ✅ | | [URL path-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access) (eg. `host.tld/bucket/key`) | ✅ Implemented | ✅ | ✅ | ❓| ✅ | | [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) | ✅ Implemented | ❌| ✅| ✅ | ✅ | diff --git a/flake.lock b/flake.lock index f1b77c6c..2a3bebf1 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "crane": { "locked": { - "lastModified": 1737689766, - "narHash": "sha256-ivVXYaYlShxYoKfSo5+y5930qMKKJ8CLcAoIBPQfJ6s=", + "lastModified": 1768873933, + "narHash": "sha256-CfyzdaeLNGkyAHp3kT5vjvXhA1pVVK7nyDziYxCPsNk=", "owner": "ipetkov", "repo": "crane", - "rev": "6fe74265bbb6d016d663b1091f015e2976c4a527", + "rev": "0bda7e7d005ccb5522a76d11ccfbf562b71953ca", "type": "github" }, "original": { @@ -17,11 +17,11 @@ }, "flake-compat": { "locked": { - "lastModified": 1717312683, - "narHash": "sha256-FrlieJH50AuvagamEvWMIE6D2OAnERuDboFDYAED/dE=", + "lastModified": 1761640442, + "narHash": "sha256-AtrEP6Jmdvrqiv4x2xa5mrtaIp3OEe8uBYCDZDS+hu8=", "owner": "nix-community", "repo": "flake-compat", - "rev": "38fd3954cf65ce6faf3d0d45cd26059e059f07ea", + "rev": "4a56054d8ffc173222d09dad23adf4ba946c8884", "type": "github" }, "original": { @@ -50,17 +50,17 @@ }, "nixpkgs": { "locked": { - "lastModified": 1747825515, - "narHash": "sha256-BWpMQymVI73QoKZdcVCxUCCK3GNvr/xa2Dc4DM1o2BE=", + "lastModified": 1763977559, + "narHash": "sha256-g4MKqsIRy5yJwEsI+fYODqLUnAqIY4kZai0nldAP6EM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6", + "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632", "type": "github" }, "original": { "owner": "NixOS", "repo": "nixpkgs", - "rev": "cd2812de55cf87df88a9e09bf3be1ce63d50c1a6", + "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632", "type": "github" } }, @@ -80,17 +80,17 @@ ] }, "locked": { - "lastModified": 1738549608, - "narHash": "sha256-GdyT9QEUSx5k/n8kILuNy83vxxdyUfJ8jL5mMpQZWfw=", + "lastModified": 1763952169, + "narHash": "sha256-+PeDBD8P+NKauH+w7eO/QWCIp8Cx4mCfWnh9sJmy9CM=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d", + "rev": "ab726555a9a72e6dc80649809147823a813fa95b", "type": "github" }, "original": { "owner": "oxalica", "repo": "rust-overlay", - "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d", + "rev": "ab726555a9a72e6dc80649809147823a813fa95b", "type": "github" } }, diff --git a/flake.nix b/flake.nix index e3e5e69d..cb51ac2d 100644 --- a/flake.nix +++ b/flake.nix @@ -2,13 +2,13 @@ description = "Garage, an S3-compatible distributed object store for self-hosted deployments"; - # Nixpkgs 25.05 as of 2025-05-22 + # Nixpkgs 25.05 as of 2025-11-24 inputs.nixpkgs.url = - "github:NixOS/nixpkgs/cd2812de55cf87df88a9e09bf3be1ce63d50c1a6"; + "github:NixOS/nixpkgs/cfe2c7d5b5d3032862254e68c37a6576b633d632"; - # Rust overlay as of 2025-02-03 + # Rust overlay as of 2025-11-24 inputs.rust-overlay.url = - "github:oxalica/rust-overlay/35c6f8c4352f995ecd53896200769f80a3e8f22d"; + "github:oxalica/rust-overlay/ab726555a9a72e6dc80649809147823a813fa95b"; inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs"; inputs.crane.url = "github:ipetkov/crane"; @@ -30,6 +30,10 @@ inherit system nixpkgs crane rust-overlay extraTestEnv; release = false; }).garage-test; + lints = (compile { + inherit system nixpkgs crane rust-overlay; + release = false; + }); in { packages = { @@ -56,6 +60,10 @@ tests-fjall = testWith { GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall"; }; + + # lints (fmt, clippy) + fmt = lints.garage-cargo-fmt; + clippy = lints.garage-cargo-clippy; }; # ---- developpment shell, for making native builds only ---- diff --git a/nix/compile.nix b/nix/compile.nix index 7e9f79ab..c6df9dbd 100644 --- a/nix/compile.nix +++ b/nix/compile.nix @@ -48,7 +48,7 @@ let inherit (pkgs) lib stdenv; - toolchainFn = (p: p.rust-bin.stable."1.82.0".default.override { + toolchainFn = (p: p.rust-bin.stable."1.91.0".default.override { targets = lib.optionals (target != null) [ rustTarget ]; extensions = [ "rust-src" @@ -190,4 +190,15 @@ in rec { pkgs.cacert ]; } // extraTestEnv); + + # ---- source code linting ---- + + garage-cargo-fmt = craneLib.cargoFmt (commonArgs // { + cargoExtraArgs = ""; + }); + + garage-cargo-clippy = craneLib.cargoClippy (commonArgs // { + cargoArtifacts = garage-deps; + cargoClippyExtraArgs = "--all-targets -- -D warnings"; + }); } diff --git a/script/dev-env-aws.sh b/script/dev-env-aws.sh index 808f9cf1..41f1fdde 100644 --- a/script/dev-env-aws.sh +++ b/script/dev-env-aws.sh @@ -1,6 +1,7 @@ export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1` export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2` export AWS_DEFAULT_REGION='garage' +export AWS_REQUEST_CHECKSUM_CALCULATION='when_required' # FUTUREWORK: set AWS_ENDPOINT_URL instead, once nixpkgs bumps awscli to >=2.13.0. function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; } diff --git a/shell.nix b/shell.nix index 373507b3..42796408 100644 --- a/shell.nix +++ b/shell.nix @@ -36,6 +36,8 @@ in jq ]; shellHook = '' + export AWS_REQUEST_CHECKSUM_CALCULATION='when_required' + function to_s3 { AWS_REQUEST_CHECKSUM_CALCULATION=WHEN_REQUIRED AWS_RESPONSE_CHECKSUM_VALIDATION=WHEN_REQUIRED \ aws \ diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index 2cdf7582..b8be278e 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -26,7 +26,7 @@ pub enum Error { NoSuchAdminToken(String), /// The API access key does not exist - #[error("Access key not found: {00}")] + #[error("Access key not found: {0}")] NoSuchAccessKey(String), /// The requested block does not exist diff --git a/src/api/common/signature/payload.rs b/src/api/common/signature/payload.rs index b7ffc599..0c657601 100644 --- a/src/api/common/signature/payload.rs +++ b/src/api/common/signature/payload.rs @@ -105,7 +105,7 @@ fn check_standard_signature( // Verify that all necessary request headers are included in signed_headers // The following must be included for all signatures: // - the Host header (mandatory) - // - all x-amz-* headers used in the request + // - all x-amz-* headers used in the request (except x-amz-content-sha256) // AWS also indicates that the Content-Type header should be signed if // it is used, but Minio client doesn't sign it so we don't check it for compatibility. let signed_headers = split_signed_headers(&authorization)?; @@ -152,7 +152,7 @@ fn check_presigned_signature( // Verify that all necessary request headers are included in signed_headers // For AWSv4 pre-signed URLs, the following must be included: // - the Host header (mandatory) - // - all x-amz-* headers used in the request + // - all x-amz-* headers used in the request (except x-amz-content-sha256) let signed_headers = split_signed_headers(&authorization)?; verify_signed_headers(request.headers(), &signed_headers)?; @@ -269,7 +269,9 @@ fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) -> return Err(Error::bad_request("Header `Host` should be signed")); } for (name, _) in headers.iter() { - if name.as_str().starts_with("x-amz-") { + // Enforce signature of all x-amz-* headers, except x-amz-content-sh256 + // because it is included in the canonical request in all cases + if name.as_str().starts_with("x-amz-") && name != X_AMZ_CONTENT_SHA256 { if !signed_headers.contains(name) { return Err(Error::bad_request(format!( "Header `{}` should be signed", @@ -475,8 +477,7 @@ impl Authorization { let date = headers .get(X_AMZ_DATE) - .ok_or_bad_request("Missing X-Amz-Date field") - .map_err(Error::from)? + .ok_or_bad_request("Missing X-Amz-Date field")? .to_str()?; let date = parse_date(date)?; diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index 77d8a41a..16e7a3da 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -853,7 +853,9 @@ impl PreconditionHeaders { } fn check(&self, v: &ObjectVersion, etag: &str) -> Result, Error> { - let v_date = UNIX_EPOCH + Duration::from_millis(v.timestamp); + // we store date with ms precision, but headers are precise to the second: truncate + // the timestamp to handle the same-second edge case + let v_date = UNIX_EPOCH + Duration::from_secs(v.timestamp / 1000); // Implemented from https://datatracker.ietf.org/doc/html/rfc7232#section-6 diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index a2ea1039..7a9c4a62 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -39,8 +39,6 @@ use crate::encryption::{EncryptionParams, OekDerivationInfo}; use crate::error::*; use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION; -const PUT_BLOCKS_MAX_PARALLEL: usize = 3; - pub(crate) struct SaveStreamResult { pub(crate) version_uuid: Uuid, pub(crate) version_timestamp: u64, @@ -507,7 +505,7 @@ pub(crate) async fn read_and_put_blocks> + }; let recv_next = async { // If more than a maximum number of writes are in progress, don't add more for now - if currently_running >= PUT_BLOCKS_MAX_PARALLEL { + if currently_running >= ctx.garage.config.block_max_concurrent_writes_per_request { futures::future::pending().await } else { block_rx3.recv().await diff --git a/src/garage/tests/s3/objects.rs b/src/garage/tests/s3/objects.rs index d63ac000..53e8231d 100644 --- a/src/garage/tests/s3/objects.rs +++ b/src/garage/tests/s3/objects.rs @@ -198,6 +198,7 @@ async fn test_precondition() { ); } let older_date = DateTime::from_secs_f64(last_modified.as_secs_f64() - 10.0); + let same_date = DateTime::from_secs_f64(last_modified.as_secs_f64()); let newer_date = DateTime::from_secs_f64(last_modified.as_secs_f64() + 10.0); { let err = ctx @@ -212,6 +213,18 @@ async fn test_precondition() { matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304) ); + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_modified_since(same_date) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304) + ); + let o = ctx .client .get_object() @@ -236,6 +249,17 @@ async fn test_precondition() { matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 412) ); + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_unmodified_since(same_date) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); + let o = ctx .client .get_object() diff --git a/src/util/config.rs b/src/util/config.rs index 83f07d03..bc476a35 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -45,6 +45,11 @@ pub struct Config { )] pub block_size: usize, + /// Maximum number of parallel block writes per PUT request + /// Higher values improve throughput but increase memory usage + /// Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD + #[serde(default = "default_block_max_concurrent_writes_per_request")] + pub block_max_concurrent_writes_per_request: usize, /// Number of replicas. Can be any positive integer, but uneven numbers are more favorable. /// - 1 for single-node clusters, or to disable replication /// - 3 is the recommended and supported setting. @@ -272,6 +277,9 @@ pub struct KubernetesDiscoveryConfig { pub skip_crd: bool, } +pub fn default_block_max_concurrent_writes_per_request() -> usize { + 3 +} /// Read and parse configuration pub fn read_config(config_file: PathBuf) -> Result { let config = std::fs::read_to_string(config_file)?;