From 0f1b488be051f08e49ae0fcc03cc34cb608fc2bf Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 14 Sep 2025 21:25:37 +0200 Subject: [PATCH 01/19] fix rust warnings --- src/api/common/generic_server.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/api/common/generic_server.rs b/src/api/common/generic_server.rs index 5783c276..3f14c07d 100644 --- a/src/api/common/generic_server.rs +++ b/src/api/common/generic_server.rs @@ -33,7 +33,6 @@ use garage_util::metrics::{gen_trace_id, RecordDuration}; use garage_util::socket_address::UnixOrTCPSocketAddress; use crate::helpers::{BoxBody, ErrorBody}; -use crate::signature::payload::Authorization; pub trait ApiEndpoint: Send + Sync + 'static { fn name(&self) -> &'static str; @@ -62,7 +61,7 @@ pub trait ApiHandler: Send + Sync + 'static { /// Returns the key id used to authenticate this request. The ID returned must be safe to /// log. - fn key_id_from_request(&self, req: &Request) -> Option { + fn key_id_from_request(&self, _req: &Request) -> Option { None } } From 4b1fdbef55ee6a6bd68e904aa91863e7c3289555 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 14 Sep 2025 21:36:33 +0200 Subject: [PATCH 02/19] bump version to v1.3.0 --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 24 ++++++++++++------------ doc/book/cookbook/real-world.md | 10 +++++----- doc/book/quick-start/_index.md | 2 +- doc/drafts/admin-api.md | 2 +- script/helm/garage/Chart.yaml | 4 ++-- script/helm/garage/README.md | 2 +- src/api/admin/Cargo.toml | 2 +- src/api/common/Cargo.toml | 2 +- src/api/k2v/Cargo.toml | 2 +- src/api/s3/Cargo.toml | 2 +- src/block/Cargo.toml | 2 +- src/db/Cargo.toml | 2 +- src/garage/Cargo.toml | 2 +- src/model/Cargo.toml | 4 ++-- src/net/Cargo.toml | 2 +- src/rpc/Cargo.toml | 2 +- src/table/Cargo.toml | 2 +- src/util/Cargo.toml | 2 +- src/web/Cargo.toml | 2 +- 20 files changed, 49 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5824294e..c516bfbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1280,7 +1280,7 @@ dependencies = [ [[package]] name = "garage" -version = "1.2.0" +version = "1.3.0" dependencies = [ "assert-json-diff", "async-trait", @@ -1336,7 +1336,7 @@ dependencies = [ [[package]] name = "garage_api_admin" -version = "1.2.0" +version = "1.3.0" dependencies = [ "argon2", "async-trait", @@ -1362,7 +1362,7 @@ dependencies = [ [[package]] name = "garage_api_common" -version = "1.2.0" +version = "1.3.0" dependencies = [ "base64 0.21.7", "bytes", @@ -1396,7 +1396,7 @@ dependencies = [ [[package]] name = "garage_api_k2v" -version = "1.2.0" +version = "1.3.0" dependencies = [ "base64 0.21.7", "err-derive", @@ -1419,7 +1419,7 @@ dependencies = [ [[package]] name = "garage_api_s3" -version = "1.2.0" +version = "1.3.0" dependencies = [ "aes-gcm", "async-compression", @@ -1464,7 +1464,7 @@ dependencies = [ [[package]] name = "garage_block" -version = "1.2.0" +version = "1.3.0" dependencies = [ "arc-swap", "async-compression", @@ -1489,7 +1489,7 @@ dependencies = [ [[package]] name = "garage_db" -version = "1.2.0" +version = "1.3.0" dependencies = [ "err-derive", "fjall", @@ -1504,7 +1504,7 @@ dependencies = [ [[package]] name = "garage_model" -version = "1.2.0" +version = "1.3.0" dependencies = [ "async-trait", "base64 0.21.7", @@ -1531,7 +1531,7 @@ dependencies = [ [[package]] name = "garage_net" -version = "1.2.0" +version = "1.3.0" dependencies = [ "arc-swap", "bytes", @@ -1556,7 +1556,7 @@ dependencies = [ [[package]] name = "garage_rpc" -version = "1.2.0" +version = "1.3.0" dependencies = [ "arc-swap", "async-trait", @@ -1588,7 +1588,7 @@ dependencies = [ [[package]] name = "garage_table" -version = "1.2.0" +version = "1.3.0" dependencies = [ "arc-swap", "async-trait", @@ -1609,7 +1609,7 @@ dependencies = [ [[package]] name = "garage_util" -version = "1.2.0" +version = "1.3.0" dependencies = [ "arc-swap", "async-trait", @@ -1641,7 +1641,7 @@ dependencies = [ [[package]] name = "garage_web" -version = "1.2.0" +version = "1.3.0" dependencies = [ "err-derive", "garage_api_common", diff --git a/Cargo.toml b/Cargo.toml index 75d5bcfb..5bc76e3c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,18 +24,18 @@ default-members = ["src/garage"] # Internal Garage crates format_table = { version = "0.1.1", path = "src/format-table" } -garage_api_common = { version = "1.2.0", path = "src/api/common" } -garage_api_admin = { version = "1.2.0", path = "src/api/admin" } -garage_api_s3 = { version = "1.2.0", path = "src/api/s3" } -garage_api_k2v = { version = "1.2.0", path = "src/api/k2v" } -garage_block = { version = "1.2.0", path = "src/block" } -garage_db = { version = "1.2.0", path = "src/db", default-features = false } -garage_model = { version = "1.2.0", path = "src/model", default-features = false } -garage_net = { version = "1.2.0", path = "src/net" } -garage_rpc = { version = "1.2.0", path = "src/rpc" } -garage_table = { version = "1.2.0", path = "src/table" } -garage_util = { version = "1.2.0", path = "src/util" } -garage_web = { version = "1.2.0", path = "src/web" } +garage_api_common = { version = "1.3.0", path = "src/api/common" } +garage_api_admin = { version = "1.3.0", path = "src/api/admin" } +garage_api_s3 = { version = "1.3.0", path = "src/api/s3" } +garage_api_k2v = { version = "1.3.0", path = "src/api/k2v" } +garage_block = { version = "1.3.0", path = "src/block" } +garage_db = { version = "1.3.0", path = "src/db", default-features = false } +garage_model = { version = "1.3.0", path = "src/model", default-features = false } +garage_net = { version = "1.3.0", path = "src/net" } +garage_rpc = { version = "1.3.0", path = "src/rpc" } +garage_table = { version = "1.3.0", path = "src/table" } +garage_util = { version = "1.3.0", path = "src/util" } +garage_web = { version = "1.3.0", path = "src/web" } k2v-client = { version = "0.0.4", path = "src/k2v-client" } # External crates from crates.io diff --git a/doc/book/cookbook/real-world.md b/doc/book/cookbook/real-world.md index 998c02a5..b9927c06 100644 --- a/doc/book/cookbook/real-world.md +++ b/doc/book/cookbook/real-world.md @@ -96,14 +96,14 @@ to store 2 TB of data in total. ## Get a Docker image Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). -We encourage you to use a fixed tag (eg. `v1.2.0`) and not the `latest` tag. -For this example, we will use the latest published version at the time of the writing which is `v1.2.0` but it's up to you +We encourage you to use a fixed tag (eg. `v1.3.0`) and not the `latest` tag. +For this example, we will use the latest published version at the time of the writing which is `v1.3.0` but it's up to you to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated). For example: ``` -sudo docker pull dxflrs/garage:v1.2.0 +sudo docker pull dxflrs/garage:v1.3.0 ``` ## Deploying and configuring Garage @@ -171,7 +171,7 @@ docker run \ -v /etc/garage.toml:/etc/garage.toml \ -v /var/lib/garage/meta:/var/lib/garage/meta \ -v /var/lib/garage/data:/var/lib/garage/data \ - dxflrs/garage:v1.2.0 + dxflrs/garage:v1.3.0 ``` With this command line, Garage should be started automatically at each boot. @@ -185,7 +185,7 @@ If you want to use `docker-compose`, you may use the following `docker-compose.y version: "3" services: garage: - image: dxflrs/garage:v1.2.0 + image: dxflrs/garage:v1.3.0 network_mode: "host" restart: unless-stopped volumes: diff --git a/doc/book/quick-start/_index.md b/doc/book/quick-start/_index.md index 45a4a43b..633b785a 100644 --- a/doc/book/quick-start/_index.md +++ b/doc/book/quick-start/_index.md @@ -132,7 +132,7 @@ docker run \ -v /path/to/garage.toml:/etc/garage.toml \ -v /path/to/garage/meta:/var/lib/garage/meta \ -v /path/to/garage/data:/var/lib/garage/data \ - dxflrs/garage:v1.2.0 + dxflrs/garage:v1.3.0 ``` Under Linux, you can substitute `--network host` for `-p 3900:3900 -p 3901:3901 -p 3902:3902 -p 3903:3903` diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index a3d03c41..3ee948cb 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -70,7 +70,7 @@ Example response body: ```json { "node": "b10c110e4e854e5aa3f4637681befac755154b20059ec163254ddbfae86b09df", - "garageVersion": "v1.2.0", + "garageVersion": "v1.3.0", "garageFeatures": [ "k2v", "lmdb", diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml index 6806e593..51f98bbb 100644 --- a/script/helm/garage/Chart.yaml +++ b/script/helm/garage/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: garage description: S3-compatible object store for small self-hosted geo-distributed deployments type: application -version: 0.7.1 -appVersion: "v1.2.0" +version: 0.7.2 +appVersion: "v1.3.0" home: https://garagehq.deuxfleurs.fr/ icon: https://garagehq.deuxfleurs.fr/images/garage-logo.svg diff --git a/script/helm/garage/README.md b/script/helm/garage/README.md index 05d444a3..25e548ec 100644 --- a/script/helm/garage/README.md +++ b/script/helm/garage/README.md @@ -1,6 +1,6 @@ # garage -![Version: 0.7.1](https://img.shields.io/badge/Version-0.7.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.2.0](https://img.shields.io/badge/AppVersion-v1.2.0-informational?style=flat-square) +![Version: 0.7.2](https://img.shields.io/badge/Version-0.7.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.3.0](https://img.shields.io/badge/AppVersion-v1.3.0-informational?style=flat-square) S3-compatible object store for small self-hosted geo-distributed deployments diff --git a/src/api/admin/Cargo.toml b/src/api/admin/Cargo.toml index 6b039eeb..d7184068 100644 --- a/src/api/admin/Cargo.toml +++ b/src/api/admin/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_api_admin" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml index a67e9d9c..fd159c96 100644 --- a/src/api/common/Cargo.toml +++ b/src/api/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_api_common" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/api/k2v/Cargo.toml b/src/api/k2v/Cargo.toml index 845d23f6..628d2db1 100644 --- a/src/api/k2v/Cargo.toml +++ b/src/api/k2v/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_api_k2v" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/api/s3/Cargo.toml b/src/api/s3/Cargo.toml index 1ba7565d..15f6858c 100644 --- a/src/api/s3/Cargo.toml +++ b/src/api/s3/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_api_s3" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/block/Cargo.toml b/src/block/Cargo.toml index d5f8e58e..effa8dba 100644 --- a/src/block/Cargo.toml +++ b/src/block/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_block" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index e9ed15c9..6dee2fa6 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_db" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 7d60313e..ad2b917b 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 14f92253..e59765d7 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_model" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -44,4 +44,4 @@ default = [ "lmdb", "sqlite" ] k2v = [ "garage_util/k2v" ] lmdb = [ "garage_db/lmdb" ] sqlite = [ "garage_db/sqlite" ] -fjall = [ "garage_db/fjall" ] \ No newline at end of file +fjall = [ "garage_db/fjall" ] diff --git a/src/net/Cargo.toml b/src/net/Cargo.toml index 17a0eb24..83b3b15b 100644 --- a/src/net/Cargo.toml +++ b/src/net/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_net" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index a314271f..1e764c77 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_rpc" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index c76c5b78..91ab110c 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_table" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index f59e44e2..0d693a97 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_util" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 5d208e6e..c1056509 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_web" -version = "1.2.0" +version = "1.3.0" authors = ["Alex Auvolat ", "Quentin Dufour "] edition = "2018" license = "AGPL-3.0" From 42fd8583bd81c9855d25c5ac0171e9eb9d265ab7 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Wed, 8 Oct 2025 17:54:15 +0200 Subject: [PATCH 03/19] properly handle precondition time equal to object time --- src/api/s3/get.rs | 4 +++- src/garage/tests/s3/objects.rs | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index 888a040a..a1e4ce10 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -845,7 +845,9 @@ impl PreconditionHeaders { } fn check(&self, v: &ObjectVersion, etag: &str) -> Result, Error> { - let v_date = UNIX_EPOCH + Duration::from_millis(v.timestamp); + // we store date with ms precision, but headers are precise to the second: truncate + // the timestamp to handle the same-second edge case + let v_date = UNIX_EPOCH + Duration::from_secs(v.timestamp / 1000); // Implemented from https://datatracker.ietf.org/doc/html/rfc7232#section-6 diff --git a/src/garage/tests/s3/objects.rs b/src/garage/tests/s3/objects.rs index d63ac000..53e8231d 100644 --- a/src/garage/tests/s3/objects.rs +++ b/src/garage/tests/s3/objects.rs @@ -198,6 +198,7 @@ async fn test_precondition() { ); } let older_date = DateTime::from_secs_f64(last_modified.as_secs_f64() - 10.0); + let same_date = DateTime::from_secs_f64(last_modified.as_secs_f64()); let newer_date = DateTime::from_secs_f64(last_modified.as_secs_f64() + 10.0); { let err = ctx @@ -212,6 +213,18 @@ async fn test_precondition() { matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304) ); + let err = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_modified_since(same_date) + .send() + .await; + assert!( + matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 304) + ); + let o = ctx .client .get_object() @@ -236,6 +249,17 @@ async fn test_precondition() { matches!(err, Err(SdkError::ServiceError(se)) if se.raw().status().as_u16() == 412) ); + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .if_unmodified_since(same_date) + .send() + .await + .unwrap(); + assert_eq!(o.e_tag.as_ref().unwrap().as_str(), etag); + let o = ctx .client .get_object() From 1c29d04cc5c05aae1b0b7928e573c8f1ac6d65e4 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 14 Oct 2025 11:16:35 +0200 Subject: [PATCH 04/19] sigv4: don't enforce x-amz-content-sha256 to be in signed headers list (fix #770) From the following page: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html > In both cases, because the x-amz-content-sha256 header value is already > part of your HashedPayload, you are not required to include the > x-amz-content-sha256 header as a canonical header. --- src/api/common/signature/payload.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/api/common/signature/payload.rs b/src/api/common/signature/payload.rs index c3a7f231..3939da19 100644 --- a/src/api/common/signature/payload.rs +++ b/src/api/common/signature/payload.rs @@ -104,7 +104,7 @@ async fn check_standard_signature( // Verify that all necessary request headers are included in signed_headers // The following must be included for all signatures: // - the Host header (mandatory) - // - all x-amz-* headers used in the request + // - all x-amz-* headers used in the request (except x-amz-content-sha256) // AWS also indicates that the Content-Type header should be signed if // it is used, but Minio client doesn't sign it so we don't check it for compatibility. let signed_headers = split_signed_headers(&authorization)?; @@ -151,7 +151,7 @@ async fn check_presigned_signature( // Verify that all necessary request headers are included in signed_headers // For AWSv4 pre-signed URLs, the following must be included: // - the Host header (mandatory) - // - all x-amz-* headers used in the request + // - all x-amz-* headers used in the request (except x-amz-content-sha256) let signed_headers = split_signed_headers(&authorization)?; verify_signed_headers(request.headers(), &signed_headers)?; @@ -268,7 +268,9 @@ fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) -> return Err(Error::bad_request("Header `Host` should be signed")); } for (name, _) in headers.iter() { - if name.as_str().starts_with("x-amz-") { + // Enforce signature of all x-amz-* headers, except x-amz-content-sh256 + // because it is included in the canonical request in all cases + if name.as_str().starts_with("x-amz-") && name != X_AMZ_CONTENT_SHA256 { if !signed_headers.contains(name) { return Err(Error::bad_request(format!( "Header `{}` should be signed", @@ -468,8 +470,7 @@ impl Authorization { let date = headers .get(X_AMZ_DATE) - .ok_or_bad_request("Missing X-Amz-Date field") - .map_err(Error::from)? + .ok_or_bad_request("Missing X-Amz-Date field")? .to_str()?; let date = parse_date(date)?; From b43c58cbe553deb970fe403316d621ec57f0fac0 Mon Sep 17 00:00:00 2001 From: fgberry Date: Fri, 24 Oct 2025 11:22:32 +0200 Subject: [PATCH 05/19] fix: default config path changed for alpine binary --- doc/book/cookbook/binary-packages.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/book/cookbook/binary-packages.md b/doc/book/cookbook/binary-packages.md index 0a6ad8fc..8c30b360 100644 --- a/doc/book/cookbook/binary-packages.md +++ b/doc/book/cookbook/binary-packages.md @@ -15,8 +15,10 @@ Alpine Linux repositories (available since v3.17): apk add garage ``` -The default configuration file is installed to `/etc/garage.toml`. You can run -Garage using: `rc-service garage start`. If you don't specify `rpc_secret`, it +The default configuration file is installed to `/etc/garage/garage.toml`. You can run +Garage using: `rc-service garage start`. + +If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start. Please note that this package is built without Consul discovery, Kubernetes From 1aac7b4875b1acde44e98b6a98b343cb659572f5 Mon Sep 17 00:00:00 2001 From: fgberry Date: Fri, 24 Oct 2025 11:25:33 +0200 Subject: [PATCH 06/19] chore: spacing --- doc/book/cookbook/binary-packages.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/book/cookbook/binary-packages.md b/doc/book/cookbook/binary-packages.md index 8c30b360..6476ff51 100644 --- a/doc/book/cookbook/binary-packages.md +++ b/doc/book/cookbook/binary-packages.md @@ -18,8 +18,7 @@ apk add garage The default configuration file is installed to `/etc/garage/garage.toml`. You can run Garage using: `rc-service garage start`. -If you don't specify `rpc_secret`, it -will be automatically replaced with a random string on the first start. +If you don't specify `rpc_secret`, it will be automatically replaced with a random string on the first start. Please note that this package is built without Consul discovery, Kubernetes discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once From 174f4f01a8d2daff681152d62a1a23c5fe7bcc9e Mon Sep 17 00:00:00 2001 From: teo-tsirpanis Date: Sun, 26 Oct 2025 15:54:08 +0000 Subject: [PATCH 07/19] Update link to signature v2. --- doc/book/reference-manual/s3-compatibility.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/book/reference-manual/s3-compatibility.md b/doc/book/reference-manual/s3-compatibility.md index edf8de0d..b869b6f4 100644 --- a/doc/book/reference-manual/s3-compatibility.md +++ b/doc/book/reference-manual/s3-compatibility.md @@ -27,7 +27,7 @@ Feel free to open a PR to suggest fixes this table. Minio is missing because the | Feature | Garage | [Openstack Swift](https://docs.openstack.org/swift/latest/s3_compat.html) | [Ceph Object Gateway](https://docs.ceph.com/en/latest/radosgw/s3/) | [Riak CS](https://docs.riak.com/riak/cs/2.1.1/references/apis/storage/s3/index.html) | [OpenIO](https://docs.openio.io/latest/source/arch-design/s3_compliancy.html) | |------------------------------|----------------------------------|-----------------|---------------|---------|-----| -| [signature v2](https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html) (deprecated) | ❌ Missing | ✅ | ✅ | ✅ | ✅ | +| [signature v2](https://docs.aws.amazon.com/AmazonS3/latest/API/Appendix-Sigv2.html) (deprecated) | ❌ Missing | ✅ | ✅ | ✅ | ✅ | | [signature v4](https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html) | ✅ Implemented | ✅ | ✅ | ❌ | ✅ | | [URL path-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#path-style-access) (eg. `host.tld/bucket/key`) | ✅ Implemented | ✅ | ✅ | ❓| ✅ | | [URL vhost-style](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html#virtual-hosted-style-access) URL (eg. `bucket.host.tld/key`) | ✅ Implemented | ❌| ✅| ✅ | ✅ | From 82297371bff85476ad10d05d98ad8ebe316060a4 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Sat, 1 Nov 2025 17:20:39 +0100 Subject: [PATCH 08/19] migrate to this error it doesn't generate a bazillion warning at compile time --- Cargo.lock | 58 ++++++------------------ Cargo.toml | 3 +- src/api/admin/Cargo.toml | 2 +- src/api/admin/error.rs | 13 +++--- src/api/common/Cargo.toml | 2 +- src/api/common/common_error.rs | 30 ++++++------- src/api/common/signature/error.rs | 12 ++--- src/api/k2v/Cargo.toml | 2 +- src/api/k2v/error.rs | 24 +++++----- src/api/s3/Cargo.toml | 2 +- src/api/s3/error.rs | 46 ++++++++++--------- src/db/Cargo.toml | 2 +- src/db/lib.rs | 6 +-- src/model/Cargo.toml | 2 +- src/model/garage.rs | 6 +-- src/model/helper/error.rs | 14 +++--- src/net/Cargo.toml | 2 +- src/net/endpoint.rs | 4 +- src/net/error.rs | 44 +++++++++--------- src/rpc/Cargo.toml | 4 +- src/rpc/consul.rs | 16 +++---- src/util/Cargo.toml | 2 +- src/util/error.rs | 75 ++++++++++++++----------------- src/web/Cargo.toml | 2 +- src/web/error.rs | 8 ++-- 25 files changed, 172 insertions(+), 209 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c516bfbf..69bad7a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1099,20 +1099,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" -[[package]] -name = "err-derive" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c34a887c8df3ed90498c1c437ce21f211c8e27672921a8ffa293cb8d6d4caa9e" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "rustversion", - "syn 1.0.109", - "synstructure 0.12.6", -] - [[package]] name = "errno" version = "0.3.10" @@ -1340,7 +1326,6 @@ version = "1.3.0" dependencies = [ "argon2", "async-trait", - "err-derive", "futures", "garage_api_common", "garage_model", @@ -1355,6 +1340,7 @@ dependencies = [ "prometheus", "serde", "serde_json", + "thiserror 2.0.12", "tokio", "tracing", "url", @@ -1370,7 +1356,6 @@ dependencies = [ "crc32c", "crc32fast", "crypto-common", - "err-derive", "futures", "garage_model", "garage_table", @@ -1389,6 +1374,7 @@ dependencies = [ "serde_json", "sha1", "sha2", + "thiserror 2.0.12", "tokio", "tracing", "url", @@ -1399,7 +1385,6 @@ name = "garage_api_k2v" version = "1.3.0" dependencies = [ "base64 0.21.7", - "err-derive", "futures", "garage_api_common", "garage_model", @@ -1412,6 +1397,7 @@ dependencies = [ "percent-encoding", "serde", "serde_json", + "thiserror 2.0.12", "tokio", "tracing", "url", @@ -1428,7 +1414,6 @@ dependencies = [ "chrono", "crc32c", "crc32fast", - "err-derive", "form_urlencoded", "futures", "garage_api_common", @@ -1455,6 +1440,7 @@ dependencies = [ "serde_json", "sha1", "sha2", + "thiserror 2.0.12", "tokio", "tokio-stream", "tokio-util 0.7.14", @@ -1491,7 +1477,6 @@ dependencies = [ name = "garage_db" version = "1.3.0" dependencies = [ - "err-derive", "fjall", "heed", "mktemp", @@ -1499,6 +1484,7 @@ dependencies = [ "r2d2", "r2d2_sqlite", "rusqlite", + "thiserror 2.0.12", "tracing", ] @@ -1510,7 +1496,6 @@ dependencies = [ "base64 0.21.7", "blake2", "chrono", - "err-derive", "futures", "garage_block", "garage_db", @@ -1524,6 +1509,7 @@ dependencies = [ "rand", "serde", "serde_bytes", + "thiserror 2.0.12", "tokio", "tracing", "zstd", @@ -1536,7 +1522,6 @@ dependencies = [ "arc-swap", "bytes", "cfg-if", - "err-derive", "futures", "hex", "kuska-handshake", @@ -1549,6 +1534,7 @@ dependencies = [ "rand", "rmp-serde", "serde", + "thiserror 2.0.12", "tokio", "tokio-stream", "tokio-util 0.7.14", @@ -1561,7 +1547,6 @@ dependencies = [ "arc-swap", "async-trait", "bytesize", - "err-derive", "format_table", "futures", "garage_net", @@ -1582,6 +1567,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "thiserror 2.0.12", "tokio", "tracing", ] @@ -1616,7 +1602,6 @@ dependencies = [ "blake2", "bytesize", "chrono", - "err-derive", "futures", "garage_db", "garage_net", @@ -1633,6 +1618,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "thiserror 2.0.12", "tokio", "toml", "tracing", @@ -1643,7 +1629,6 @@ dependencies = [ name = "garage_web" version = "1.3.0" dependencies = [ - "err-derive", "garage_api_common", "garage_api_s3", "garage_model", @@ -1654,6 +1639,7 @@ dependencies = [ "hyper 1.6.0", "opentelemetry", "percent-encoding", + "thiserror 2.0.12", "tokio", "tracing", ] @@ -2445,7 +2431,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "thiserror 1.0.69", + "thiserror 2.0.12", "tokio", "tracing-subscriber", ] @@ -4233,18 +4219,6 @@ dependencies = [ "crossbeam-queue", ] -[[package]] -name = "synstructure" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", - "unicode-xid", -] - [[package]] name = "synstructure" version = "0.13.1" @@ -4764,12 +4738,6 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" -[[package]] -name = "unicode-xid" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" - [[package]] name = "universal-hash" version = "0.5.1" @@ -5272,7 +5240,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.100", - "synstructure 0.13.1", + "synstructure", ] [[package]] @@ -5333,7 +5301,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.100", - "synstructure 0.13.1", + "synstructure", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5bc76e3c..a21ac072 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,6 @@ chrono = "0.4" crc32fast = "1.4" crc32c = "0.6" crypto-common = "0.1" -err-derive = "0.3" gethostname = "0.4" git-version = "0.3.4" hex = "0.4" @@ -137,7 +136,7 @@ prometheus = "0.13" aws-sigv4 = { version = "1.1", default-features = false } hyper-rustls = { version = "0.26", default-features = false, features = ["http1", "http2", "ring", "rustls-native-certs"] } log = "0.4" -thiserror = "1.0" +thiserror = "2.0" # ---- used only as build / dev dependencies ---- assert-json-diff = "2.0" diff --git a/src/api/admin/Cargo.toml b/src/api/admin/Cargo.toml index d7184068..81735a85 100644 --- a/src/api/admin/Cargo.toml +++ b/src/api/admin/Cargo.toml @@ -22,7 +22,7 @@ garage_api_common.workspace = true argon2.workspace = true async-trait.workspace = true -err-derive.workspace = true +thiserror.workspace = true hex.workspace = true tracing.workspace = true diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index 201f9b40..97f02156 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -1,6 +1,6 @@ use std::convert::TryFrom; -use err_derive::Error; +use thiserror::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; @@ -16,20 +16,17 @@ use garage_api_common::helpers::*; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error - Common(#[error(source)] CommonError), + Common(#[from] CommonError), // Category: cannot process /// The API access key does not exist - #[error(display = "Access key not found: {}", _0)] + #[error("Access key not found: {0}")] NoSuchAccessKey(String), /// In Import key, the key already exists - #[error( - display = "Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.", - _0 - )] + #[error("Key {0} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.")] KeyAlreadyExists(String), } diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml index fd159c96..b337cd69 100644 --- a/src/api/common/Cargo.toml +++ b/src/api/common/Cargo.toml @@ -24,7 +24,7 @@ chrono.workspace = true crc32fast.workspace = true crc32c.workspace = true crypto-common.workspace = true -err-derive.workspace = true +thiserror.workspace = true hex.workspace = true hmac.workspace = true md-5.workspace = true diff --git a/src/api/common/common_error.rs b/src/api/common/common_error.rs index 597a3511..1335fece 100644 --- a/src/api/common/common_error.rs +++ b/src/api/common/common_error.rs @@ -1,6 +1,6 @@ use std::convert::TryFrom; -use err_derive::Error; +use thiserror::Error; use hyper::StatusCode; use garage_util::error::Error as GarageError; @@ -12,48 +12,48 @@ use garage_model::helper::error::Error as HelperError; pub enum CommonError { // ---- INTERNAL ERRORS ---- /// Error related to deeper parts of Garage - #[error(display = "Internal error: {}", _0)] - InternalError(#[error(source)] GarageError), + #[error("Internal error: {0}")] + InternalError(#[from] GarageError), /// Error related to Hyper - #[error(display = "Internal error (Hyper error): {}", _0)] - Hyper(#[error(source)] hyper::Error), + #[error("Internal error (Hyper error): {0}")] + Hyper(#[from] hyper::Error), /// Error related to HTTP - #[error(display = "Internal error (HTTP error): {}", _0)] - Http(#[error(source)] http::Error), + #[error("Internal error (HTTP error): {0}")] + Http(#[from] http::Error), // ---- GENERIC CLIENT ERRORS ---- /// Proper authentication was not provided - #[error(display = "Forbidden: {}", _0)] + #[error("Forbidden: {0}")] Forbidden(String), /// Generic bad request response with custom message - #[error(display = "Bad request: {}", _0)] + #[error("Bad request: {0}")] BadRequest(String), /// The client sent a header with invalid value - #[error(display = "Invalid header value: {}", _0)] - InvalidHeader(#[error(source)] hyper::header::ToStrError), + #[error("Invalid header value: {0}")] + InvalidHeader(#[from] hyper::header::ToStrError), // ---- SPECIFIC ERROR CONDITIONS ---- // These have to be error codes referenced in the S3 spec here: // https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList /// The bucket requested don't exists - #[error(display = "Bucket not found: {}", _0)] + #[error("Bucket not found: {0}")] NoSuchBucket(String), /// Tried to create a bucket that already exist - #[error(display = "Bucket already exists")] + #[error("Bucket already exists")] BucketAlreadyExists, /// Tried to delete a non-empty bucket - #[error(display = "Tried to delete a non-empty bucket")] + #[error("Tried to delete a non-empty bucket")] BucketNotEmpty, // Category: bad request /// Bucket name is not valid according to AWS S3 specs - #[error(display = "Invalid bucket name: {}", _0)] + #[error("Invalid bucket name: {0}")] InvalidBucketName(String), } diff --git a/src/api/common/signature/error.rs b/src/api/common/signature/error.rs index b2f396b5..a1b353e1 100644 --- a/src/api/common/signature/error.rs +++ b/src/api/common/signature/error.rs @@ -1,4 +1,4 @@ -use err_derive::Error; +use thiserror::Error; use crate::common_error::CommonError; pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; @@ -6,21 +6,21 @@ pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInterna /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error Common(CommonError), /// Authorization Header Malformed - #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] + #[error("Authorization header malformed, unexpected scope: {0}")] AuthorizationHeaderMalformed(String), // Category: bad request /// The request contained an invalid UTF-8 sequence in its path or in other parameters - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8Str(#[error(source)] std::str::Utf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8Str(#[from] std::str::Utf8Error), /// The provided digest (checksum) value was invalid - #[error(display = "Invalid digest: {}", _0)] + #[error("Invalid digest: {0}")] InvalidDigest(String), } diff --git a/src/api/k2v/Cargo.toml b/src/api/k2v/Cargo.toml index 628d2db1..2b77f676 100644 --- a/src/api/k2v/Cargo.toml +++ b/src/api/k2v/Cargo.toml @@ -20,7 +20,7 @@ garage_util = { workspace = true, features = [ "k2v" ] } garage_api_common.workspace = true base64.workspace = true -err-derive.workspace = true +thiserror.workspace = true tracing.workspace = true futures.workspace = true diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index 257ff893..c860ab98 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -1,4 +1,4 @@ -use err_derive::Error; +use thiserror::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; @@ -14,38 +14,38 @@ use garage_api_common::signature::error::Error as SignatureError; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error - Common(#[error(source)] CommonError), + Common(#[from] CommonError), // Category: cannot process /// Authorization Header Malformed - #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] + #[error("Authorization header malformed, unexpected scope: {0}")] AuthorizationHeaderMalformed(String), /// The provided digest (checksum) value was invalid - #[error(display = "Invalid digest: {}", _0)] + #[error("Invalid digest: {0}")] InvalidDigest(String), /// The object requested don't exists - #[error(display = "Key not found")] + #[error("Key not found")] NoSuchKey, /// Some base64 encoded data was badly encoded - #[error(display = "Invalid base64: {}", _0)] - InvalidBase64(#[error(source)] base64::DecodeError), + #[error("Invalid base64: {0}")] + InvalidBase64(#[from] base64::DecodeError), /// Invalid causality token - #[error(display = "Invalid causality token")] + #[error("Invalid causality token")] InvalidCausalityToken, /// The client asked for an invalid return format (invalid Accept header) - #[error(display = "Not acceptable: {}", _0)] + #[error("Not acceptable: {0}")] NotAcceptable(String), /// The request contained an invalid UTF-8 sequence in its path or in other parameters - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8Str(#[error(source)] std::str::Utf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8Str(#[from] std::str::Utf8Error), } commonErrorDerivative!(Error); diff --git a/src/api/s3/Cargo.toml b/src/api/s3/Cargo.toml index 15f6858c..56f90864 100644 --- a/src/api/s3/Cargo.toml +++ b/src/api/s3/Cargo.toml @@ -29,7 +29,7 @@ bytes.workspace = true chrono.workspace = true crc32fast.workspace = true crc32c.workspace = true -err-derive.workspace = true +thiserror.workspace = true hex.workspace = true tracing.workspace = true md-5.workspace = true diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index 6d4b7a11..6f4dfb5c 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -1,6 +1,6 @@ use std::convert::TryInto; -use err_derive::Error; +use thiserror::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; @@ -25,67 +25,67 @@ use crate::xml as s3_xml; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - #[error(display = "{}", _0)] + #[error("{0}")] /// Error from common error - Common(#[error(source)] CommonError), + Common(#[from] CommonError), // Category: cannot process /// Authorization Header Malformed - #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] + #[error("Authorization header malformed, unexpected scope: {0}")] AuthorizationHeaderMalformed(String), /// The object requested don't exists - #[error(display = "Key not found")] + #[error("Key not found")] NoSuchKey, /// The multipart upload requested don't exists - #[error(display = "Upload not found")] + #[error("Upload not found")] NoSuchUpload, /// Precondition failed (e.g. x-amz-copy-source-if-match) - #[error(display = "At least one of the preconditions you specified did not hold")] + #[error("At least one of the preconditions you specified did not hold")] PreconditionFailed, /// Parts specified in CMU request do not match parts actually uploaded - #[error(display = "Parts given to CompleteMultipartUpload do not match uploaded parts")] + #[error("Parts given to CompleteMultipartUpload do not match uploaded parts")] InvalidPart, /// Parts given to CompleteMultipartUpload were not in ascending order - #[error(display = "Parts given to CompleteMultipartUpload were not in ascending order")] + #[error("Parts given to CompleteMultipartUpload were not in ascending order")] InvalidPartOrder, /// In CompleteMultipartUpload: not enough data /// (here we are more lenient than AWS S3) - #[error(display = "Proposed upload is smaller than the minimum allowed object size")] + #[error("Proposed upload is smaller than the minimum allowed object size")] EntityTooSmall, // Category: bad request /// The request contained an invalid UTF-8 sequence in its path or in other parameters - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8Str(#[error(source)] std::str::Utf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8Str(#[from] std::str::Utf8Error), /// The request used an invalid path - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8String(#[error(source)] std::string::FromUtf8Error), + #[error("Invalid UTF-8: {0}")] + InvalidUtf8String(#[from] std::string::FromUtf8Error), /// The client sent invalid XML data - #[error(display = "Invalid XML: {}", _0)] + #[error("Invalid XML: {0}")] InvalidXml(String), /// The client sent a range header with invalid value - #[error(display = "Invalid HTTP range: {:?}", _0)] - InvalidRange(#[error(from)] (http_range::HttpRangeParseError, u64)), + #[error("Invalid HTTP range: {0:?}")] + InvalidRange((http_range::HttpRangeParseError, u64)), /// The client sent a range header with invalid value - #[error(display = "Invalid encryption algorithm: {:?}, should be AES256", _0)] + #[error("Invalid encryption algorithm: {0:?}, should be AES256")] InvalidEncryptionAlgorithm(String), /// The provided digest (checksum) value was invalid - #[error(display = "Invalid digest: {}", _0)] + #[error("Invalid digest: {0}")] InvalidDigest(String), /// The client sent a request for an action not supported by garage - #[error(display = "Unimplemented action: {}", _0)] + #[error("Unimplemented action: {0}")] NotImplemented(String), } @@ -99,6 +99,12 @@ impl From for Error { } } +impl From<(http_range::HttpRangeParseError, u64)> for Error { + fn from (err: (http_range::HttpRangeParseError, u64)) -> Error { + Error::InvalidRange(err) + } +} + impl From for Error { fn from(err: roxmltree::Error) -> Self { Self::InvalidXml(format!("{}", err)) diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index 6dee2fa6..7c1c8d90 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -12,7 +12,7 @@ readme = "../../README.md" path = "lib.rs" [dependencies] -err-derive.workspace = true +thiserror.workspace = true tracing.workspace = true heed = { workspace = true, optional = true } diff --git a/src/db/lib.rs b/src/db/lib.rs index 71826255..2a467c7c 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -20,7 +20,7 @@ use std::cell::Cell; use std::path::PathBuf; use std::sync::Arc; -use err_derive::Error; +use thiserror::Error; pub use open::*; @@ -44,7 +44,7 @@ pub type TxValueIter<'a> = Box); impl From for Error { @@ -56,7 +56,7 @@ impl From for Error { pub type Result = std::result::Result; #[derive(Debug, Error)] -#[error(display = "{}", _0)] +#[error("{0}")] pub struct TxOpError(pub(crate) Error); pub type TxOpResult = std::result::Result; diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index e59765d7..579092d2 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -24,7 +24,7 @@ garage_net.workspace = true async-trait.workspace = true blake2.workspace = true chrono.workspace = true -err-derive.workspace = true +thiserror.workspace = true hex.workspace = true http.workspace = true base64.workspace = true diff --git a/src/model/garage.rs b/src/model/garage.rs index 38f8f1f7..f4f6f693 100644 --- a/src/model/garage.rs +++ b/src/model/garage.rs @@ -315,15 +315,15 @@ impl Garage { Ok(()) } - pub fn bucket_helper(&self) -> helper::bucket::BucketHelper { + pub fn bucket_helper(&self) -> helper::bucket::BucketHelper<'_> { helper::bucket::BucketHelper(self) } - pub fn key_helper(&self) -> helper::key::KeyHelper { + pub fn key_helper(&self) -> helper::key::KeyHelper<'_> { helper::key::KeyHelper(self) } - pub async fn locked_helper(&self) -> helper::locked::LockedHelper { + pub async fn locked_helper(&self) -> helper::locked::LockedHelper<'_> { let lock = self.bucket_lock.lock().await; helper::locked::LockedHelper(self, Some(lock)) } diff --git a/src/model/helper/error.rs b/src/model/helper/error.rs index e2ffdd68..6a78546d 100644 --- a/src/model/helper/error.rs +++ b/src/model/helper/error.rs @@ -1,24 +1,24 @@ -use err_derive::Error; +use thiserror::Error; use serde::{Deserialize, Serialize}; use garage_util::error::Error as GarageError; #[derive(Debug, Error, Serialize, Deserialize)] pub enum Error { - #[error(display = "Internal error: {}", _0)] - Internal(#[error(source)] GarageError), + #[error("Internal error: {0}")] + Internal(#[from] GarageError), - #[error(display = "Bad request: {}", _0)] + #[error("Bad request: {0}")] BadRequest(String), /// Bucket name is not valid according to AWS S3 specs - #[error(display = "Invalid bucket name: {}", _0)] + #[error("Invalid bucket name: {0}")] InvalidBucketName(String), - #[error(display = "Access key not found: {}", _0)] + #[error("Access key not found: {0}")] NoSuchAccessKey(String), - #[error(display = "Bucket not found: {}", _0)] + #[error("Bucket not found: {0}")] NoSuchBucket(String), } diff --git a/src/net/Cargo.toml b/src/net/Cargo.toml index 83b3b15b..8ff78680 100644 --- a/src/net/Cargo.toml +++ b/src/net/Cargo.toml @@ -30,7 +30,7 @@ rand.workspace = true log.workspace = true arc-swap.workspace = true -err-derive.workspace = true +thiserror.workspace = true bytes.workspace = true cfg-if.workspace = true diff --git a/src/net/endpoint.rs b/src/net/endpoint.rs index d46acc42..3ab1048a 100644 --- a/src/net/endpoint.rs +++ b/src/net/endpoint.rs @@ -159,7 +159,7 @@ where pub(crate) type DynEndpoint = Box; pub(crate) trait GenericEndpoint { - fn handle(&self, req_enc: ReqEnc, from: NodeID) -> BoxFuture>; + fn handle(&self, req_enc: ReqEnc, from: NodeID) -> BoxFuture<'_, Result>; fn drop_handler(&self); fn clone_endpoint(&self) -> DynEndpoint; } @@ -175,7 +175,7 @@ where M: Message, H: StreamingEndpointHandler + 'static, { - fn handle(&self, req_enc: ReqEnc, from: NodeID) -> BoxFuture> { + fn handle(&self, req_enc: ReqEnc, from: NodeID) -> BoxFuture<'_, Result> { async move { match self.0.handler.load_full() { None => Err(Error::NoHandler), diff --git a/src/net/error.rs b/src/net/error.rs index cddb1eaa..899fe21c 100644 --- a/src/net/error.rs +++ b/src/net/error.rs @@ -1,49 +1,49 @@ use std::io; -use err_derive::Error; +use thiserror::Error; use log::error; #[derive(Debug, Error)] pub enum Error { - #[error(display = "IO error: {}", _0)] - Io(#[error(source)] io::Error), + #[error("IO error: {0}")] + Io(#[from] io::Error), - #[error(display = "Messagepack encode error: {}", _0)] - RMPEncode(#[error(source)] rmp_serde::encode::Error), - #[error(display = "Messagepack decode error: {}", _0)] - RMPDecode(#[error(source)] rmp_serde::decode::Error), + #[error("Messagepack encode error: {0}")] + RMPEncode(#[from] rmp_serde::encode::Error), + #[error("Messagepack decode error: {0}")] + RMPDecode(#[from] rmp_serde::decode::Error), - #[error(display = "Tokio join error: {}", _0)] - TokioJoin(#[error(source)] tokio::task::JoinError), + #[error("Tokio join error: {0}")] + TokioJoin(#[from] tokio::task::JoinError), - #[error(display = "oneshot receive error: {}", _0)] - OneshotRecv(#[error(source)] tokio::sync::oneshot::error::RecvError), + #[error("oneshot receive error: {0}")] + OneshotRecv(#[from] tokio::sync::oneshot::error::RecvError), - #[error(display = "Handshake error: {}", _0)] - Handshake(#[error(source)] kuska_handshake::async_std::Error), + #[error("Handshake error: {0}")] + Handshake(#[from] kuska_handshake::async_std::Error), - #[error(display = "UTF8 error: {}", _0)] - UTF8(#[error(source)] std::string::FromUtf8Error), + #[error("UTF8 error: {0}")] + UTF8(#[from] std::string::FromUtf8Error), - #[error(display = "Framing protocol error")] + #[error("Framing protocol error")] Framing, - #[error(display = "Remote error ({:?}): {}", _0, _1)] + #[error("Remote error ({0:?}): {1}")] Remote(io::ErrorKind, String), - #[error(display = "Request ID collision")] + #[error("Request ID collision")] IdCollision, - #[error(display = "{}", _0)] + #[error("{0}")] Message(String), - #[error(display = "No handler / shutting down")] + #[error("No handler / shutting down")] NoHandler, - #[error(display = "Connection closed")] + #[error("Connection closed")] ConnectionClosed, - #[error(display = "Version mismatch: {}", _0)] + #[error("Version mismatch: {0}")] VersionMismatch(String), } diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 1e764c77..9e886748 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -33,7 +33,7 @@ async-trait.workspace = true serde.workspace = true serde_bytes.workspace = true serde_json.workspace = true -err-derive = { workspace = true, optional = true } +thiserror = { workspace = true, optional = true } # newer version requires rust edition 2021 kube = { workspace = true, optional = true } @@ -49,5 +49,5 @@ opentelemetry.workspace = true [features] kubernetes-discovery = [ "kube", "k8s-openapi", "schemars" ] -consul-discovery = [ "reqwest", "err-derive" ] +consul-discovery = [ "reqwest", "thiserror" ] system-libs = [ "sodiumoxide/use-pkg-config" ] diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs index f088bf3f..801e937f 100644 --- a/src/rpc/consul.rs +++ b/src/rpc/consul.rs @@ -3,7 +3,7 @@ use std::fs::File; use std::io::Read; use std::net::{IpAddr, SocketAddr}; -use err_derive::Error; +use thiserror::Error; use serde::{Deserialize, Serialize}; use garage_net::NodeID; @@ -219,12 +219,12 @@ impl ConsulDiscovery { /// Regroup all Consul discovery errors #[derive(Debug, Error)] pub enum ConsulError { - #[error(display = "IO error: {}", _0)] - Io(#[error(source)] std::io::Error), - #[error(display = "HTTP error: {}", _0)] - Reqwest(#[error(source)] reqwest::Error), - #[error(display = "Invalid Consul TLS configuration")] + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("HTTP error: {0}")] + Reqwest(#[from] reqwest::Error), + #[error("Invalid Consul TLS configuration")] InvalidTLSConfig, - #[error(display = "Token error: {}", _0)] - Token(#[error(source)] reqwest::header::InvalidHeaderValue), + #[error("Token error: {0}")] + Token(#[from] reqwest::header::InvalidHeaderValue), } diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 0d693a97..b5c1454f 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -21,7 +21,7 @@ arc-swap.workspace = true async-trait.workspace = true blake2.workspace = true bytesize.workspace = true -err-derive.workspace = true +thiserror.workspace = true hexdump.workspace = true xxhash-rust.workspace = true hex.workspace = true diff --git a/src/util/error.rs b/src/util/error.rs index 75fd3f9c..170d2687 100644 --- a/src/util/error.rs +++ b/src/util/error.rs @@ -2,7 +2,7 @@ use std::fmt; use std::io; -use err_derive::Error; +use thiserror::Error; use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer}; @@ -12,68 +12,61 @@ use crate::encode::debug_serialize; /// Regroup all Garage errors #[derive(Debug, Error)] pub enum Error { - #[error(display = "IO error: {}", _0)] - Io(#[error(source)] io::Error), + #[error("IO error: {0}")] + Io(#[from] io::Error), - #[error(display = "Hyper error: {}", _0)] - Hyper(#[error(source)] hyper::Error), + #[error("Hyper error: {0}")] + Hyper(#[from] hyper::Error), - #[error(display = "HTTP error: {}", _0)] - Http(#[error(source)] http::Error), + #[error("HTTP error: {0}")] + Http(#[from] http::Error), - #[error(display = "Invalid HTTP header value: {}", _0)] - HttpHeader(#[error(source)] http::header::ToStrError), + #[error("Invalid HTTP header value: {0}")] + HttpHeader(#[from] http::header::ToStrError), - #[error(display = "Network error: {}", _0)] - Net(#[error(source)] garage_net::error::Error), + #[error("Network error: {0}")] + Net(#[from] garage_net::error::Error), - #[error(display = "DB error: {}", _0)] - Db(#[error(source)] garage_db::Error), + #[error("DB error: {0}")] + Db(#[from] garage_db::Error), - #[error(display = "Messagepack encode error: {}", _0)] - RmpEncode(#[error(source)] rmp_serde::encode::Error), - #[error(display = "Messagepack decode error: {}", _0)] - RmpDecode(#[error(source)] rmp_serde::decode::Error), - #[error(display = "JSON error: {}", _0)] - Json(#[error(source)] serde_json::error::Error), - #[error(display = "TOML decode error: {}", _0)] - TomlDecode(#[error(source)] toml::de::Error), + #[error("Messagepack encode error: {0}")] + RmpEncode(#[from] rmp_serde::encode::Error), + #[error("Messagepack decode error: {0}")] + RmpDecode(#[from] rmp_serde::decode::Error), + #[error("JSON error: {0}")] + Json(#[from] serde_json::error::Error), + #[error("TOML decode error: {0}")] + TomlDecode(#[from] toml::de::Error), - #[error(display = "Tokio join error: {}", _0)] - TokioJoin(#[error(source)] tokio::task::JoinError), + #[error("Tokio join error: {0}")] + TokioJoin(#[from] tokio::task::JoinError), - #[error(display = "Tokio semaphore acquire error: {}", _0)] - TokioSemAcquire(#[error(source)] tokio::sync::AcquireError), + #[error("Tokio semaphore acquire error: {0}")] + TokioSemAcquire(#[from] tokio::sync::AcquireError), - #[error(display = "Tokio broadcast receive error: {}", _0)] - TokioBcastRecv(#[error(source)] tokio::sync::broadcast::error::RecvError), + #[error("Tokio broadcast receive error: {0}")] + TokioBcastRecv(#[from] tokio::sync::broadcast::error::RecvError), - #[error(display = "Remote error: {}", _0)] + #[error("Remote error: {0}")] RemoteError(String), - #[error(display = "Timeout")] + #[error("Timeout")] Timeout, - #[error( - display = "Could not reach quorum of {} (sets={:?}). {} of {} request succeeded, others returned errors: {:?}", - _0, - _1, - _2, - _3, - _4 - )] + #[error("Could not reach quorum of {0} (sets={1:?}). {2} of {3} request succeeded, others returned errors: {4:?}")] Quorum(usize, Option, usize, usize, Vec), - #[error(display = "Unexpected RPC message: {}", _0)] + #[error("Unexpected RPC message: {0}")] UnexpectedRpcMessage(String), - #[error(display = "Corrupt data: does not match hash {:?}", _0)] + #[error("Corrupt data: does not match hash {0:?}")] CorruptData(Hash), - #[error(display = "Missing block {:?}: no node returned a valid block", _0)] + #[error("Missing block {0:?}: no node returned a valid block")] MissingBlock(Hash), - #[error(display = "{}", _0)] + #[error("{0}")] Message(String), } diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index c1056509..a2daf84d 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -20,7 +20,7 @@ garage_model.workspace = true garage_util.workspace = true garage_table.workspace = true -err-derive.workspace = true +thiserror.workspace = true tracing.workspace = true percent-encoding.workspace = true diff --git a/src/web/error.rs b/src/web/error.rs index 7e6d4542..49650b1d 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -1,4 +1,4 @@ -use err_derive::Error; +use thiserror::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; @@ -8,15 +8,15 @@ use garage_api_common::generic_server::ApiError; #[derive(Debug, Error)] pub enum Error { /// An error received from the API crate - #[error(display = "API error: {}", _0)] + #[error("API error: {0}")] ApiError(garage_api_s3::error::Error), /// The file does not exist - #[error(display = "Not found")] + #[error("Not found")] NotFound, /// The client sent a request without host, or with unsupported method - #[error(display = "Bad request: {}", _0)] + #[error("Bad request: {0}")] BadRequest(String), } From ac851d6dee762576415541cf1b6eb5345d03ea9b Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Sat, 1 Nov 2025 18:04:54 +0100 Subject: [PATCH 09/19] fmt --- src/api/admin/error.rs | 2 +- src/api/common/common_error.rs | 2 +- src/api/k2v/error.rs | 2 +- src/api/s3/error.rs | 8 ++++---- src/model/helper/error.rs | 2 +- src/net/error.rs | 2 +- src/rpc/consul.rs | 2 +- src/web/error.rs | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index 97f02156..17d4c200 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -1,8 +1,8 @@ use std::convert::TryFrom; -use thiserror::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; pub use garage_model::helper::error::Error as HelperError; diff --git a/src/api/common/common_error.rs b/src/api/common/common_error.rs index 1335fece..e596a6e9 100644 --- a/src/api/common/common_error.rs +++ b/src/api/common/common_error.rs @@ -1,7 +1,7 @@ use std::convert::TryFrom; -use thiserror::Error; use hyper::StatusCode; +use thiserror::Error; use garage_util::error::Error as GarageError; diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index c860ab98..f1937fe5 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -1,6 +1,6 @@ -use thiserror::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; use garage_api_common::common_error::{commonErrorDerivative, CommonError}; pub(crate) use garage_api_common::common_error::{helper_error_as_internal, pass_helper_error}; diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index 6f4dfb5c..64112084 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -1,8 +1,8 @@ use std::convert::TryInto; -use thiserror::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; use garage_model::helper::error::Error as HelperError; @@ -100,9 +100,9 @@ impl From for Error { } impl From<(http_range::HttpRangeParseError, u64)> for Error { - fn from (err: (http_range::HttpRangeParseError, u64)) -> Error { - Error::InvalidRange(err) - } + fn from(err: (http_range::HttpRangeParseError, u64)) -> Error { + Error::InvalidRange(err) + } } impl From for Error { diff --git a/src/model/helper/error.rs b/src/model/helper/error.rs index 6a78546d..bc483c7d 100644 --- a/src/model/helper/error.rs +++ b/src/model/helper/error.rs @@ -1,5 +1,5 @@ -use thiserror::Error; use serde::{Deserialize, Serialize}; +use thiserror::Error; use garage_util::error::Error as GarageError; diff --git a/src/net/error.rs b/src/net/error.rs index 899fe21c..f67794ed 100644 --- a/src/net/error.rs +++ b/src/net/error.rs @@ -1,7 +1,7 @@ use std::io; -use thiserror::Error; use log::error; +use thiserror::Error; #[derive(Debug, Error)] pub enum Error { diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs index 801e937f..760e9fcb 100644 --- a/src/rpc/consul.rs +++ b/src/rpc/consul.rs @@ -3,8 +3,8 @@ use std::fs::File; use std::io::Read; use std::net::{IpAddr, SocketAddr}; -use thiserror::Error; use serde::{Deserialize, Serialize}; +use thiserror::Error; use garage_net::NodeID; diff --git a/src/web/error.rs b/src/web/error.rs index 49650b1d..aef74923 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -1,6 +1,6 @@ -use thiserror::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; +use thiserror::Error; use garage_api_common::generic_server::ApiError; From a057ab23ea19221e9c646bc55092fe7c20648e80 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 24 Nov 2025 11:09:46 +0100 Subject: [PATCH 10/19] Update rust toolchain --- flake.lock | 16 ++++++++-------- flake.nix | 8 ++++---- nix/compile.nix | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/flake.lock b/flake.lock index 2cfbfda4..211b70e0 100644 --- a/flake.lock +++ b/flake.lock @@ -50,17 +50,17 @@ }, "nixpkgs": { "locked": { - "lastModified": 1736692550, - "narHash": "sha256-7tk8xH+g0sJkKLTJFOxphJxxOjMDFMWv24nXslaU2ro=", + "lastModified": 1763977559, + "narHash": "sha256-g4MKqsIRy5yJwEsI+fYODqLUnAqIY4kZai0nldAP6EM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815", + "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632", "type": "github" }, "original": { "owner": "NixOS", "repo": "nixpkgs", - "rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815", + "rev": "cfe2c7d5b5d3032862254e68c37a6576b633d632", "type": "github" } }, @@ -80,17 +80,17 @@ ] }, "locked": { - "lastModified": 1738549608, - "narHash": "sha256-GdyT9QEUSx5k/n8kILuNy83vxxdyUfJ8jL5mMpQZWfw=", + "lastModified": 1763952169, + "narHash": "sha256-+PeDBD8P+NKauH+w7eO/QWCIp8Cx4mCfWnh9sJmy9CM=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d", + "rev": "ab726555a9a72e6dc80649809147823a813fa95b", "type": "github" }, "original": { "owner": "oxalica", "repo": "rust-overlay", - "rev": "35c6f8c4352f995ecd53896200769f80a3e8f22d", + "rev": "ab726555a9a72e6dc80649809147823a813fa95b", "type": "github" } }, diff --git a/flake.nix b/flake.nix index 2fb8c48e..48880347 100644 --- a/flake.nix +++ b/flake.nix @@ -2,13 +2,13 @@ description = "Garage, an S3-compatible distributed object store for self-hosted deployments"; - # Nixpkgs 24.11 as of 2025-01-12 + # Nixpkgs 25.05 as of 2025-11-24 inputs.nixpkgs.url = - "github:NixOS/nixpkgs/7c4869c47090dd7f9f1bdfb49a22aea026996815"; + "github:NixOS/nixpkgs/cfe2c7d5b5d3032862254e68c37a6576b633d632"; - # Rust overlay as of 2025-02-03 + # Rust overlay as of 2025-11-24 inputs.rust-overlay.url = - "github:oxalica/rust-overlay/35c6f8c4352f995ecd53896200769f80a3e8f22d"; + "github:oxalica/rust-overlay/ab726555a9a72e6dc80649809147823a813fa95b"; inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs"; inputs.crane.url = "github:ipetkov/crane"; diff --git a/nix/compile.nix b/nix/compile.nix index 7e9f79ab..5a49526f 100644 --- a/nix/compile.nix +++ b/nix/compile.nix @@ -48,7 +48,7 @@ let inherit (pkgs) lib stdenv; - toolchainFn = (p: p.rust-bin.stable."1.82.0".default.override { + toolchainFn = (p: p.rust-bin.stable."1.91.0".default.override { targets = lib.optionals (target != null) [ rustTarget ]; extensions = [ "rust-src" From ca3b4a050d25cbfe774cb0db14aefdc61a1f2446 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 24 Nov 2025 17:03:02 +0100 Subject: [PATCH 11/19] update nixos image used in woodpecker ci --- .woodpecker/debug.yaml | 12 ++++++------ .woodpecker/publish.yaml | 4 ++-- .woodpecker/release.yaml | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.woodpecker/debug.yaml b/.woodpecker/debug.yaml index 4c729672..0f60b4e3 100644 --- a/.woodpecker/debug.yaml +++ b/.woodpecker/debug.yaml @@ -12,32 +12,32 @@ when: steps: - name: check formatting - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-shell --attr devShell --run "cargo fmt -- --check" - name: build - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build -j4 --attr flakePackages.dev - name: unit + func tests (lmdb) - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build -j4 --attr flakePackages.tests-lmdb - name: unit + func tests (sqlite) - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build -j4 --attr flakePackages.tests-sqlite - name: unit + func tests (fjall) - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build -j4 --attr flakePackages.tests-fjall - name: integration tests - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build -j4 --attr flakePackages.dev - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) diff --git a/.woodpecker/publish.yaml b/.woodpecker/publish.yaml index 24a84463..8f3b482f 100644 --- a/.woodpecker/publish.yaml +++ b/.woodpecker/publish.yaml @@ -11,7 +11,7 @@ depends_on: steps: - name: refresh-index - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 environment: AWS_ACCESS_KEY_ID: from_secret: garagehq_aws_access_key_id @@ -22,7 +22,7 @@ steps: - nix-shell --attr ci --run "refresh_index" - name: multiarch-docker - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 environment: DOCKER_AUTH: from_secret: docker_auth diff --git a/.woodpecker/release.yaml b/.woodpecker/release.yaml index bf2bd8ba..a94a9ccf 100644 --- a/.woodpecker/release.yaml +++ b/.woodpecker/release.yaml @@ -19,17 +19,17 @@ matrix: steps: - name: build - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} - name: check is static binary - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage" - name: integration tests - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) when: @@ -39,7 +39,7 @@ steps: ARCH: i386 - name: upgrade tests - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 commands: - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false) when: @@ -47,7 +47,7 @@ steps: ARCH: amd64 - name: push static binary - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 environment: TARGET: "${TARGET}" AWS_ACCESS_KEY_ID: @@ -58,7 +58,7 @@ steps: - nix-shell --attr ci --run "to_s3" - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 + image: nixpkgs/nix:nixos-24.05 environment: DOCKER_PLATFORM: "linux/${ARCH}" CONTAINER_NAME: "dxflrs/${ARCH}_garage" From ca296477f3adc024b6606712c4f47b8ef877868f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 24 Nov 2025 17:56:28 +0100 Subject: [PATCH 12/19] disable checksums in aws cli (todo: revert in main-v2) --- script/dev-env-aws.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/script/dev-env-aws.sh b/script/dev-env-aws.sh index 808f9cf1..41f1fdde 100644 --- a/script/dev-env-aws.sh +++ b/script/dev-env-aws.sh @@ -1,6 +1,7 @@ export AWS_ACCESS_KEY_ID=`cat /tmp/garage.s3 |cut -d' ' -f1` export AWS_SECRET_ACCESS_KEY=`cat /tmp/garage.s3 |cut -d' ' -f2` export AWS_DEFAULT_REGION='garage' +export AWS_REQUEST_CHECKSUM_CALCULATION='when_required' # FUTUREWORK: set AWS_ENDPOINT_URL instead, once nixpkgs bumps awscli to >=2.13.0. function aws { command aws --endpoint-url http://127.0.0.1:3911 $@ ; } From 95693d45b20c08122c9b9cdcb259f9c70d233522 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 24 Nov 2025 18:09:53 +0100 Subject: [PATCH 13/19] run cargo fmt as a nix derivation --- .woodpecker/debug.yaml | 2 +- flake.nix | 8 ++++++++ nix/compile.nix | 11 +++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.woodpecker/debug.yaml b/.woodpecker/debug.yaml index 0f60b4e3..4dc7d3c9 100644 --- a/.woodpecker/debug.yaml +++ b/.woodpecker/debug.yaml @@ -14,7 +14,7 @@ steps: - name: check formatting image: nixpkgs/nix:nixos-24.05 commands: - - nix-shell --attr devShell --run "cargo fmt -- --check" + - nix-build -j4 --attr flakePackages.fmt - name: build image: nixpkgs/nix:nixos-24.05 diff --git a/flake.nix b/flake.nix index 48880347..01a077c4 100644 --- a/flake.nix +++ b/flake.nix @@ -30,6 +30,10 @@ inherit system nixpkgs crane rust-overlay extraTestEnv; release = false; }).garage-test; + lints = (compile { + inherit system nixpkgs crane rust-overlay; + release = false; + }); in { packages = { @@ -56,6 +60,10 @@ tests-fjall = testWith { GARAGE_TEST_INTEGRATION_DB_ENGINE = "fjall"; }; + + # lints (fmt, clippy) + fmt = lints.garage-cargo-fmt; + clippy = lints.garage-cargo-clippy; }; # ---- developpment shell, for making native builds only ---- diff --git a/nix/compile.nix b/nix/compile.nix index 5a49526f..c6df9dbd 100644 --- a/nix/compile.nix +++ b/nix/compile.nix @@ -190,4 +190,15 @@ in rec { pkgs.cacert ]; } // extraTestEnv); + + # ---- source code linting ---- + + garage-cargo-fmt = craneLib.cargoFmt (commonArgs // { + cargoExtraArgs = ""; + }); + + garage-cargo-clippy = craneLib.cargoClippy (commonArgs // { + cargoArtifacts = garage-deps; + cargoClippyExtraArgs = "--all-targets -- -D warnings"; + }); } From 511cf0c6ec3d4ab6cd5a40cd0be299765e15671e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 24 Nov 2025 18:37:34 +0100 Subject: [PATCH 14/19] disable awscli checksumming in ci scripts required because garage.deuxfleurs.fr is still running v1.x --- shell.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shell.nix b/shell.nix index cfccfe94..c3dedca8 100644 --- a/shell.nix +++ b/shell.nix @@ -34,6 +34,8 @@ in jq ]; shellHook = '' + export AWS_REQUEST_CHECKSUM_CALCULATION='when_required' + function to_s3 { aws \ --endpoint-url https://garage.deuxfleurs.fr \ From 4d124e1c76b1ffdd9e1944500c0587194a9aa05d Mon Sep 17 00:00:00 2001 From: "perrynzhou@gmail.com" Date: Wed, 10 Dec 2025 06:43:51 +0800 Subject: [PATCH 15/19] Add the parameter, which replaces . This is to accommodate different storage media such as HDD and NVMe. --- src/api/s3/put.rs | 4 +--- src/util/config.rs | 8 ++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index 830a7998..5f8845e7 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -39,8 +39,6 @@ use crate::encryption::EncryptionParams; use crate::error::*; use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION; -const PUT_BLOCKS_MAX_PARALLEL: usize = 3; - pub(crate) struct SaveStreamResult { pub(crate) version_uuid: Uuid, pub(crate) version_timestamp: u64, @@ -493,7 +491,7 @@ pub(crate) async fn read_and_put_blocks> + }; let recv_next = async { // If more than a maximum number of writes are in progress, don't add more for now - if currently_running >= PUT_BLOCKS_MAX_PARALLEL { + if currently_running >= ctx.garage.config.put_blocks_max_parallel { futures::future::pending().await } else { block_rx3.recv().await diff --git a/src/util/config.rs b/src/util/config.rs index e351185f..76b40aa9 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -45,6 +45,11 @@ pub struct Config { )] pub block_size: usize, + /// Maximum number of parallel block writes per PUT request + /// Higher values improve throughput but increase memory usage + /// Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD + #[serde(default = "default_put_blocks_max_parallel")] + pub put_blocks_max_parallel: usize, /// Number of replicas. Can be any positive integer, but uneven numbers are more favorable. /// - 1 for single-node clusters, or to disable replication /// - 3 is the recommended and supported setting. @@ -267,6 +272,9 @@ pub struct KubernetesDiscoveryConfig { pub skip_crd: bool, } +pub fn default_put_blocks_max_parallel() -> usize { + 3 +} /// Read and parse configuration pub fn read_config(config_file: PathBuf) -> Result { let config = std::fs::read_to_string(config_file)?; From e3a5ec6ef6ab1cc4741e1e26f10aa6cde591a214 Mon Sep 17 00:00:00 2001 From: "perrynzhou@gmail.com" Date: Fri, 12 Dec 2025 07:09:38 +0800 Subject: [PATCH 16/19] rename put_blocks_max_parallel to block_max_concurrent_writes_per_request and update configuration.md --- doc/book/reference-manual/configuration.md | 11 ++++++++++- src/api/s3/put.rs | 2 +- src/util/config.rs | 6 +++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index c6dce089..1f583fe6 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -25,7 +25,7 @@ db_engine = "lmdb" block_size = "1M" block_ram_buffer_max = "256MiB" block_max_concurrent_reads = 16 - +block_max_concurrent_writes_per_request =10 lmdb_map_size = "1T" compression_level = 1 @@ -99,6 +99,7 @@ Top-level configuration options, in alphabetical order: [`allow_world_readable_secrets`](#allow_world_readable_secrets), [`block_max_concurrent_reads`](`block_max_concurrent_reads), [`block_ram_buffer_max`](#block_ram_buffer_max), +[`block_max_concurrent_writes_per_request`](#block_max_concurrent_writes_per_request), [`block_size`](#block_size), [`bootstrap_peers`](#bootstrap_peers), [`compression_level`](#compression_level), @@ -547,6 +548,14 @@ metric in Prometheus: a non-zero number of such events indicates an I/O bottleneck on HDD read speed. +#### `block_max_concurrent_writes_per_request` (since `v2.1.0`) {#block_max_concurrent_writes_per_request} + +This parameter is designed to adapt to the concurrent write performance of +different storage media.Maximum number of parallel block writes per put request +Higher values improve throughput but increase memory usage. + +Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD + #### `lmdb_map_size` {#lmdb_map_size} This parameters can be used to set the map size used by LMDB, diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index 5f8845e7..b915f2ec 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -491,7 +491,7 @@ pub(crate) async fn read_and_put_blocks> + }; let recv_next = async { // If more than a maximum number of writes are in progress, don't add more for now - if currently_running >= ctx.garage.config.put_blocks_max_parallel { + if currently_running >= ctx.garage.config.block_max_concurrent_writes_per_request { futures::future::pending().await } else { block_rx3.recv().await diff --git a/src/util/config.rs b/src/util/config.rs index 76b40aa9..eb889ebe 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -48,8 +48,8 @@ pub struct Config { /// Maximum number of parallel block writes per PUT request /// Higher values improve throughput but increase memory usage /// Default: 3, Recommended: 10-30 for NVMe, 3-10 for HDD - #[serde(default = "default_put_blocks_max_parallel")] - pub put_blocks_max_parallel: usize, + #[serde(default = "default_block_max_concurrent_writes_per_request")] + pub block_max_concurrent_writes_per_request: usize, /// Number of replicas. Can be any positive integer, but uneven numbers are more favorable. /// - 1 for single-node clusters, or to disable replication /// - 3 is the recommended and supported setting. @@ -272,7 +272,7 @@ pub struct KubernetesDiscoveryConfig { pub skip_crd: bool, } -pub fn default_put_blocks_max_parallel() -> usize { +pub fn default_block_max_concurrent_writes_per_request() -> usize { 3 } /// Read and parse configuration From dcc2fe4ac549e07bbefa1879743e7bd42296dcc5 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Tue, 16 Dec 2025 10:16:44 +0100 Subject: [PATCH 17/19] docs: fix typo in doc/book/cookbook/kubernetes.md --- doc/book/cookbook/kubernetes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/book/cookbook/kubernetes.md b/doc/book/cookbook/kubernetes.md index 1e7674d7..f5bceec8 100644 --- a/doc/book/cookbook/kubernetes.md +++ b/doc/book/cookbook/kubernetes.md @@ -11,7 +11,7 @@ Firstly clone the repository: ```bash git clone https://git.deuxfleurs.fr/Deuxfleurs/garage -cd garage/scripts/helm +cd garage/script/helm ``` Deploy with default options: From bf5290036f16563818293d3367ac84a024f46587 Mon Sep 17 00:00:00 2001 From: Pierre Mavro Date: Thu, 18 Dec 2025 18:12:22 +0100 Subject: [PATCH 18/19] feat: add service annotations --- script/helm/garage/templates/service.yaml | 6 +++++- script/helm/garage/values.yaml | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/script/helm/garage/templates/service.yaml b/script/helm/garage/templates/service.yaml index 37218872..887c90d0 100644 --- a/script/helm/garage/templates/service.yaml +++ b/script/helm/garage/templates/service.yaml @@ -4,6 +4,10 @@ metadata: name: {{ include "garage.fullname" . }} labels: {{- include "garage.labels" . | nindent 4 }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: type: {{ .Values.service.type }} ports: @@ -37,4 +41,4 @@ spec: name: metrics selector: {{- include "garage.selectorLabels" . | nindent 4 }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml index bbb60db2..5e419fe2 100644 --- a/script/helm/garage/values.yaml +++ b/script/helm/garage/values.yaml @@ -124,6 +124,8 @@ service: # - NodePort (+ Ingress) # - LoadBalancer type: ClusterIP + # -- Annotations to add to the service + annotations: {} s3: api: port: 3900 From 424d4f8d4d6cfd2dff5ab027b82fda9a233e2b31 Mon Sep 17 00:00:00 2001 From: nmstoker Date: Sat, 20 Dec 2025 13:16:38 +0000 Subject: [PATCH 19/19] Update doc/book/cookbook/binary-packages.md Correct the Arch Linux link as garage is now available in the official repos under extra, and no longer in AUR. --- doc/book/cookbook/binary-packages.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/book/cookbook/binary-packages.md b/doc/book/cookbook/binary-packages.md index 6476ff51..ce6beb7b 100644 --- a/doc/book/cookbook/binary-packages.md +++ b/doc/book/cookbook/binary-packages.md @@ -27,7 +27,7 @@ it's stable). ## Arch Linux -Garage is available in the [AUR](https://aur.archlinux.org/packages/garage). +Garage is available in the official repositories under [extra](https://archlinux.org/packages/extra/x86_64/garage). ## FreeBSD