diff --git a/doc/api/garage-admin-v2.json b/doc/api/garage-admin-v2.json index 3b5e66a3..c63feec4 100644 --- a/doc/api/garage-admin-v2.json +++ b/doc/api/garage-admin-v2.json @@ -2607,6 +2607,12 @@ "freeform" ], "properties": { + "bucketCount": { + "type": "integer", + "format": "int64", + "description": "number of buckets in the cluster", + "minimum": 0 + }, "dataAvail": { "type": "integer", "format": "int64", @@ -2617,7 +2623,7 @@ "type": "string", "description": "cluster statistics as a free-form string, kept for compatibility with nodes\nrunning older v2.x versions of garage" }, - "incompleteInfo": { + "incompleteAvailInfo": { "type": "boolean", "description": "true if the available storage space statistics are imprecise due to missing\ninformation of disconnected nodes. When this is the case, the actual\nspace available in the cluster might be lower than the reported values." }, @@ -2626,6 +2632,18 @@ "format": "int64", "description": "available storage space for object metadata in the entire cluster, in bytes", "minimum": 0 + }, + "totalObjectBytes": { + "type": "integer", + "format": "int64", + "description": "total size of objects stored in all buckets, before compression, deduplication and\nreplication (this is NOT equivalent to actual disk usage in the cluster)", + "minimum": 0 + }, + "totalObjectCount": { + "type": "integer", + "format": "int64", + "description": "total number of objects stored in all buckets", + "minimum": 0 } } }, diff --git a/src/api/admin/api.rs b/src/api/admin/api.rs index c0630e21..53bb2f0b 100644 --- a/src/api/admin/api.rs +++ b/src/api/admin/api.rs @@ -299,7 +299,17 @@ pub struct GetClusterStatisticsResponse { /// information of disconnected nodes. When this is the case, the actual /// space available in the cluster might be lower than the reported values. #[serde(default)] - pub incomplete_info: bool, + pub incomplete_avail_info: bool, + /// number of buckets in the cluster + #[serde(default)] + pub bucket_count: u64, + /// total number of objects stored in all buckets + #[serde(default)] + pub total_object_count: u64, + /// total size of objects stored in all buckets, before compression, deduplication and + /// replication (this is NOT equivalent to actual disk usage in the cluster) + #[serde(default)] + pub total_object_bytes: u64, } // ---- ConnectClusterNodes ---- diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 93766c7b..e0b7b3ad 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -38,7 +38,7 @@ impl RequestHandler for ListBucketsRequest { &EmptyKey, None, Some(DeletedFilter::NotDeleted), - 10000, + 1_000_000, EnumerationOrder::Forward, ) .await?; diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 96efa7f3..eee657e3 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -8,8 +8,10 @@ use garage_util::data::*; use garage_rpc::layout; use garage_rpc::layout::PARTITION_BITS; +use garage_table::*; use garage_model::garage::Garage; +use garage_model::s3::object_table; use crate::api::*; use crate::error::*; @@ -152,7 +154,6 @@ impl RequestHandler for GetClusterHealthRequest { impl RequestHandler for GetClusterStatisticsRequest { type Response = GetClusterStatisticsResponse; - // FIXME: return this as a JSON struct instead of text async fn handle( self, garage: &Arc, @@ -160,8 +161,45 @@ impl RequestHandler for GetClusterStatisticsRequest { ) -> Result { let mut ret = String::new(); - // Gather storage node and free space statistics for current nodes + // Gather info on number of buckets, objects and object size + let buckets = garage + .bucket_table + .get_range( + &EmptyKey, + None, + Some(DeletedFilter::NotDeleted), + 1_000_000, + EnumerationOrder::Forward, + ) + .await?; + + let bucket_stats = futures::future::try_join_all( + buckets + .iter() + .map(|b| garage.object_counter_table.table.get(&b.id, &EmptyKey)), + ) + .await?; + let layout = &garage.system.cluster_layout(); + + let bucket_stats = bucket_stats + .into_iter() + .filter_map(|cnt| cnt.map(|x| x.filtered_values(layout))) + .collect::>(); + + let bucket_count = buckets.len() as u64; + let total_object_count = bucket_stats + .iter() + .clone() + .map(|cnt| *cnt.get(object_table::OBJECTS).unwrap_or(&0) as u64) + .sum(); + let total_object_bytes = bucket_stats + .iter() + .clone() + .map(|cnt| *cnt.get(object_table::BYTES).unwrap_or(&0) as u64) + .sum(); + + // Gather storage node and free space statistics for current nodes let mut node_partition_count = HashMap::::new(); if let Ok(current_layout) = layout.current() { for short_id in current_layout.ring_assignment_data.iter() { @@ -242,9 +280,20 @@ impl RequestHandler for GetClusterStatisticsRequest { let incomplete_info = meta_part_avail.len() < node_partition_count.len() || data_part_avail.len() < node_partition_count.len(); + // Display bucket statistics + let bucket_stats = vec![ + format!("Number of buckets:\t{}", bucket_count), + format!("Total number of objects:\t{}", total_object_count), + format!( + "Total size of objects:\t{}", + bytesize::ByteSize(total_object_bytes) + ), + ]; + writeln!(&mut ret, "\n{}", format_table_to_string(bucket_stats)).unwrap(); + writeln!( &mut ret, - "\nEstimated available storage space cluster-wide (might be lower in practice):" + "Estimated available storage space cluster-wide (might be lower in practice):" ) .unwrap(); if incomplete_info { @@ -264,7 +313,10 @@ impl RequestHandler for GetClusterStatisticsRequest { freeform: ret, metadata_avail, data_avail, - incomplete_info, + incomplete_avail_info: incomplete_info, + bucket_count, + total_object_count, + total_object_bytes, }) } }