diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket/list.rb b/google-cloud-storage/lib/google/cloud/storage/bucket/list.rb index 5ed6cf39271b..2378ed0034a9 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket/list.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket/list.rb @@ -27,6 +27,14 @@ class List < DelegateClass(::Array) # that match the request and this value should be passed to # the next {Google::Cloud::Storage::Project#buckets} to continue. attr_accessor :token + ## + # Provides a list of bucket names that are unreachable. + # + # This is only populated when `return_partial_success` is set to `true` + # in the call to {Google::Cloud::Storage::Project#buckets}. + # + # @return [Array] + attr_reader :unreachable ## # @private Create a new Bucket::List with an array of values. @@ -147,7 +155,7 @@ def all request_limit: nil, &block # @private New Bucket::List from a Google API Client # Google::Apis::StorageV1::Buckets object. def self.from_gapi gapi_list, service, prefix = nil, max = nil, - user_project: nil, soft_deleted: nil + user_project: nil, soft_deleted: nil, return_partial_success: nil buckets = new(Array(gapi_list.items).map do |gapi_object| Bucket.from_gapi gapi_object, service, user_project: user_project end) @@ -157,6 +165,7 @@ def self.from_gapi gapi_list, service, prefix = nil, max = nil, buckets.instance_variable_set :@max, max buckets.instance_variable_set :@user_project, user_project buckets.instance_variable_set :@soft_deleted, soft_deleted + buckets.instance_variable_set :@unreachable, Array(gapi_list.unreachable) if return_partial_success buckets end diff --git a/google-cloud-storage/lib/google/cloud/storage/project.rb b/google-cloud-storage/lib/google/cloud/storage/project.rb index 74c915acf5a8..47e4ffc32da4 100644 --- a/google-cloud-storage/lib/google/cloud/storage/project.rb +++ b/google-cloud-storage/lib/google/cloud/storage/project.rb @@ -158,6 +158,9 @@ def add_custom_header header_name, header_value # bucket instances and their files. # # See also {Bucket#requester_pays=} and {Bucket#requester_pays}. + # @param [Boolean] return_partial_success + # If true, the response will contain a list of unreachable buckets. + # If false, ListBuckets will throw an error if there are any unreachable buckets. # # @return [Array] (See # {Google::Cloud::Storage::Bucket::List}) @@ -201,11 +204,22 @@ def add_custom_header header_name, header_value # soft_deleted_buckets.each do |bucket| # puts bucket.name # end - def buckets prefix: nil, token: nil, max: nil, user_project: nil, soft_deleted: nil + # @example Retrieve list of unreachable buckets + # require "google/cloud/storage" + # + # storage = Google::Cloud::Storage.new + # + # buckets = storage.buckets return_partial_success: true + # buckets.unreachable.each do |unreachable_bucket_name| + # puts unreachable_bucket_name + # end + # + def buckets prefix: nil, token: nil, max: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil gapi = service.list_buckets \ - prefix: prefix, token: token, max: max, user_project: user_project, soft_deleted: soft_deleted + prefix: prefix, token: token, max: max, user_project: user_project, soft_deleted: soft_deleted, return_partial_success: return_partial_success Bucket::List.from_gapi \ - gapi, service, prefix, max, user_project: user_project, soft_deleted: soft_deleted + gapi, service, prefix, max, user_project: user_project, soft_deleted: soft_deleted, return_partial_success: return_partial_success + end alias find_buckets buckets diff --git a/google-cloud-storage/lib/google/cloud/storage/service.rb b/google-cloud-storage/lib/google/cloud/storage/service.rb index d9adf2acff65..df61e682f8e0 100644 --- a/google-cloud-storage/lib/google/cloud/storage/service.rb +++ b/google-cloud-storage/lib/google/cloud/storage/service.rb @@ -96,12 +96,14 @@ def project_service_account ## # Retrieves a list of buckets for the given project. - def list_buckets prefix: nil, token: nil, max: nil, user_project: nil, soft_deleted: nil, options: {} + def list_buckets prefix: nil, token: nil, max: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} execute do service.list_buckets \ @project, prefix: prefix, page_token: token, max_results: max, user_project: user_project(user_project), - soft_deleted: soft_deleted, options: options + soft_deleted: soft_deleted, + return_partial_success: return_partial_success, + options: options end end diff --git a/google-cloud-storage/samples/acceptance/buckets_test.rb b/google-cloud-storage/samples/acceptance/buckets_test.rb index 7afc71d14415..bb78ba9bbe44 100644 --- a/google-cloud-storage/samples/acceptance/buckets_test.rb +++ b/google-cloud-storage/samples/acceptance/buckets_test.rb @@ -42,6 +42,7 @@ require_relative "../storage_get_retention_policy" require_relative "../storage_get_uniform_bucket_level_access" require_relative "../storage_list_buckets" +require_relative "../storage_list_buckets_with_partial_success" require_relative "../storage_lock_retention_policy" require_relative "../storage_remove_bucket_label" require_relative "../storage_remove_cors_configuration" @@ -616,4 +617,22 @@ assert_equal "invalid: Source and destination object names must be different.", exception.message end end + + describe "list buckets with partial success" do + it 'returns a list of bucket names if return_partial_success_flag is true' do + result = list_buckets_with_partial_success return_partial_success_flag: true + assert_kind_of Array, result + assert result.all? { |n| n.is_a? String }, "expected all items to be String" + end + + it 'returns nil for unreachable if return_partial_success_flag is false' do + result = list_buckets_with_partial_success return_partial_success_flag: false + assert_nil result + end + + it 'returns nil for unreachable if return_partial_success_flag is not passed' do + result = list_buckets_with_partial_success return_partial_success_flag: nil + assert_nil result + end + end end diff --git a/google-cloud-storage/samples/storage_list_buckets_with_partial_success.rb b/google-cloud-storage/samples/storage_list_buckets_with_partial_success.rb new file mode 100644 index 000000000000..c7f23469f54e --- /dev/null +++ b/google-cloud-storage/samples/storage_list_buckets_with_partial_success.rb @@ -0,0 +1,47 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_list_buckets_partial_success] +# Demonstrates listing Google Cloud Storage buckets with support for partial success. +# +# This method initializes a Google Cloud Storage client and requests a list of buckets. +# When `return_partial_success` is true, the API will return available buckets +# and a list of any buckets that were unreachable. +# +# @param return_partial_success_flag [Boolean] Whether to allow partial success from the API. +# - true: returns the available buckets and populates `unreachable` with bucket names if any. +# - false: throws an error if any buckets are unreachable. +def list_buckets_with_partial_success return_partial_success_flag: + require "google/cloud/storage" + + storage = Google::Cloud::Storage.new + bucket_list = storage.buckets return_partial_success: return_partial_success_flag + + puts "Reachable buckets:" + # limiting the bucket count to be printed to 10 for brevity + bucket_list.take(10).each do |bucket| + puts bucket.name + end + + if bucket_list.unreachable + puts "\nUnreachable buckets:" + # limiting the bucket count to be printed to 10 for brevity + bucket_list.unreachable.take(10).each do |unreachable_bucket_name| + puts unreachable_bucket_name + end + end +end +# [END storage_list_buckets_partial_success] + +list_buckets_with_partial_success return_partial_success_flag: ARGV.shift if $PROGRAM_NAME == __FILE__ diff --git a/google-cloud-storage/test/google/cloud/storage/project_test.rb b/google-cloud-storage/test/google/cloud/storage/project_test.rb index b7e2fd7c0269..05724226ca34 100644 --- a/google-cloud-storage/test/google/cloud/storage/project_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/project_test.rb @@ -615,7 +615,7 @@ def stub.insert_bucket *args num_buckets = 3 mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(num_buckets), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(num_buckets), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -633,7 +633,7 @@ def stub.insert_bucket *args num_buckets = 3 mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(num_buckets), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(num_buckets), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil,return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -650,7 +650,8 @@ def stub.insert_bucket *args mock = Minitest::Mock.new mock.expect :list_buckets, list_buckets_gapi(num_buckets,"next_page_token",soft_deleted), [project], prefix: nil, page_token: nil, -max_results: nil, user_project: nil, soft_deleted: true, options: {} + max_results: nil, user_project: nil, soft_deleted: true, + return_partial_success: nil, options: {} storage.service.mocked_service = mock buckets = storage.buckets soft_deleted: true @@ -667,8 +668,8 @@ def stub.insert_bucket *args it "paginates buckets" do mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, options: {} - mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: nil, soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -687,8 +688,8 @@ def stub.insert_bucket *args it "paginates buckets with max set" do mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: 3, user_project: nil, soft_deleted: nil, options: {} - mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: 3, user_project: nil, soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: 3, user_project: nil, soft_deleted: nil,return_partial_success: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: 3, user_project: nil, soft_deleted: nil,return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -709,7 +710,7 @@ def stub.insert_bucket *args num_buckets = 3 mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -726,8 +727,8 @@ def stub.insert_bucket *args it "paginates buckets with next? and next" do mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, options: {} - mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: nil, soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -745,8 +746,8 @@ def stub.insert_bucket *args it "paginates buckets with next? and next and max set" do mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: 3, user_project: nil, soft_deleted: nil, options: {} - mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: 3, user_project: nil, soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: 3, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: 3, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -764,8 +765,8 @@ def stub.insert_bucket *args it "paginates buckets with all" do mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, options: {} - mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: nil, soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -778,8 +779,8 @@ def stub.insert_bucket *args it "paginates buckets with all and max set" do mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: 3, user_project: nil, soft_deleted: nil, options: {} - mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: 3, user_project: nil, soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: 3, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: "next_page_token", max_results: 3, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -792,8 +793,8 @@ def stub.insert_bucket *args it "iterates buckets with all using Enumerator" do mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, options: {} - mock.expect :list_buckets, list_buckets_gapi(3, "second_page_token"), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: nil, soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "second_page_token"), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -806,8 +807,8 @@ def stub.insert_bucket *args it "iterates buckets with all and request_limit set" do mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, options: {} - mock.expect :list_buckets, list_buckets_gapi(3, "second_page_token"), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: nil,soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "second_page_token"), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: nil,soft_deleted: nil, return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -820,8 +821,8 @@ def stub.insert_bucket *args it "iterates buckets with all and user_project set to true" do mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: "test", soft_deleted: nil, options: {} - mock.expect :list_buckets, list_buckets_gapi(3, "second_page_token"), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: "test", soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: "test", soft_deleted: nil, return_partial_success: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "second_page_token"), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: "test", soft_deleted: nil, return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -835,8 +836,8 @@ def stub.insert_bucket *args it "iterates buckets with all and user_project set to another project ID" do mock = Minitest::Mock.new - mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: "my-other-project", soft_deleted: nil, options: {} - mock.expect :list_buckets, list_buckets_gapi(3, "second_page_token"), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: "my-other-project", soft_deleted: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "next_page_token"), [project], prefix: nil, page_token: nil, max_results: nil, user_project: "my-other-project", soft_deleted: nil, return_partial_success: nil, options: {} + mock.expect :list_buckets, list_buckets_gapi(3, "second_page_token"), [project], prefix: nil, page_token: "next_page_token", max_results: nil, user_project: "my-other-project", soft_deleted: nil, return_partial_success: nil, options: {} storage.service.mocked_service = mock @@ -1041,6 +1042,54 @@ def stub.insert_bucket *args _(bucket).must_be :lazy? end + it "Lists the unreachable buckets if return_partial_success is true" do + unreachable_buckets = ["projects/_/buckets/bucket1", + "projects/_/buckets/bucket2", + "projects/_/buckets/bucket3"] + + mock = Minitest::Mock.new + + mock.expect :list_buckets, list_unreachable_buckets_gapi(2,nil,nil,unreachable_buckets), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil,return_partial_success: true, options: {} + + storage.service.mocked_service = mock + + buckets = storage.buckets(return_partial_success: true) + + mock.verify + + _(buckets.unreachable).must_equal unreachable_buckets + end + + it "returns empty array for unreachable buckets if return_partial_success is true and unreachable bucket list is empty" do + unreachable_buckets = [] + + mock = Minitest::Mock.new + + mock.expect :list_buckets, list_unreachable_buckets_gapi(2,nil,nil,unreachable_buckets), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil,return_partial_success: true, options: {} + + storage.service.mocked_service = mock + + buckets = storage.buckets(return_partial_success: true) + + mock.verify + + _(buckets.unreachable).must_equal unreachable_buckets + end + + it "returns nil array for unreachable buckets if return_partial_success is passed as false" do + + mock = Minitest::Mock.new + mock.expect :list_buckets, list_buckets_gapi(2), [project], prefix: nil, page_token: nil, max_results: nil, user_project: nil, soft_deleted: nil, return_partial_success: false, options: {} + + storage.service.mocked_service = mock + + buckets = storage.buckets(return_partial_success: false) + + mock.verify + + _(buckets.unreachable).must_be :nil? + end + def bucket_with_location created_bucket, location_type: bucket_location_type resp_bucket = created_bucket.dup @@ -1081,6 +1130,14 @@ def list_buckets_gapi count = 2, token = nil, soft_deleted = nil ) end + def list_unreachable_buckets_gapi count = 2, token = nil, soft_deleted = nil, unreachable = [] + buckets = count.times.map { Google::Apis::StorageV1::Bucket.from_json random_bucket_hash(soft_deleted: soft_deleted).to_json } + buckets_obj = Google::Apis::StorageV1::Buckets.new( + kind: "storage#buckets", items: buckets, next_page_token: token, unreachable: unreachable + ) + buckets_obj + end + def restored_bucket_gapi name Google::Apis::StorageV1::Bucket.from_json random_bucket_hash(name: name).to_json end