From 93c4e538c1fe0a0c2e45c0ff79ea61fb44cea767 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Fri, 7 Mar 2025 08:54:15 +0000 Subject: [PATCH 01/14] initial sample changes --- .../samples/acceptance/files_test.rb | 1157 +++++++++-------- .../storage_initiate_resumable_upload.rb | 38 + .../storage_restart_resumable_upload.rb | 45 + 3 files changed, 680 insertions(+), 560 deletions(-) create mode 100644 google-cloud-storage/samples/storage_initiate_resumable_upload.rb create mode 100644 google-cloud-storage/samples/storage_restart_resumable_upload.rb diff --git a/google-cloud-storage/samples/acceptance/files_test.rb b/google-cloud-storage/samples/acceptance/files_test.rb index 55fbf5200c67..67208d9c4a4b 100644 --- a/google-cloud-storage/samples/acceptance/files_test.rb +++ b/google-cloud-storage/samples/acceptance/files_test.rb @@ -31,6 +31,7 @@ require_relative "../storage_generate_signed_url_v4" require_relative "../storage_generate_upload_signed_url_v4" require_relative "../storage_get_metadata" +require_relative "../storage_initiate_resumable_upload" require_relative "../storage_list_files" require_relative "../storage_list_files_with_prefix" require_relative "../storage_list_file_archived_generations" @@ -49,6 +50,7 @@ require_relative "../storage_upload_file" require_relative "../storage_upload_from_memory" require_relative "../storage_upload_with_kms_key" +require 'pry' describe "Files Snippets" do let(:storage_client) { Google::Cloud::Storage.new } @@ -78,583 +80,618 @@ bucket.files.each(&:delete) end - it "list_files" do - bucket.create_file local_file, file_1_name - bucket.create_file local_file, file_2_name + # it "list_files" do + # bucket.create_file local_file, file_1_name + # bucket.create_file local_file, file_2_name - out, _err = capture_io do - list_files bucket_name: bucket.name - end + # out, _err = capture_io do + # list_files bucket_name: bucket.name + # end - assert_match file_1_name, out - assert_match file_2_name, out - end + # assert_match file_1_name, out + # assert_match file_2_name, out + # end - it "list_files_with_prefix" do - ["foo/file.txt", "foo/data.txt", "bar/file.txt", "bar/data.txt"].each do |file| - bucket.create_file local_file, file - end + # it "list_files_with_prefix" do + # ["foo/file.txt", "foo/data.txt", "bar/file.txt", "bar/data.txt"].each do |file| + # bucket.create_file local_file, file + # end - out, _err = capture_io do - list_files_with_prefix bucket_name: bucket.name, prefix: "foo/" - end + # out, _err = capture_io do + # list_files_with_prefix bucket_name: bucket.name, prefix: "foo/" + # end - assert_match "foo/file.txt", out - assert_match "foo/data.txt", out - refute_match "bar/file.txt", out - refute_match "bar/data.txt", out - end + # assert_match "foo/file.txt", out + # assert_match "foo/data.txt", out + # refute_match "bar/file.txt", out + # refute_match "bar/data.txt", out + # end - it "list_file_archived_generations" do - file_1 = bucket.create_file local_file, file_1_name - file_2 = bucket.create_file local_file, file_2_name + # it "list_file_archived_generations" do + # file_1 = bucket.create_file local_file, file_1_name + # file_2 = bucket.create_file local_file, file_2_name - out, _err = capture_io do - list_file_archived_generations bucket_name: bucket.name - end + # out, _err = capture_io do + # list_file_archived_generations bucket_name: bucket.name + # end - assert_match "#{file_1_name},#{file_1.generation}", out - assert_match "#{file_2_name},#{file_2.generation}", out - end + # assert_match "#{file_1_name},#{file_1.generation}", out + # assert_match "#{file_2_name},#{file_2.generation}", out + # end - it "generate_encryption_key" do - mock_cipher = Minitest::Mock.new + # it "generate_encryption_key" do + # mock_cipher = Minitest::Mock.new - def mock_cipher.encrypt - self - end + # def mock_cipher.encrypt + # self + # end - def mock_cipher.random_key - @random_key ||= OpenSSL::Cipher.new("aes-256-cfb").encrypt.random_key - end + # def mock_cipher.random_key + # @random_key ||= OpenSSL::Cipher.new("aes-256-cfb").encrypt.random_key + # end - encryption_key_base64 = Base64.encode64 mock_cipher.random_key + # encryption_key_base64 = Base64.encode64 mock_cipher.random_key - OpenSSL::Cipher.stub :new, mock_cipher do - assert_output "Sample encryption key: #{encryption_key_base64}" do - generate_encryption_key - end - end - end - - it "upload_file" do - assert_output "Uploaded #{local_file} as #{remote_file_name} in bucket #{bucket.name}\n" do - upload_file bucket_name: bucket.name, local_file_path: local_file, file_name: remote_file_name - end - - assert_equal bucket.files.first.name, remote_file_name - end - - it "upload_file_from_memory" do - assert_output "Uploaded file #{remote_file_name} to bucket #{bucket.name} with content: #{file_content}\n" do - upload_file_from_memory bucket_name: bucket.name, - file_name: remote_file_name, - file_content: file_content - end - end - - it "upload_encrypted_file" do - assert_output "Uploaded #{remote_file_name} with encryption key\n" do - upload_encrypted_file bucket_name: bucket.name, - local_file_path: local_file, - file_name: remote_file_name, - encryption_key: encryption_key - end + # OpenSSL::Cipher.stub :new, mock_cipher do + # assert_output "Sample encryption key: #{encryption_key_base64}" do + # generate_encryption_key + # end + # end + # end - assert_equal bucket.files.first.name, remote_file_name - refute_nil bucket.files.first.encryption_key_sha256 - end + it "starts resumable upload" do + file_content = "A" * (3 * 1024 * 1024) # 3 MB of 'A' characters + file = StringIO.new file_content + chunk_size = 256 * 1024 * 3 - it "upload_with_kms_key" do - assert_output(/Uploaded #{remote_file_name} and encrypted service side using #{kms_key}/) do - upload_with_kms_key bucket_name: bucket.name, - local_file_path: local_file, - file_name: remote_file_name, - kms_key: kms_key + assert_output "Resumable upload completed for #{remote_file_name}\n" do + initiate_resumable_upload bucket_name: bucket.name, file: file, file_name: remote_file_name, chunk_size: chunk_size end - assert_equal bucket.files.first.name, remote_file_name - assert_match kms_key, bucket.files.first.kms_key - end - - it "download_file" do - bucket.create_file local_file, remote_file_name - - Tempfile.open [downloaded_file] do |tmpfile| - tmpfile.binmode - - assert_output "Downloaded #{remote_file_name} to #{tmpfile}\n" do - download_file bucket_name: bucket.name, - file_name: remote_file_name, - local_file_path: tmpfile - end - - assert File.file? tmpfile - end - end - - it "download_file_into_memory" do - bucket.create_file StringIO.new(file_content), remote_file_name - - assert_output "Contents of storage object #{remote_file_name} in bucket #{bucket.name} are: #{file_content}\n" do - download_file_into_memory bucket_name: bucket.name, - file_name: remote_file_name - end - end - - it "download_public_file" do - bucket.create_file local_file, remote_file_name - - Tempfile.open [downloaded_file] do |tmpfile| - tmpfile.binmode - - assert_output "Downloaded #{remote_file_name} to #{tmpfile}\n" do - download_file bucket_name: bucket.name, - file_name: remote_file_name, - local_file_path: tmpfile - end - - assert File.file? tmpfile - end - end - - it "download_file_requester_pays" do - bucket.requester_pays = true - bucket.create_file local_file, remote_file_name - - Tempfile.open [downloaded_file] do |tmpfile| - tmpfile.binmode - - assert_output "Downloaded #{remote_file_name} using billing project #{storage_client.project}\n" do - download_file_requester_pays bucket_name: bucket.name, - file_name: remote_file_name, - local_file_path: tmpfile - end - - assert File.file? tmpfile - end - end - - it "download_encrypted_file" do - bucket.create_file local_file, remote_file_name, encryption_key: encryption_key - - Tempfile.open [downloaded_file] do |tmpfile| - tmpfile.binmode - - assert_output "Downloaded encrypted #{remote_file_name} to #{tmpfile}\n" do - download_encrypted_file bucket_name: bucket.name, - file_name: remote_file_name, - local_file_path: tmpfile, - encryption_key: encryption_key - end - - assert File.file? tmpfile - assert_equal File.read(local_file), File.read(tmpfile) - end - end - - it "delete_file" do - bucket.create_file local_file, remote_file_name - - assert_output "Deleted #{remote_file_name}\n" do - delete_file bucket_name: bucket.name, file_name: remote_file_name - end - - assert_nil bucket.file remote_file_name - end - - it "delete_file_archived_generation" do - file = bucket.create_file local_file, remote_file_name - - assert_output "Generation #{file.generation} of file #{remote_file_name} was deleted from #{bucket.name}\n" do - delete_file_archived_generation bucket_name: bucket.name, file_name: remote_file_name, generation: file.generation - end - - assert_nil bucket.file remote_file_name - end - - it "get_metadata" do - bucket.create_file local_file, remote_file_name - - file = bucket.file remote_file_name - expected_output = <<~OUTPUT - Name: #{file.name} - Bucket: #{bucket.name} - Storage class: #{bucket.storage_class} - ID: #{file.id} - Size: #{file.size} bytes - Created: #{file.created_at} - Updated: #{file.updated_at} - Generation: #{file.generation} - Metageneration: #{file.metageneration} - Etag: #{file.etag} - Owners: #{file.acl.owners.join ','} - Crc32c: #{file.crc32c} - md5_hash: #{file.md5} - Cache-control: #{file.cache_control} - Content-type: #{file.content_type} - Content-disposition: #{file.content_disposition} - Content-encoding: #{file.content_encoding} - Content-language: #{file.content_language} - KmsKeyName: #{file.kms_key} - Event-based hold enabled?: #{file.event_based_hold?} - Temporary hold enaled?: #{file.temporary_hold?} - Retention Expiration: #{file.retention_expires_at} - Custom Time: #{file.custom_time} - Metadata: - OUTPUT - - assert_output expected_output do - get_metadata bucket_name: bucket.name, - file_name: remote_file_name - end - end - - it "set_metadata" do - bucket.create_file local_file, remote_file_name - - metadata_key = "your-metadata-key" - metadata_value = "your-metadata-value" - - assert_output "Metadata for #{remote_file_name} has been updated.\n" do - set_metadata bucket_name: bucket.name, file_name: remote_file_name - end - - assert_equal bucket.file(remote_file_name).metadata[metadata_key], metadata_value - end - - it "make_public" do - bucket.create_file local_file, remote_file_name - response = Net::HTTP.get URI(bucket.file(remote_file_name).public_url) - refute_equal File.read(local_file), response - - assert_output "#{remote_file_name} is publicly accessible at #{bucket.file(remote_file_name).public_url}\n" do - make_public bucket_name: bucket.name, - file_name: remote_file_name - end - - response = Net::HTTP.get URI(bucket.file(remote_file_name).public_url) - assert_equal File.read(local_file), response - end - - it "move_file" do - bucket.create_file local_file, remote_file_name - - new_name = "path/new_name.txt" - assert_nil bucket.file new_name - - assert_output "#{remote_file_name} has been renamed to #{new_name}\n" do - move_file bucket_name: bucket.name, - file_name: remote_file_name, - new_name: new_name - end - - assert_nil bucket.file remote_file_name - refute_nil bucket.file new_name - end - - it "compose_file" do - file_1 = bucket.create_file local_file, file_1_name - file_2 = bucket.create_file local_file, file_2_name - - expected_out = "Composed new file #{remote_file_name} in the bucket #{bucket.name} " \ - "by combining #{file_1.name} and #{file_2.name}\n" - assert_output expected_out do - compose_file bucket_name: bucket.name, - first_file_name: file_1.name, - second_file_name: file_2.name, - destination_file_name: remote_file_name - end - - refute_nil bucket.file remote_file_name - end - - it "copy_file" do - bucket.create_file local_file, remote_file_name - assert_nil secondary_bucket.file remote_file_name - - assert_output "#{remote_file_name} in #{bucket.name} copied to #{remote_file_name} in #{secondary_bucket.name}\n" do - copy_file source_bucket_name: bucket.name, - source_file_name: remote_file_name, - destination_bucket_name: secondary_bucket.name, - destination_file_name: remote_file_name - end - - refute_nil bucket.file remote_file_name - refute_nil secondary_bucket.file remote_file_name - end - - it "copy_file_archived_generation" do - file = bucket.create_file local_file, remote_file_name - assert_nil secondary_bucket.file remote_file_name - - expected_out = "Generation #{file.generation} of the file #{remote_file_name} in bucket #{bucket.name} copied " \ - "to file #{remote_file_name} in bucket #{secondary_bucket.name}\n" - assert_output expected_out do - copy_file_archived_generation source_bucket_name: bucket.name, - source_file_name: remote_file_name, - generation: file.generation, - destination_bucket_name: secondary_bucket.name, - destination_file_name: remote_file_name - end - - refute_nil bucket.file remote_file_name - refute_nil secondary_bucket.file remote_file_name - end - - it "rotate_encryption_key" do - bucket.create_file local_file, remote_file_name, encryption_key: encryption_key - - new_encryption_key = OpenSSL::Cipher.new("aes-256-cfb").encrypt.random_key - file_contents = File.read local_file - - assert_output "The encryption key for #{remote_file_name} in #{bucket.name} was rotated.\n" do - rotate_encryption_key bucket_name: bucket.name, - file_name: remote_file_name, - current_encryption_key: encryption_key, - new_encryption_key: new_encryption_key - end - - Tempfile.open [downloaded_file] do |tmpfile| - tmpfile.binmode - - bucket.file(remote_file_name).download tmpfile, encryption_key: new_encryption_key - downloaded_contents = File.read tmpfile - assert_equal file_contents, downloaded_contents - end - end - - it "object_csek_to_cmek" do - file = bucket.create_file local_file, remote_file_name, encryption_key: encryption_key - assert file.encryption_key_sha256 - expected_out = "File #{file.name} in bucket #{bucket.name} is now managed by the KMS key #{kms_key} instead of " \ - "a customer-supplied encryption key\n" - assert_output expected_out do - object_csek_to_cmek bucket_name: bucket.name, - file_name: file.name, - encryption_key: encryption_key, - kms_key_name: kms_key - end - - file = bucket.file file.name - - assert file.exists? - assert_match kms_key, file.kms_key - assert_nil file.encryption_key_sha256 end - it "generate_signed_url_v2" do - bucket.create_file local_file, remote_file_name - - out, _err = capture_io do - generate_signed_url_v2 bucket_name: bucket.name, - file_name: remote_file_name - end - - assert_match "The signed url for #{remote_file_name} is", out - signed_url = out.scan(/http.*$/).first - refute_nil signed_url - - file_contents = Net::HTTP.get URI(signed_url) - assert_equal file_contents, File.read(local_file) - end - - it "generate_signed_url_v4" do - bucket.create_file local_file, remote_file_name - - out, _err = capture_io do - generate_signed_url_v4 bucket_name: bucket.name, - file_name: remote_file_name - end - - signed_url = out.scan(/http.*$/).first - refute_nil signed_url - - file_contents = Net::HTTP.get URI(signed_url) - assert_equal file_contents, File.read(local_file) - end - - it "generate_upload_signed_url_v4" do - refute bucket.file remote_file_name - - out, _err = capture_io do - generate_upload_signed_url_v4 bucket_name: bucket.name, - file_name: remote_file_name - end - - signed_url = out.scan(/http.*$/).first - refute_nil signed_url - - uri = URI.parse signed_url - http = Net::HTTP.new uri.host - request = Net::HTTP::Put.new uri.request_uri - request.body = File.read local_file - request["Content-Type"] = "text/plain" - request["Content-Length"] = File.size local_file - - response = http.request request - assert_equal response.code, "200" - - assert bucket.file remote_file_name - end - describe "post object" do - require "net/http" - require "uri" - let(:uri) { URI.parse Google::Cloud::Storage::GOOGLEAPIS_URL } - let(:data) { File.expand_path "../../acceptance/data/logo.jpg", __dir__ } - - it "generate_signed_post_policy_v4" do - refute bucket.file remote_file_name - post_object = nil - out, _err = capture_io do - post_object = generate_signed_post_policy_v4 bucket_name: bucket.name, - file_name: remote_file_name - end - - assert_includes out, "
" - assert_includes out, "" - - assert post_object - expected_keys = [ - "key", - "policy", - "x-goog-algorithm", - "x-goog-credential", - "x-goog-date", - "x-goog-meta-test", - "x-goog-signature" - ] - assert_equal expected_keys, post_object.fields.keys.sort - - # For some weird (as yet unidentified) reason, keeping file as the first value - # makes the http request fail intermittently with a 400 error. - # Moving file as the last entry in the form_data array works fine. - form_data = post_object.fields.map do |key, value| - [key, value] - end - form_data.push ["file", File.open(data)] - - http = Net::HTTP.new uri.host, uri.port - http.use_ssl = true - request = Net::HTTP::Post.new post_object.url - request.set_form form_data, "multipart/form-data" - - response = http.request request - - _(response.code).must_equal "204" - file = bucket.file post_object.fields["key"] - _(file).wont_be :nil? - Tempfile.open ["google-cloud-logo", ".jpg"] do |tmpfile| - tmpfile.binmode - downloaded = file.download tmpfile - _(File.read(downloaded.path, mode: "rb")).must_equal File.read(data, mode: "rb") - end - end - end - - it "set_event_based_hold" do - bucket.create_file local_file, remote_file_name - - assert_output "Event-based hold was set for #{remote_file_name}.\n" do - set_event_based_hold bucket_name: bucket.name, - file_name: remote_file_name - end - - assert bucket.file(remote_file_name).event_based_hold? - bucket.file(remote_file_name).release_event_based_hold! - end - - it "release_event_based_hold" do - bucket.create_file local_file, remote_file_name - bucket.file(remote_file_name).set_event_based_hold! - assert bucket.file(remote_file_name).event_based_hold? - - assert_output "Event-based hold was released for #{remote_file_name}.\n" do - release_event_based_hold bucket_name: bucket.name, - file_name: remote_file_name - end - - refute bucket.file(remote_file_name).event_based_hold? - end - - it "set_temporary_hold" do - bucket.create_file local_file, remote_file_name - refute bucket.file(remote_file_name).temporary_hold? - - assert_output "Temporary hold was set for #{remote_file_name}.\n" do - set_temporary_hold bucket_name: bucket.name, - file_name: remote_file_name - end - - assert bucket.file(remote_file_name).temporary_hold? - bucket.file(remote_file_name).release_temporary_hold! - end - - it "release_temporary_hold" do - bucket.create_file local_file, remote_file_name - bucket.file(remote_file_name).set_temporary_hold! - assert bucket.file(remote_file_name).temporary_hold? - - assert_output "Temporary hold was released for #{remote_file_name}.\n" do - release_temporary_hold bucket_name: bucket.name, file_name: remote_file_name - end - - refute bucket.file(remote_file_name).temporary_hold? - end - - it "change_file_storage_class" do - bucket.create_file local_file, remote_file_name - assert_equal "STANDARD", bucket.file(remote_file_name).storage_class - - assert_output "File #{remote_file_name} in bucket #{bucket.name} had its storage class set to NEARLINE\n" do - change_file_storage_class bucket_name: bucket.name, file_name: remote_file_name - end - - assert_equal "NEARLINE", bucket.file(remote_file_name).storage_class - end - - it "storage_download_byte_range" do - bucket.create_file local_file, remote_file_name - - Tempfile.open [downloaded_file] do |tmpfile| - tmpfile.binmode - - assert_output "Downloaded bytes 0 to 3 of object #{remote_file_name} from bucket #{bucket.name}" \ - + " to local file #{tmpfile}.\n" do - StorageDownloadByteRange.new.storage_download_byte_range bucket_name: bucket.name, - file_name: remote_file_name, - start_byte: 0, - end_byte: 3, - local_file_path: tmpfile - end - - assert File.file? tmpfile - end - end - - it "storage_stream_file_upload" do - file_obj = StringIO.new file_content - assert_output "Stream data uploaded to #{remote_file_name} in bucket #{bucket.name}\n" do - StorageStreamFileUpload.new.storage_stream_file_upload bucket_name: bucket.name, - local_file_obj: file_obj, - file_name: remote_file_name - end - end - - it "storage_stream_file_download" do - bucket.create_file StringIO.new(file_content), remote_file_name - assert_output "The full downloaded file contents are: \"#{file_content}\"\n" do - StorageStreamFileDownload.new.storage_stream_file_download bucket_name: bucket.name, - file_name: remote_file_name, - local_file_obj: StringIO.new - end - end + # it "Delete resumable upload" do + # file_content = "A" * (3 * 1024 * 1024) # 3 MB of 'A' characters + # file = StringIO.new file_content + # chunk_size = 256 * 1024 * 3 + + # assert_output "Resumable upload completed for #{remote_file_name}\n" do + # initiate_resumable_upload bucket_name: bucket.name, file: file, file_name: remote_file_name, chunk_size: chunk_size + # end + + # assert_not_equal bucket.files.first.name, remote_file_name + # end + + # it "restarts a resumable upload" do + # file_content = "A" * (3 * 1024 * 1024) # 3 MB of 'A' characters + # file = StringIO.new file_content + # chunk_size = 256 * 1024 * 3 + # initiate_resumable_upload bucket_name: bucket.name, file: file, file_name: remote_file_name, chunk_size: chunk_size + # # assert_output "Resumable upload completed for #{remote_file_name}\n" do + # # initiate_resumable_upload bucket_name: bucket.name, file: file, file_name: remote_file_name, chunk_size: chunk_size + # # end + # assert_equal bucket.files.first.name, remote_file_name + # end + + + # it "upload_file" do + # assert_output "Uploaded #{local_file} as #{remote_file_name} in bucket #{bucket.name}\n" do + # upload_file bucket_name: bucket.name, local_file_path: local_file, file_name: remote_file_name + # end + + # assert_equal bucket.files.first.name, remote_file_name + # end + + # it "upload_file_from_memory" do + # assert_output "Uploaded file #{remote_file_name} to bucket #{bucket.name} with content: #{file_content}\n" do + # upload_file_from_memory bucket_name: bucket.name, + # file_name: remote_file_name, + # file_content: file_content + # end + # end + + # it "upload_encrypted_file" do + # assert_output "Uploaded #{remote_file_name} with encryption key\n" do + # upload_encrypted_file bucket_name: bucket.name, + # local_file_path: local_file, + # file_name: remote_file_name, + # encryption_key: encryption_key + # end + + # assert_equal bucket.files.first.name, remote_file_name + # refute_nil bucket.files.first.encryption_key_sha256 + # end + + # it "upload_with_kms_key" do + # assert_output(/Uploaded #{remote_file_name} and encrypted service side using #{kms_key}/) do + # upload_with_kms_key bucket_name: bucket.name, + # local_file_path: local_file, + # file_name: remote_file_name, + # kms_key: kms_key + # end + + # assert_equal bucket.files.first.name, remote_file_name + # assert_match kms_key, bucket.files.first.kms_key + # end + + # it "download_file" do + # bucket.create_file local_file, remote_file_name + + # Tempfile.open [downloaded_file] do |tmpfile| + # tmpfile.binmode + + # assert_output "Downloaded #{remote_file_name} to #{tmpfile}\n" do + # download_file bucket_name: bucket.name, + # file_name: remote_file_name, + # local_file_path: tmpfile + # end + + # assert File.file? tmpfile + # end + # end + + # it "download_file_into_memory" do + # bucket.create_file StringIO.new(file_content), remote_file_name + + # assert_output "Contents of storage object #{remote_file_name} in bucket #{bucket.name} are: #{file_content}\n" do + # download_file_into_memory bucket_name: bucket.name, + # file_name: remote_file_name + # end + # end + + # it "download_public_file" do + # bucket.create_file local_file, remote_file_name + + # Tempfile.open [downloaded_file] do |tmpfile| + # tmpfile.binmode + + # assert_output "Downloaded #{remote_file_name} to #{tmpfile}\n" do + # download_file bucket_name: bucket.name, + # file_name: remote_file_name, + # local_file_path: tmpfile + # end + + # assert File.file? tmpfile + # end + # end + + # it "download_file_requester_pays" do + # bucket.requester_pays = true + # bucket.create_file local_file, remote_file_name + + # Tempfile.open [downloaded_file] do |tmpfile| + # tmpfile.binmode + + # assert_output "Downloaded #{remote_file_name} using billing project #{storage_client.project}\n" do + # download_file_requester_pays bucket_name: bucket.name, + # file_name: remote_file_name, + # local_file_path: tmpfile + # end + + # assert File.file? tmpfile + # end + # end + + # it "download_encrypted_file" do + # bucket.create_file local_file, remote_file_name, encryption_key: encryption_key + + # Tempfile.open [downloaded_file] do |tmpfile| + # tmpfile.binmode + + # assert_output "Downloaded encrypted #{remote_file_name} to #{tmpfile}\n" do + # download_encrypted_file bucket_name: bucket.name, + # file_name: remote_file_name, + # local_file_path: tmpfile, + # encryption_key: encryption_key + # end + + # assert File.file? tmpfile + # assert_equal File.read(local_file), File.read(tmpfile) + # end + # end + + # it "delete_file" do + # bucket.create_file local_file, remote_file_name + + # assert_output "Deleted #{remote_file_name}\n" do + # delete_file bucket_name: bucket.name, file_name: remote_file_name + # end + + # assert_nil bucket.file remote_file_name + # end + + # it "delete_file_archived_generation" do + # file = bucket.create_file local_file, remote_file_name + + # assert_output "Generation #{file.generation} of file #{remote_file_name} was deleted from #{bucket.name}\n" do + # delete_file_archived_generation bucket_name: bucket.name, file_name: remote_file_name, generation: file.generation + # end + + # assert_nil bucket.file remote_file_name + # end + + # it "get_metadata" do + # bucket.create_file local_file, remote_file_name + + # file = bucket.file remote_file_name + # expected_output = <<~OUTPUT + # Name: #{file.name} + # Bucket: #{bucket.name} + # Storage class: #{bucket.storage_class} + # ID: #{file.id} + # Size: #{file.size} bytes + # Created: #{file.created_at} + # Updated: #{file.updated_at} + # Generation: #{file.generation} + # Metageneration: #{file.metageneration} + # Etag: #{file.etag} + # Owners: #{file.acl.owners.join ','} + # Crc32c: #{file.crc32c} + # md5_hash: #{file.md5} + # Cache-control: #{file.cache_control} + # Content-type: #{file.content_type} + # Content-disposition: #{file.content_disposition} + # Content-encoding: #{file.content_encoding} + # Content-language: #{file.content_language} + # KmsKeyName: #{file.kms_key} + # Event-based hold enabled?: #{file.event_based_hold?} + # Temporary hold enaled?: #{file.temporary_hold?} + # Retention Expiration: #{file.retention_expires_at} + # Custom Time: #{file.custom_time} + # Metadata: + # OUTPUT + + # assert_output expected_output do + # get_metadata bucket_name: bucket.name, + # file_name: remote_file_name + # end + # end + + # it "set_metadata" do + # bucket.create_file local_file, remote_file_name + + # metadata_key = "your-metadata-key" + # metadata_value = "your-metadata-value" + + # assert_output "Metadata for #{remote_file_name} has been updated.\n" do + # set_metadata bucket_name: bucket.name, file_name: remote_file_name + # end + + # assert_equal bucket.file(remote_file_name).metadata[metadata_key], metadata_value + # end + + # it "make_public" do + # bucket.create_file local_file, remote_file_name + # response = Net::HTTP.get URI(bucket.file(remote_file_name).public_url) + # refute_equal File.read(local_file), response + + # assert_output "#{remote_file_name} is publicly accessible at #{bucket.file(remote_file_name).public_url}\n" do + # make_public bucket_name: bucket.name, + # file_name: remote_file_name + # end + + # response = Net::HTTP.get URI(bucket.file(remote_file_name).public_url) + # assert_equal File.read(local_file), response + # end + + # it "move_file" do + # bucket.create_file local_file, remote_file_name + + # new_name = "path/new_name.txt" + # assert_nil bucket.file new_name + + # assert_output "#{remote_file_name} has been renamed to #{new_name}\n" do + # move_file bucket_name: bucket.name, + # file_name: remote_file_name, + # new_name: new_name + # end + + # assert_nil bucket.file remote_file_name + # refute_nil bucket.file new_name + # end + + # it "compose_file" do + # file_1 = bucket.create_file local_file, file_1_name + # file_2 = bucket.create_file local_file, file_2_name + + # expected_out = "Composed new file #{remote_file_name} in the bucket #{bucket.name} " \ + # "by combining #{file_1.name} and #{file_2.name}\n" + # assert_output expected_out do + # compose_file bucket_name: bucket.name, + # first_file_name: file_1.name, + # second_file_name: file_2.name, + # destination_file_name: remote_file_name + # end + + # refute_nil bucket.file remote_file_name + # end + + # it "copy_file" do + # bucket.create_file local_file, remote_file_name + # assert_nil secondary_bucket.file remote_file_name + + # assert_output "#{remote_file_name} in #{bucket.name} copied to #{remote_file_name} in #{secondary_bucket.name}\n" do + # copy_file source_bucket_name: bucket.name, + # source_file_name: remote_file_name, + # destination_bucket_name: secondary_bucket.name, + # destination_file_name: remote_file_name + # end + + # refute_nil bucket.file remote_file_name + # refute_nil secondary_bucket.file remote_file_name + # end + + # it "copy_file_archived_generation" do + # file = bucket.create_file local_file, remote_file_name + # assert_nil secondary_bucket.file remote_file_name + + # expected_out = "Generation #{file.generation} of the file #{remote_file_name} in bucket #{bucket.name} copied " \ + # "to file #{remote_file_name} in bucket #{secondary_bucket.name}\n" + # assert_output expected_out do + # copy_file_archived_generation source_bucket_name: bucket.name, + # source_file_name: remote_file_name, + # generation: file.generation, + # destination_bucket_name: secondary_bucket.name, + # destination_file_name: remote_file_name + # end + + # refute_nil bucket.file remote_file_name + # refute_nil secondary_bucket.file remote_file_name + # end + + # it "rotate_encryption_key" do + # bucket.create_file local_file, remote_file_name, encryption_key: encryption_key + + # new_encryption_key = OpenSSL::Cipher.new("aes-256-cfb").encrypt.random_key + # file_contents = File.read local_file + + # assert_output "The encryption key for #{remote_file_name} in #{bucket.name} was rotated.\n" do + # rotate_encryption_key bucket_name: bucket.name, + # file_name: remote_file_name, + # current_encryption_key: encryption_key, + # new_encryption_key: new_encryption_key + # end + + # Tempfile.open [downloaded_file] do |tmpfile| + # tmpfile.binmode + + # bucket.file(remote_file_name).download tmpfile, encryption_key: new_encryption_key + # downloaded_contents = File.read tmpfile + # assert_equal file_contents, downloaded_contents + # end + # end + + # it "object_csek_to_cmek" do + # file = bucket.create_file local_file, remote_file_name, encryption_key: encryption_key + # assert file.encryption_key_sha256 + # expected_out = "File #{file.name} in bucket #{bucket.name} is now managed by the KMS key #{kms_key} instead of " \ + # "a customer-supplied encryption key\n" + # assert_output expected_out do + # object_csek_to_cmek bucket_name: bucket.name, + # file_name: file.name, + # encryption_key: encryption_key, + # kms_key_name: kms_key + # end + + # file = bucket.file file.name + + # assert file.exists? + # assert_match kms_key, file.kms_key + # assert_nil file.encryption_key_sha256 + # end + + # it "generate_signed_url_v2" do + # bucket.create_file local_file, remote_file_name + + # out, _err = capture_io do + # generate_signed_url_v2 bucket_name: bucket.name, + # file_name: remote_file_name + # end + + # assert_match "The signed url for #{remote_file_name} is", out + # signed_url = out.scan(/http.*$/).first + # refute_nil signed_url + + # file_contents = Net::HTTP.get URI(signed_url) + # assert_equal file_contents, File.read(local_file) + # end + + # it "generate_signed_url_v4" do + # bucket.create_file local_file, remote_file_name + + # out, _err = capture_io do + # generate_signed_url_v4 bucket_name: bucket.name, + # file_name: remote_file_name + # end + + # signed_url = out.scan(/http.*$/).first + # refute_nil signed_url + + # file_contents = Net::HTTP.get URI(signed_url) + # assert_equal file_contents, File.read(local_file) + # end + + # it "generate_upload_signed_url_v4" do + # refute bucket.file remote_file_name + + # out, _err = capture_io do + # generate_upload_signed_url_v4 bucket_name: bucket.name, + # file_name: remote_file_name + # end + + # signed_url = out.scan(/http.*$/).first + # refute_nil signed_url + + # uri = URI.parse signed_url + # http = Net::HTTP.new uri.host + # request = Net::HTTP::Put.new uri.request_uri + # request.body = File.read local_file + # request["Content-Type"] = "text/plain" + # request["Content-Length"] = File.size local_file + + # response = http.request request + # assert_equal response.code, "200" + + # assert bucket.file remote_file_name + # end + # describe "post object" do + # require "net/http" + # require "uri" + # let(:uri) { URI.parse Google::Cloud::Storage::GOOGLEAPIS_URL } + # let(:data) { File.expand_path "../../acceptance/data/logo.jpg", __dir__ } + + # it "generate_signed_post_policy_v4" do + # refute bucket.file remote_file_name + # post_object = nil + # out, _err = capture_io do + # post_object = generate_signed_post_policy_v4 bucket_name: bucket.name, + # file_name: remote_file_name + # end + + # assert_includes out, "" + # assert_includes out, "" + + # assert post_object + # expected_keys = [ + # "key", + # "policy", + # "x-goog-algorithm", + # "x-goog-credential", + # "x-goog-date", + # "x-goog-meta-test", + # "x-goog-signature" + # ] + # assert_equal expected_keys, post_object.fields.keys.sort + + # # For some weird (as yet unidentified) reason, keeping file as the first value + # # makes the http request fail intermittently with a 400 error. + # # Moving file as the last entry in the form_data array works fine. + # form_data = post_object.fields.map do |key, value| + # [key, value] + # end + # form_data.push ["file", File.open(data)] + + # http = Net::HTTP.new uri.host, uri.port + # http.use_ssl = true + # request = Net::HTTP::Post.new post_object.url + # request.set_form form_data, "multipart/form-data" + + # response = http.request request + + # _(response.code).must_equal "204" + # file = bucket.file post_object.fields["key"] + # _(file).wont_be :nil? + # Tempfile.open ["google-cloud-logo", ".jpg"] do |tmpfile| + # tmpfile.binmode + # downloaded = file.download tmpfile + # _(File.read(downloaded.path, mode: "rb")).must_equal File.read(data, mode: "rb") + # end + # end + # end + + # it "set_event_based_hold" do + # bucket.create_file local_file, remote_file_name + + # assert_output "Event-based hold was set for #{remote_file_name}.\n" do + # set_event_based_hold bucket_name: bucket.name, + # file_name: remote_file_name + # end + + # assert bucket.file(remote_file_name).event_based_hold? + # bucket.file(remote_file_name).release_event_based_hold! + # end + + # it "release_event_based_hold" do + # bucket.create_file local_file, remote_file_name + # bucket.file(remote_file_name).set_event_based_hold! + # assert bucket.file(remote_file_name).event_based_hold? + + # assert_output "Event-based hold was released for #{remote_file_name}.\n" do + # release_event_based_hold bucket_name: bucket.name, + # file_name: remote_file_name + # end + + # refute bucket.file(remote_file_name).event_based_hold? + # end + + # it "set_temporary_hold" do + # bucket.create_file local_file, remote_file_name + # refute bucket.file(remote_file_name).temporary_hold? + + # assert_output "Temporary hold was set for #{remote_file_name}.\n" do + # set_temporary_hold bucket_name: bucket.name, + # file_name: remote_file_name + # end + + # assert bucket.file(remote_file_name).temporary_hold? + # bucket.file(remote_file_name).release_temporary_hold! + # end + + # it "release_temporary_hold" do + # bucket.create_file local_file, remote_file_name + # bucket.file(remote_file_name).set_temporary_hold! + # assert bucket.file(remote_file_name).temporary_hold? + + # assert_output "Temporary hold was released for #{remote_file_name}.\n" do + # release_temporary_hold bucket_name: bucket.name, file_name: remote_file_name + # end + + # refute bucket.file(remote_file_name).temporary_hold? + # end + + # it "change_file_storage_class" do + # bucket.create_file local_file, remote_file_name + # assert_equal "STANDARD", bucket.file(remote_file_name).storage_class + + # assert_output "File #{remote_file_name} in bucket #{bucket.name} had its storage class set to NEARLINE\n" do + # change_file_storage_class bucket_name: bucket.name, file_name: remote_file_name + # end + + # assert_equal "NEARLINE", bucket.file(remote_file_name).storage_class + # end + + # it "storage_download_byte_range" do + # bucket.create_file local_file, remote_file_name + + # Tempfile.open [downloaded_file] do |tmpfile| + # tmpfile.binmode + + # assert_output "Downloaded bytes 0 to 3 of object #{remote_file_name} from bucket #{bucket.name}" \ + # + " to local file #{tmpfile}.\n" do + # StorageDownloadByteRange.new.storage_download_byte_range bucket_name: bucket.name, + # file_name: remote_file_name, + # start_byte: 0, + # end_byte: 3, + # local_file_path: tmpfile + # end + + # assert File.file? tmpfile + # end + # end + + # it "storage_stream_file_upload" do + # file_obj = StringIO.new file_content + # assert_output "Stream data uploaded to #{remote_file_name} in bucket #{bucket.name}\n" do + # StorageStreamFileUpload.new.storage_stream_file_upload bucket_name: bucket.name, + # local_file_obj: file_obj, + # file_name: remote_file_name + # end + # end + + # it "storage_stream_file_download" do + # bucket.create_file StringIO.new(file_content), remote_file_name + # assert_output "The full downloaded file contents are: \"#{file_content}\"\n" do + # StorageStreamFileDownload.new.storage_stream_file_download bucket_name: bucket.name, + # file_name: remote_file_name, + # local_file_obj: StringIO.new + # end + # end end diff --git a/google-cloud-storage/samples/storage_initiate_resumable_upload.rb b/google-cloud-storage/samples/storage_initiate_resumable_upload.rb new file mode 100644 index 000000000000..b8f874435860 --- /dev/null +++ b/google-cloud-storage/samples/storage_initiate_resumable_upload.rb @@ -0,0 +1,38 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require "google/cloud/storage" + +# [START storage_initiate_resumable_upload] + +def initiate_resumable_upload bucket_name:, file:, file_name:, chunk_size: + # The ID of your GCS bucket + # bucket_name = "your-unique-bucket-name" + storage = Google::Cloud::Storage.new upload_chunk_size: chunk_size + + # storage = Google::Cloud::Storage.new(upload_chunk_size: chunk_size, upload_url: upload_url, delete_upload: true) + + bucket = storage.bucket bucket_name + my_file= bucket.create_file file, file_name + binding.pry + + puts "Resumable upload completed for #{file_name}" + + +end +# [END storage_initiate_resumable_upload] + +if $PROGRAM_NAME == __FILE__ + storage_initiate_resumable_upload bucket_name: ARGV.shift, file: ARGV.shift, file_name: ARGV.shift, + chunk_size: ARGV.shift +end diff --git a/google-cloud-storage/samples/storage_restart_resumable_upload.rb b/google-cloud-storage/samples/storage_restart_resumable_upload.rb new file mode 100644 index 000000000000..f45e3934ad78 --- /dev/null +++ b/google-cloud-storage/samples/storage_restart_resumable_upload.rb @@ -0,0 +1,45 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require "google/cloud/storage" + require_relative '../lib/google/cloud/storage/service' + require_relative '../lib/google/cloud/storage' + require "pry" + + +# [START storage_restart_resumable_upload] +def restart_resumable_upload bucket_name: + # The ID of your GCS bucket + # bucket_name = "your-unique-bucket-name" + + # Create a 3 MB file using StringIO + file_content = "A" * (3 * 1024 * 1024) # 3 MB of 'A' characters + file = StringIO.new file_content + + file_name = "my_io_3mb4.txt" + chunk_size = 256 * 1024 * 3 + + # storage = Google::Cloud::Storage.new upload_chunk_size: chunk_size + +# upload_url= "https://storage.googleapis.com/upload/storage/v1/b/ruby_7/o?name=my_io_3mb4.txt&name=my_io_3mb4.txt&uploadType=resumable&upload_id=AHMx-iED_BP-upLbxEmspBKvvtpCWIWvslGEP8RsPFChF8NEluVBNsvScs3OaVzREDrLvT3UJYTR1FSaRAMFEY5s2i4PM8Jmw2BuGR8kTYyQ4ZY" + + storage = Google::Cloud::Storage.new(upload_chunk_size: chunk_size, upload_url: upload_url) + bucket = storage.bucket bucket_name + + bucket.create_file file, file_name +end +# [END storage_restart_resumable_upload] + +# if $PROGRAM_NAME == __FILE__ + storage_restart_resumable_upload bucket_name: "ruby_7" +# end From 3d001128d4da755d89dffdeb9f9b360e06870baf Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 27 Feb 2025 05:23:56 +0000 Subject: [PATCH 02/14] adding missing config --- google-cloud-storage/lib/google-cloud-storage.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/google-cloud-storage/lib/google-cloud-storage.rb b/google-cloud-storage/lib/google-cloud-storage.rb index 1dba01af5dcc..dd0ccb660b28 100644 --- a/google-cloud-storage/lib/google-cloud-storage.rb +++ b/google-cloud-storage/lib/google-cloud-storage.rb @@ -142,7 +142,7 @@ def storage scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_time def self.storage project_id = nil, credentials = nil, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, - upload_chunk_size: nil + upload_chunk_size: nil, upload_url: nil, delete_upload: nil require "google/cloud/storage" Google::Cloud::Storage.new project_id: project_id, credentials: credentials, @@ -156,7 +156,9 @@ def self.storage project_id = nil, credentials = nil, scope: nil, base_interval: base_interval, max_interval: max_interval, multiplier: multiplier, - upload_chunk_size: upload_chunk_size + upload_chunk_size: upload_chunk_size, + upload_url: upload_url, + delete_upload: delete_upload end end end @@ -193,4 +195,6 @@ def self.storage project_id = nil, credentials = nil, scope: nil, config.add_field! :upload_chunk_size, nil, match: Integer config.add_field! :endpoint, nil, match: String, allow_nil: true config.add_field! :universe_domain, nil, match: String, allow_nil: true + config.add_field! :upload_url, nil, match: String, allow_nil: true + config.add_field! :delete_upload, nil, match: Integer, allow_nil: true end From fa4517779d006731bb2ed532d00e7ad4f683d6e6 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 27 Feb 2025 19:17:39 +0000 Subject: [PATCH 03/14] fixing unit tests --- .../test/google/cloud/storage_test.rb | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/google-cloud-storage/test/google/cloud/storage_test.rb b/google-cloud-storage/test/google/cloud/storage_test.rb index 66033eb11448..a5a9b73ce214 100644 --- a/google-cloud-storage/test/google/cloud/storage_test.rb +++ b/google-cloud-storage/test/google/cloud/storage_test.rb @@ -126,7 +126,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_be :nil? @@ -223,7 +223,7 @@ def creds.is_a? target it "uses provided endpoint and universe_domain" do stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { _(project).must_equal "project-id" _(credentials).must_equal default_credentials _(retries).must_be :nil? @@ -263,7 +263,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_be :nil? @@ -305,7 +305,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_be :nil? @@ -347,7 +347,7 @@ def creds.is_a? target OpenStruct.new project_id: "project-id" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { _(project).must_equal "project-id" _(credentials).must_be_kind_of OpenStruct _(credentials.project_id).must_equal "project-id" @@ -401,7 +401,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_be :nil? @@ -449,7 +449,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_be :nil? @@ -497,7 +497,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_equal 3 @@ -553,7 +553,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_equal 3 @@ -609,7 +609,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_equal 3 @@ -667,7 +667,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_equal 3 From f5965fe6a9300145daf8c6db0760abddb6e90283 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 27 Feb 2025 19:22:06 +0000 Subject: [PATCH 04/14] fixing lint issues --- google-cloud-storage/lib/google/cloud/storage.rb | 1 - google-cloud-storage/lib/google/cloud/storage/service.rb | 1 - 2 files changed, 2 deletions(-) diff --git a/google-cloud-storage/lib/google/cloud/storage.rb b/google-cloud-storage/lib/google/cloud/storage.rb index 38fce2374552..a2ead9271120 100644 --- a/google-cloud-storage/lib/google/cloud/storage.rb +++ b/google-cloud-storage/lib/google/cloud/storage.rb @@ -96,7 +96,6 @@ def self.new project_id: nil, credentials: nil, scope: nil, retries: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil - scope ||= configure.scope retries ||= configure.retries timeout ||= configure.timeout diff --git a/google-cloud-storage/lib/google/cloud/storage/service.rb b/google-cloud-storage/lib/google/cloud/storage/service.rb index 5e2c88588dfe..6d52b1385657 100644 --- a/google-cloud-storage/lib/google/cloud/storage/service.rb +++ b/google-cloud-storage/lib/google/cloud/storage/service.rb @@ -51,7 +51,6 @@ def initialize project, credentials, retries: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil - host ||= Google::Cloud::Storage.configure.endpoint @project = project @credentials = credentials From a48e2024b4c09f18a4d36bc89fae1b7e6a892e5e Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 10 Mar 2025 19:38:27 +0000 Subject: [PATCH 05/14] changing storage_cleint approach --- .../lib/google-cloud-storage.rb | 8 ++---- .../lib/google/cloud/storage.rb | 6 +--- .../lib/google/cloud/storage/bucket.rb | 28 +++++++++++++++++-- .../lib/google/cloud/storage/service.rb | 13 ++++++--- .../test/google/cloud/storage_test.rb | 22 +++++++-------- google-cloud-storage/test/helper.rb | 2 ++ 6 files changed, 51 insertions(+), 28 deletions(-) diff --git a/google-cloud-storage/lib/google-cloud-storage.rb b/google-cloud-storage/lib/google-cloud-storage.rb index dd0ccb660b28..1dba01af5dcc 100644 --- a/google-cloud-storage/lib/google-cloud-storage.rb +++ b/google-cloud-storage/lib/google-cloud-storage.rb @@ -142,7 +142,7 @@ def storage scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_time def self.storage project_id = nil, credentials = nil, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, multiplier: nil, - upload_chunk_size: nil, upload_url: nil, delete_upload: nil + upload_chunk_size: nil require "google/cloud/storage" Google::Cloud::Storage.new project_id: project_id, credentials: credentials, @@ -156,9 +156,7 @@ def self.storage project_id = nil, credentials = nil, scope: nil, base_interval: base_interval, max_interval: max_interval, multiplier: multiplier, - upload_chunk_size: upload_chunk_size, - upload_url: upload_url, - delete_upload: delete_upload + upload_chunk_size: upload_chunk_size end end end @@ -195,6 +193,4 @@ def self.storage project_id = nil, credentials = nil, scope: nil, config.add_field! :upload_chunk_size, nil, match: Integer config.add_field! :endpoint, nil, match: String, allow_nil: true config.add_field! :universe_domain, nil, match: String, allow_nil: true - config.add_field! :upload_url, nil, match: String, allow_nil: true - config.add_field! :delete_upload, nil, match: Integer, allow_nil: true end diff --git a/google-cloud-storage/lib/google/cloud/storage.rb b/google-cloud-storage/lib/google/cloud/storage.rb index a2ead9271120..ef98c3311087 100644 --- a/google-cloud-storage/lib/google/cloud/storage.rb +++ b/google-cloud-storage/lib/google/cloud/storage.rb @@ -94,8 +94,7 @@ def self.new project_id: nil, credentials: nil, scope: nil, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, endpoint: nil, project: nil, keyfile: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, - multiplier: nil, upload_chunk_size: nil, universe_domain: nil, - upload_url: nil, delete_upload: nil + multiplier: nil, upload_chunk_size: nil, universe_domain: nil scope ||= configure.scope retries ||= configure.retries timeout ||= configure.timeout @@ -110,8 +109,6 @@ def self.new project_id: nil, credentials: nil, scope: nil, retries: nil, multiplier ||= configure.multiplier upload_chunk_size ||= configure.upload_chunk_size universe_domain ||= configure.universe_domain - upload_url ||= configure.upload_url - delete_upload ||= configure.delete_upload unless credentials.is_a? Google::Auth::Credentials credentials = Storage::Credentials.new credentials, scope: scope end @@ -127,7 +124,6 @@ def self.new project_id: nil, credentials: nil, scope: nil, retries: nil, host: endpoint, quota_project: configure.quota_project, max_elapsed_time: max_elapsed_time, base_interval: base_interval, max_interval: max_interval, multiplier: multiplier, upload_chunk_size: upload_chunk_size, - upload_url: upload_url, delete_upload: delete_upload, universe_domain: universe_domain ) ) diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index 5f80729ad74a..e112341904fd 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -1410,6 +1410,28 @@ def delete if_metageneration_match: nil, if_metageneration_not_match: nil user_project: user_project end + ## + # Delete ongoing resumable upload + # @param [String, ::File] file Path of the file on the filesystem to + # upload. Can be an File object, or File-like object such as StringIO. + # (If the object does not have path, a `path` argument must be also be + # provided.) + # @param [String] upload_id Unique Id of an Ongoing resumable upload + # + # @example + # require "google/cloud/storage" + # + # storage = Google::Cloud::Storage.new + # + # bucket = storage.bucket "my-bucket" + # bucket.delete_ongoing_resumable_upload file,upload_id + + + def delete_ongoing_resumable_upload file, upload_id + ensure_service! + ensure_io_or_file_exists! file + service.delete_ongoing_resumable_upload name, file, upload_id + end ## # Retrieves a list of files matching the criteria. # @@ -1799,7 +1821,8 @@ def create_file file, if_generation_match: nil, if_generation_not_match: nil, if_metageneration_match: nil, - if_metageneration_not_match: nil + if_metageneration_not_match: nil, + upload_id: nil ensure_service! ensure_io_or_file_exists! file path ||= file.path if file.respond_to? :path @@ -1830,7 +1853,8 @@ def create_file file, if_generation_not_match: if_generation_not_match, if_metageneration_match: if_metageneration_match, if_metageneration_not_match: if_metageneration_not_match, - user_project: user_project + user_project: user_project, + upload_id: upload_id File.from_gapi gapi, service, user_project: user_project end alias upload_file create_file diff --git a/google-cloud-storage/lib/google/cloud/storage/service.rb b/google-cloud-storage/lib/google/cloud/storage/service.rb index 6d52b1385657..ffe713e3dcbb 100644 --- a/google-cloud-storage/lib/google/cloud/storage/service.rb +++ b/google-cloud-storage/lib/google/cloud/storage/service.rb @@ -49,8 +49,7 @@ def initialize project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, max_interval: nil, - multiplier: nil, upload_chunk_size: nil, universe_domain: nil, - upload_url: nil, delete_upload: nil + multiplier: nil, upload_chunk_size: nil, universe_domain: nil host ||= Google::Cloud::Storage.configure.endpoint @project = project @credentials = credentials @@ -74,8 +73,6 @@ def initialize project, credentials, retries: nil, @service.request_options.multiplier = multiplier if multiplier @service.request_options.add_invocation_id_header = true @service.request_options.upload_chunk_size = upload_chunk_size if upload_chunk_size - @service.request_options.upload_url = upload_url if upload_url - @service.request_options.delete_upload = delete_upload if delete_upload @service.authorization = @credentials.client if @credentials @service.root_url = host if host @service.universe_domain = universe_domain || Google::Cloud::Storage.configure.universe_domain @@ -408,6 +405,7 @@ def insert_file bucket_name, if_metageneration_match: nil, if_metageneration_not_match: nil, user_project: nil, + upload_id: nil, options: {} params = { cache_control: cache_control, @@ -447,6 +445,7 @@ def insert_file bucket_name, if_metageneration_not_match: if_metageneration_not_match, kms_key_name: kms_key, user_project: user_project(user_project), + upload_id: upload_id, options: options end end @@ -657,6 +656,12 @@ def delete_file bucket_name, end end + def delete_ongoing_resumable_upload bucket_name, source, upload_id + execute do + service.delete_ongoing_resumable_upload bucket_name, source, upload_id + end + end + ## # Restores a soft-deleted object. def restore_file bucket_name, diff --git a/google-cloud-storage/test/google/cloud/storage_test.rb b/google-cloud-storage/test/google/cloud/storage_test.rb index a5a9b73ce214..66033eb11448 100644 --- a/google-cloud-storage/test/google/cloud/storage_test.rb +++ b/google-cloud-storage/test/google/cloud/storage_test.rb @@ -126,7 +126,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_be :nil? @@ -223,7 +223,7 @@ def creds.is_a? target it "uses provided endpoint and universe_domain" do stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { _(project).must_equal "project-id" _(credentials).must_equal default_credentials _(retries).must_be :nil? @@ -263,7 +263,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_be :nil? @@ -305,7 +305,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_be :nil? @@ -347,7 +347,7 @@ def creds.is_a? target OpenStruct.new project_id: "project-id" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { _(project).must_equal "project-id" _(credentials).must_be_kind_of OpenStruct _(credentials.project_id).must_equal "project-id" @@ -401,7 +401,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_be :nil? @@ -449,7 +449,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_be :nil? @@ -497,7 +497,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_equal 3 @@ -553,7 +553,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_equal 3 @@ -609,7 +609,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_equal 3 @@ -667,7 +667,7 @@ def creds.is_a? target "storage-credentials" } stubbed_service = ->(project, credentials, retries: nil, timeout: nil, open_timeout: nil, read_timeout: nil, send_timeout: nil, host: nil, quota_project: nil, max_elapsed_time: nil, base_interval: nil, - max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil, upload_url: nil, delete_upload: nil) { + max_interval: nil, multiplier: nil, upload_chunk_size: nil, universe_domain: nil) { _(project).must_equal "project-id" _(credentials).must_equal "storage-credentials" _(retries).must_equal 3 diff --git a/google-cloud-storage/test/helper.rb b/google-cloud-storage/test/helper.rb index 50455d7c7e16..13ef23f4a773 100644 --- a/google-cloud-storage/test/helper.rb +++ b/google-cloud-storage/test/helper.rb @@ -346,6 +346,7 @@ def insert_object_args name: nil, if_metageneration_match: nil, if_metageneration_not_match: nil, user_project: nil, + upload_id: nil, options: {} { name: name, @@ -359,6 +360,7 @@ def insert_object_args name: nil, if_metageneration_match: if_metageneration_match, if_metageneration_not_match: if_metageneration_not_match, user_project: user_project, + upload_id: upload_id, options: options } end From 81663327152d3d40911df9fcdef77f48578a0153 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 10 Mar 2025 20:22:50 +0000 Subject: [PATCH 06/14] fix linter --- google-cloud-storage/lib/google/cloud/storage/bucket.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index e112341904fd..65a69505d2e2 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -1487,6 +1487,7 @@ def delete_ongoing_resumable_upload file, upload_id # puts file.name # end # + def files prefix: nil, delimiter: nil, token: nil, max: nil, versions: nil, match_glob: nil, include_folders_as_prefixes: nil, soft_deleted: nil From be1473628c496b6bdef4b1166beb4b0f3af14706 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 13 Mar 2025 10:02:27 +0000 Subject: [PATCH 07/14] code refactoring --- .../lib/google/cloud/storage/bucket.rb | 14 ++++--- .../lib/google/cloud/storage/service.rb | 11 ++---- .../test/google/cloud/storage/bucket_test.rb | 38 +++++++++++++++++++ google-cloud-storage/test/helper.rb | 2 - 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index 65a69505d2e2..1718f2bf7dc1 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -1416,6 +1416,7 @@ def delete if_metageneration_match: nil, if_metageneration_not_match: nil # upload. Can be an File object, or File-like object such as StringIO. # (If the object does not have path, a `path` argument must be also be # provided.) + # @param [String] file_name Name of file specified for Ongoing resumable upload # @param [String] upload_id Unique Id of an Ongoing resumable upload # # @example @@ -1424,13 +1425,14 @@ def delete if_metageneration_match: nil, if_metageneration_not_match: nil # storage = Google::Cloud::Storage.new # # bucket = storage.bucket "my-bucket" - # bucket.delete_ongoing_resumable_upload file,upload_id + # bucket.delete_ongoing_resumable_upload file,file_name,upload_id - def delete_ongoing_resumable_upload file, upload_id + def delete_ongoing_resumable_upload file, file_name, upload_id ensure_service! ensure_io_or_file_exists! file - service.delete_ongoing_resumable_upload name, file, upload_id + raise "Upload Id missing" unless upload_id + create_file file, file_name, upload_id: upload_id, delete_upload: true end ## # Retrieves a list of files matching the criteria. @@ -1823,7 +1825,8 @@ def create_file file, if_generation_not_match: nil, if_metageneration_match: nil, if_metageneration_not_match: nil, - upload_id: nil + upload_id: nil, + delete_upload: nil ensure_service! ensure_io_or_file_exists! file path ||= file.path if file.respond_to? :path @@ -1855,7 +1858,8 @@ def create_file file, if_metageneration_match: if_metageneration_match, if_metageneration_not_match: if_metageneration_not_match, user_project: user_project, - upload_id: upload_id + upload_id: upload_id, + delete_upload: delete_upload File.from_gapi gapi, service, user_project: user_project end alias upload_file create_file diff --git a/google-cloud-storage/lib/google/cloud/storage/service.rb b/google-cloud-storage/lib/google/cloud/storage/service.rb index ffe713e3dcbb..683c461df1d8 100644 --- a/google-cloud-storage/lib/google/cloud/storage/service.rb +++ b/google-cloud-storage/lib/google/cloud/storage/service.rb @@ -406,6 +406,7 @@ def insert_file bucket_name, if_metageneration_not_match: nil, user_project: nil, upload_id: nil, + delete_upload: nil, options: {} params = { cache_control: cache_control, @@ -430,7 +431,8 @@ def insert_file bucket_name, else options = key_options(key).merge options end - + options.merge! upload_id: upload_id if upload_id + options.merge! delete_upload: delete_upload if delete_upload && upload_id execute do service.insert_object bucket_name, file_obj, @@ -445,7 +447,6 @@ def insert_file bucket_name, if_metageneration_not_match: if_metageneration_not_match, kms_key_name: kms_key, user_project: user_project(user_project), - upload_id: upload_id, options: options end end @@ -656,12 +657,6 @@ def delete_file bucket_name, end end - def delete_ongoing_resumable_upload bucket_name, source, upload_id - execute do - service.delete_ongoing_resumable_upload bucket_name, source, upload_id - end - end - ## # Restores a soft-deleted object. def restore_file bucket_name, diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb index 1dc5babfab0b..45d47f71fe83 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb @@ -1382,6 +1382,44 @@ mock.verify end + it "resstarts a resumable upload with upload_id" do + new_file_name = random_file_path + upload_id= "TEST_ID" + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world" + tmpfile.rewind + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0, upload_id: upload_id}) + + bucket.service.mocked_service = mock + bucket.create_file tmpfile, new_file_name, upload_id: upload_id + + mock.verify + end + end + + it "deletes a resumable upload with upload_id" do + new_file_name = random_file_path + upload_id= "TEST_ID" + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world" + tmpfile.rewind + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0, upload_id: upload_id, delete_upload: true}) + + bucket.service.mocked_service = mock + bucket.delete_ongoing_resumable_upload tmpfile, new_file_name, upload_id + + mock.verify + end + end + def create_file_gapi bucket=nil, name = nil Google::Apis::StorageV1::Object.from_json random_file_hash(bucket, name).to_json end diff --git a/google-cloud-storage/test/helper.rb b/google-cloud-storage/test/helper.rb index 13ef23f4a773..50455d7c7e16 100644 --- a/google-cloud-storage/test/helper.rb +++ b/google-cloud-storage/test/helper.rb @@ -346,7 +346,6 @@ def insert_object_args name: nil, if_metageneration_match: nil, if_metageneration_not_match: nil, user_project: nil, - upload_id: nil, options: {} { name: name, @@ -360,7 +359,6 @@ def insert_object_args name: nil, if_metageneration_match: if_metageneration_match, if_metageneration_not_match: if_metageneration_not_match, user_project: user_project, - upload_id: upload_id, options: options } end From 01868a9f3f403ee78170708b6d9fa4c3e2a532de Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 21 Apr 2025 12:43:21 +0000 Subject: [PATCH 08/14] wip - implementation 2 --- .../lib/google/cloud/storage/bucket.rb | 32 ++++++++++++++++--- .../lib/google/cloud/storage/service.rb | 6 ++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index 1718f2bf7dc1..1b2f8b786fef 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -717,6 +717,30 @@ def default_kms_key= new_default_kms_key patch_gapi! :encryption end + ## + # Restart ongoing resumable upload + # @param [String, ::File] file Path of the file on the filesystem to + # upload. Can be an File object, or File-like object such as StringIO. + # (If the object does not have path, a `path` argument must be also be + # provided.) + # @param [String] upload_id Unique Id of an Ongoing resumable upload + # + # @example + # require "google/cloud/storage" + # + # storage = Google::Cloud::Storage.new + # + # bucket = storage.bucket "my-bucket" + # bucket.restart_ongoing_resumable_upload file,upload_id + + + def restart_ongoing_resumable_upload file, upload_id + ensure_service! + ensure_io_or_file_exists! file + raise "Upload Id missing" unless upload_id + service.restart_delete_ongoing_resumable_upload name, file, upload_id + end + ## # The period of time (in seconds) that files in the bucket must be # retained, and cannot be deleted, overwritten, or archived. @@ -1416,7 +1440,6 @@ def delete if_metageneration_match: nil, if_metageneration_not_match: nil # upload. Can be an File object, or File-like object such as StringIO. # (If the object does not have path, a `path` argument must be also be # provided.) - # @param [String] file_name Name of file specified for Ongoing resumable upload # @param [String] upload_id Unique Id of an Ongoing resumable upload # # @example @@ -1425,14 +1448,13 @@ def delete if_metageneration_match: nil, if_metageneration_not_match: nil # storage = Google::Cloud::Storage.new # # bucket = storage.bucket "my-bucket" - # bucket.delete_ongoing_resumable_upload file,file_name,upload_id - + # bucket.delete_ongoing_resumable_upload file,upload_id - def delete_ongoing_resumable_upload file, file_name, upload_id + def delete_ongoing_resumable_upload file, upload_id ensure_service! ensure_io_or_file_exists! file raise "Upload Id missing" unless upload_id - create_file file, file_name, upload_id: upload_id, delete_upload: true + service.restart_delete_ongoing_resumable_upload name, file, upload_id, options: {delete_upload: true} end ## # Retrieves a list of files matching the criteria. diff --git a/google-cloud-storage/lib/google/cloud/storage/service.rb b/google-cloud-storage/lib/google/cloud/storage/service.rb index 683c461df1d8..a85caf806662 100644 --- a/google-cloud-storage/lib/google/cloud/storage/service.rb +++ b/google-cloud-storage/lib/google/cloud/storage/service.rb @@ -657,6 +657,12 @@ def delete_file bucket_name, end end + def restart_delete_ongoing_resumable_upload bucket_name, source, upload_id, options: {} + execute do + service.restart_delete_ongoing_resumable_upload bucket_name, source, upload_id, options: options + end + end + ## # Restores a soft-deleted object. def restore_file bucket_name, From b6bd6186d3ffd0fb96e6dc067389e42036a7efbb Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 24 Apr 2025 11:03:05 +0000 Subject: [PATCH 09/14] changes working --- .../lib/google/cloud/storage/bucket.rb | 12 ++++++------ .../lib/google/cloud/storage/service.rb | 10 ++++++++-- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index 1b2f8b786fef..427646528b24 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -731,14 +731,14 @@ def default_kms_key= new_default_kms_key # storage = Google::Cloud::Storage.new # # bucket = storage.bucket "my-bucket" - # bucket.restart_ongoing_resumable_upload file,upload_id + # bucket.restart_resumable_upload file,upload_id - def restart_ongoing_resumable_upload file, upload_id + def restart_resumable_upload file, upload_id ensure_service! ensure_io_or_file_exists! file raise "Upload Id missing" unless upload_id - service.restart_delete_ongoing_resumable_upload name, file, upload_id + service.restart_resumable_upload name, file, upload_id end ## @@ -1448,13 +1448,13 @@ def delete if_metageneration_match: nil, if_metageneration_not_match: nil # storage = Google::Cloud::Storage.new # # bucket = storage.bucket "my-bucket" - # bucket.delete_ongoing_resumable_upload file,upload_id + # bucket.delete_resumable_upload file,upload_id - def delete_ongoing_resumable_upload file, upload_id + def delete_resumable_upload file, upload_id ensure_service! ensure_io_or_file_exists! file raise "Upload Id missing" unless upload_id - service.restart_delete_ongoing_resumable_upload name, file, upload_id, options: {delete_upload: true} + service.delete_resumable_upload name, file, upload_id, options: {delete_upload: true} end ## # Retrieves a list of files matching the criteria. diff --git a/google-cloud-storage/lib/google/cloud/storage/service.rb b/google-cloud-storage/lib/google/cloud/storage/service.rb index a85caf806662..e532b2bca674 100644 --- a/google-cloud-storage/lib/google/cloud/storage/service.rb +++ b/google-cloud-storage/lib/google/cloud/storage/service.rb @@ -657,9 +657,15 @@ def delete_file bucket_name, end end - def restart_delete_ongoing_resumable_upload bucket_name, source, upload_id, options: {} + def restart_resumable_upload bucket_name, source, upload_id, options: {} execute do - service.restart_delete_ongoing_resumable_upload bucket_name, source, upload_id, options: options + service.restart_resumable_upload bucket_name, source, upload_id, options: options + end + end + + def delete_resumable_upload bucket_name, source, upload_id, options: {} + execute do + service.delete_resumable_upload bucket_name, source, upload_id, options: options end end From ece5ae330531b2aa03c7ef4f1ee198928544572c Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Wed, 30 Apr 2025 11:05:57 +0000 Subject: [PATCH 10/14] adding test cases --- .../lib/google/cloud/storage/bucket.rb | 8 +++---- .../test/google/cloud/storage/bucket_test.rb | 22 +++++++++---------- google-cloud-storage/test/helper.rb | 12 ++++++++++ 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index 427646528b24..7a049623bc09 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -718,12 +718,12 @@ def default_kms_key= new_default_kms_key end ## - # Restart ongoing resumable upload + # Restart resumable upload # @param [String, ::File] file Path of the file on the filesystem to # upload. Can be an File object, or File-like object such as StringIO. # (If the object does not have path, a `path` argument must be also be # provided.) - # @param [String] upload_id Unique Id of an Ongoing resumable upload + # @param [String] upload_id Unique Id of an resumable upload # # @example # require "google/cloud/storage" @@ -1435,12 +1435,12 @@ def delete if_metageneration_match: nil, if_metageneration_not_match: nil end ## - # Delete ongoing resumable upload + # Delete resumable upload # @param [String, ::File] file Path of the file on the filesystem to # upload. Can be an File object, or File-like object such as StringIO. # (If the object does not have path, a `path` argument must be also be # provided.) - # @param [String] upload_id Unique Id of an Ongoing resumable upload + # @param [String] upload_id Unique Id of an resumable upload # # @example # require "google/cloud/storage" diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb index 45d47f71fe83..259bf24c05c5 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb @@ -1382,27 +1382,25 @@ mock.verify end - it "resstarts a resumable upload with upload_id" do + it "restarts a resumable upload with upload_id" do new_file_name = random_file_path upload_id= "TEST_ID" Tempfile.open ["google-cloud", ".txt"] do |tmpfile| tmpfile.write "Hello world" tmpfile.rewind - mock = Minitest::Mock.new - mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), - [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0, upload_id: upload_id}) - + mock.expect :restart_resumable_upload, create_file_gapi(bucket.name, new_file_name), + [bucket.name, tmpfile, upload_id], + **restart_resumable_upload_args(options: {}) bucket.service.mocked_service = mock - bucket.create_file tmpfile, new_file_name, upload_id: upload_id + bucket.restart_resumable_upload tmpfile, upload_id mock.verify end end - it "deletes a resumable upload with upload_id" do - new_file_name = random_file_path + it "deletes a resumable upload with upload_id" do upload_id= "TEST_ID" Tempfile.open ["google-cloud", ".txt"] do |tmpfile| @@ -1410,11 +1408,11 @@ tmpfile.rewind mock = Minitest::Mock.new - mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), - [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0, upload_id: upload_id, delete_upload: true}) - + mock.expect :delete_resumable_upload, true, + [bucket.name, tmpfile, upload_id], + **delete_resumable_upload_args(options: {delete_upload: true}) bucket.service.mocked_service = mock - bucket.delete_ongoing_resumable_upload tmpfile, new_file_name, upload_id + bucket.delete_resumable_upload tmpfile, upload_id mock.verify end diff --git a/google-cloud-storage/test/helper.rb b/google-cloud-storage/test/helper.rb index 50455d7c7e16..b7597a746408 100644 --- a/google-cloud-storage/test/helper.rb +++ b/google-cloud-storage/test/helper.rb @@ -363,6 +363,18 @@ def insert_object_args name: nil, } end + def delete_resumable_upload_args options: {} + { + options: options + } + end + + def restart_resumable_upload_args options: {} + { + options: options + } + end + def get_object_args generation: nil, if_generation_match: nil, if_generation_not_match: nil, From 160c98006e1bc7aded6ba99fd20f7d8e8a9d2135 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Wed, 30 Apr 2025 11:21:32 +0000 Subject: [PATCH 11/14] fix typo --- google-cloud-storage/lib/google/cloud/storage/bucket.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index 7a049623bc09..bd8e679589d0 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -1454,7 +1454,7 @@ def delete_resumable_upload file, upload_id ensure_service! ensure_io_or_file_exists! file raise "Upload Id missing" unless upload_id - service.delete_resumable_upload name, file, upload_id, options: {delete_upload: true} + service.delete_resumable_upload name, file, upload_id, options: { delete_upload: true } end ## # Retrieves a list of files matching the criteria. From 725bea7d20491788799ce53969ecc95ab0ccb890 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Wed, 30 Apr 2025 11:32:15 +0000 Subject: [PATCH 12/14] removing unwanted change --- google-cloud-storage/lib/google/cloud/storage/bucket.rb | 8 ++------ google-cloud-storage/lib/google/cloud/storage/service.rb | 4 ---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index bd8e679589d0..5fc7537990f9 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -1846,9 +1846,7 @@ def create_file file, if_generation_match: nil, if_generation_not_match: nil, if_metageneration_match: nil, - if_metageneration_not_match: nil, - upload_id: nil, - delete_upload: nil + if_metageneration_not_match: nil ensure_service! ensure_io_or_file_exists! file path ||= file.path if file.respond_to? :path @@ -1879,9 +1877,7 @@ def create_file file, if_generation_not_match: if_generation_not_match, if_metageneration_match: if_metageneration_match, if_metageneration_not_match: if_metageneration_not_match, - user_project: user_project, - upload_id: upload_id, - delete_upload: delete_upload + user_project: user_project File.from_gapi gapi, service, user_project: user_project end alias upload_file create_file diff --git a/google-cloud-storage/lib/google/cloud/storage/service.rb b/google-cloud-storage/lib/google/cloud/storage/service.rb index e532b2bca674..0b408985bab7 100644 --- a/google-cloud-storage/lib/google/cloud/storage/service.rb +++ b/google-cloud-storage/lib/google/cloud/storage/service.rb @@ -405,8 +405,6 @@ def insert_file bucket_name, if_metageneration_match: nil, if_metageneration_not_match: nil, user_project: nil, - upload_id: nil, - delete_upload: nil, options: {} params = { cache_control: cache_control, @@ -431,8 +429,6 @@ def insert_file bucket_name, else options = key_options(key).merge options end - options.merge! upload_id: upload_id if upload_id - options.merge! delete_upload: delete_upload if delete_upload && upload_id execute do service.insert_object bucket_name, file_obj, From 950b5b422200573090bcce6f8ffb5b483e8820fe Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Wed, 30 Apr 2025 11:37:00 +0000 Subject: [PATCH 13/14] removing unwanted changes --- google-cloud-storage/lib/google/cloud/storage.rb | 1 + google-cloud-storage/lib/google/cloud/storage/bucket.rb | 1 - google-cloud-storage/lib/google/cloud/storage/service.rb | 1 + 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-storage/lib/google/cloud/storage.rb b/google-cloud-storage/lib/google/cloud/storage.rb index ef98c3311087..cfc063f60be1 100644 --- a/google-cloud-storage/lib/google/cloud/storage.rb +++ b/google-cloud-storage/lib/google/cloud/storage.rb @@ -109,6 +109,7 @@ def self.new project_id: nil, credentials: nil, scope: nil, retries: nil, multiplier ||= configure.multiplier upload_chunk_size ||= configure.upload_chunk_size universe_domain ||= configure.universe_domain + unless credentials.is_a? Google::Auth::Credentials credentials = Storage::Credentials.new credentials, scope: scope end diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index 5fc7537990f9..556a5c2d195c 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -733,7 +733,6 @@ def default_kms_key= new_default_kms_key # bucket = storage.bucket "my-bucket" # bucket.restart_resumable_upload file,upload_id - def restart_resumable_upload file, upload_id ensure_service! ensure_io_or_file_exists! file diff --git a/google-cloud-storage/lib/google/cloud/storage/service.rb b/google-cloud-storage/lib/google/cloud/storage/service.rb index 0b408985bab7..a506b9b27e50 100644 --- a/google-cloud-storage/lib/google/cloud/storage/service.rb +++ b/google-cloud-storage/lib/google/cloud/storage/service.rb @@ -429,6 +429,7 @@ def insert_file bucket_name, else options = key_options(key).merge options end + execute do service.insert_object bucket_name, file_obj, From ca4048a2bba51c1d125ae1611d8cb814809457c9 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Wed, 7 May 2025 05:50:31 +0000 Subject: [PATCH 14/14] rewritting delete upload --- .../lib/google/cloud/storage/bucket.rb | 9 ++------- .../lib/google/cloud/storage/service.rb | 4 ++-- .../test/google/cloud/storage/bucket_test.rb | 20 +++++++------------ 3 files changed, 11 insertions(+), 22 deletions(-) diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index 556a5c2d195c..a8ee4d3b6d83 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -1435,10 +1435,6 @@ def delete if_metageneration_match: nil, if_metageneration_not_match: nil ## # Delete resumable upload - # @param [String, ::File] file Path of the file on the filesystem to - # upload. Can be an File object, or File-like object such as StringIO. - # (If the object does not have path, a `path` argument must be also be - # provided.) # @param [String] upload_id Unique Id of an resumable upload # # @example @@ -1449,11 +1445,10 @@ def delete if_metageneration_match: nil, if_metageneration_not_match: nil # bucket = storage.bucket "my-bucket" # bucket.delete_resumable_upload file,upload_id - def delete_resumable_upload file, upload_id + def delete_resumable_upload upload_id ensure_service! - ensure_io_or_file_exists! file raise "Upload Id missing" unless upload_id - service.delete_resumable_upload name, file, upload_id, options: { delete_upload: true } + service.delete_resumable_upload name, upload_id, options: { delete_upload: true } end ## # Retrieves a list of files matching the criteria. diff --git a/google-cloud-storage/lib/google/cloud/storage/service.rb b/google-cloud-storage/lib/google/cloud/storage/service.rb index a506b9b27e50..0be1c0420044 100644 --- a/google-cloud-storage/lib/google/cloud/storage/service.rb +++ b/google-cloud-storage/lib/google/cloud/storage/service.rb @@ -660,9 +660,9 @@ def restart_resumable_upload bucket_name, source, upload_id, options: {} end end - def delete_resumable_upload bucket_name, source, upload_id, options: {} + def delete_resumable_upload bucket_name, upload_id, options: {} execute do - service.delete_resumable_upload bucket_name, source, upload_id, options: options + service.delete_resumable_upload bucket_name, upload_id, options: options end end diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb index 259bf24c05c5..4e79a3e93554 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb @@ -1403,19 +1403,13 @@ it "deletes a resumable upload with upload_id" do upload_id= "TEST_ID" - Tempfile.open ["google-cloud", ".txt"] do |tmpfile| - tmpfile.write "Hello world" - tmpfile.rewind - - mock = Minitest::Mock.new - mock.expect :delete_resumable_upload, true, - [bucket.name, tmpfile, upload_id], - **delete_resumable_upload_args(options: {delete_upload: true}) - bucket.service.mocked_service = mock - bucket.delete_resumable_upload tmpfile, upload_id - - mock.verify - end + mock = Minitest::Mock.new + mock.expect :delete_resumable_upload, true, + [bucket.name, upload_id], + **delete_resumable_upload_args(options: {delete_upload: true}) + bucket.service.mocked_service = mock + bucket.delete_resumable_upload upload_id + mock.verify end def create_file_gapi bucket=nil, name = nil