From 0b91b6b1c6f8e1509639e636597392eb74e17226 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 13 Feb 2025 15:14:39 +0000 Subject: [PATCH 01/26] functionality working --- .../lib/google/apis/core/storage_upload.rb | 80 ++++++++++++++++++- google-apis-core/lib/google/apis/options.rb | 4 +- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index bff1b88f60c..8f0954ce17e 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -97,8 +97,23 @@ def execute(client) opencensus_begin_span @upload_chunk_size = options.upload_chunk_size - do_retry :initiate_resumable_upload, client + if options.upload_url.nil? + do_retry :initiate_resumable_upload, client + puts 'Initiating resumable upload' + + elsif options.delete_upload && options.upload_url + @close_io_on_finish = true + @upload_incomplete = false + @upload_url = options.upload_url + cancel_resumable_upload(client) + puts 'Deleting resumable upload' + else + do_retry :reinitiate_resumable_upload, client + puts 'Restarting resumable upload' + end + while @upload_incomplete + res = do_retry :send_upload_command, client end res @@ -131,6 +146,23 @@ def initiate_resumable_upload(client) error(e, rethrow: true) end + # Initiating resumable upload + # storage = Google::Cloud::Storage.new(upload_chunk_size: chunk_size) + # Re-initiating resumable upload + # storage = Google::Cloud::Storage.new(upload_chunk_size: chunk_size, upload_url: upload_url ) + + def reinitiate_resumable_upload(client) + logger.debug { sprintf('Restarting resumable upload command to %s', url) } + @upload_url = options.upload_url unless options.upload_url.nil? + check_resumable_upload_status client + + # moves the file pointer to the position specified by uploaded_bytes/ offset. + # This is useful for resuming an upload from where it left off. + upload_io.seek(@offset) + rescue => e + error(e, rethrow: true) + end + # Send the actual content # # @param [HTTPClient] client @@ -182,6 +214,52 @@ def process_response(status, header, body) super(status, header, body) end + def check_resumable_upload_status(client) + # Setting up request header + request_header = header.dup + request_header[CONTENT_RANGE_HEADER] = "bytes */#{upload_io.size}" + request_header[CONTENT_LENGTH_HEADER] = '0' + # Initiating call + response = client.put(@upload_url, header: request_header, follow_redirect: true) + case response.code.to_i + when 308 + if response.headers['Range'] + range = response.headers['Range'] + @offset = range ? range.split('-').last.to_i + 1 : 0 + puts "Upload is incomplete. Bytes uploaded: #{response.headers['Range']}" + else + puts 'No bytes uploaded yet.' + end + @upload_incomplete = true + when 200, 201 + puts 'Upload is complete.' + @upload_incomplete = false + else + puts "Unexpected response: #{response.code} - #{response.body}" + @upload_incomplete = true + end + end + + # Cancel and resumable upload + # storage = Google::Cloud::Storage.new(upload_chunk_size: chunk_size, upload_url: upload_url, delete_upload: true) + + + def cancel_resumable_upload(client) + # Setting up request header + request_header = header.dup + request_header[CONTENT_LENGTH_HEADER] = '0' + # Initiating call + response = client.delete(@upload_url, header: request_header, follow_redirect: true) + case response.code.to_i + when 499 + puts 'Resumable upload session canceled successfully.' + when 404 + puts 'Resumable upload session not found.' + else + puts "Failed to cancel upload session. Response: #{response.code} - #{response.body}" + end + end + def streamable?(upload_source) upload_source.is_a?(IO) || upload_source.is_a?(StringIO) || upload_source.is_a?(Tempfile) end diff --git a/google-apis-core/lib/google/apis/options.rb b/google-apis-core/lib/google/apis/options.rb index b5a0c99f4d3..7a3bc97dbec 100644 --- a/google-apis-core/lib/google/apis/options.rb +++ b/google-apis-core/lib/google/apis/options.rb @@ -41,7 +41,9 @@ module Apis :quota_project, :query, :add_invocation_id_header, - :upload_chunk_size) + :upload_chunk_size, + :upload_url, + :delete_upload) # General client options class ClientOptions From 63b3af0a027f58680bcedc29b76b7147e036b075 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Tue, 25 Feb 2025 12:46:05 +0000 Subject: [PATCH 02/26] adding unit test cases --- .../lib/google/apis/core/storage_upload.rb | 34 ++---- .../google/apis/core/storage_upload_spec.rb | 100 ++++++++++++++++++ 2 files changed, 109 insertions(+), 25 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 8f0954ce17e..7cb49b5ae9d 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -18,7 +18,6 @@ require 'stringio' require 'tempfile' require 'mini_mime' - module Google module Apis module Core @@ -96,20 +95,13 @@ def execute(client) prepare! opencensus_begin_span @upload_chunk_size = options.upload_chunk_size - if options.upload_url.nil? do_retry :initiate_resumable_upload, client - puts 'Initiating resumable upload' - - elsif options.delete_upload && options.upload_url - @close_io_on_finish = true - @upload_incomplete = false + elsif options.delete_upload && !options.upload_url.nil? @upload_url = options.upload_url cancel_resumable_upload(client) - puts 'Deleting resumable upload' else do_retry :reinitiate_resumable_upload, client - puts 'Restarting resumable upload' end while @upload_incomplete @@ -147,18 +139,12 @@ def initiate_resumable_upload(client) end # Initiating resumable upload - # storage = Google::Cloud::Storage.new(upload_chunk_size: chunk_size) - # Re-initiating resumable upload - # storage = Google::Cloud::Storage.new(upload_chunk_size: chunk_size, upload_url: upload_url ) def reinitiate_resumable_upload(client) logger.debug { sprintf('Restarting resumable upload command to %s', url) } @upload_url = options.upload_url unless options.upload_url.nil? check_resumable_upload_status client - - # moves the file pointer to the position specified by uploaded_bytes/ offset. - # This is useful for resuming an upload from where it left off. - upload_io.seek(@offset) + upload_io.pos = @offset rescue => e error(e, rethrow: true) end @@ -184,7 +170,6 @@ def send_upload_command(client) else StringIO.new(upload_io.read(current_chunk_size)) end - response = client.put(@upload_url, body: chunk_body, header: request_header, follow_redirect: true) result = process_response(response.status_code, response.header, response.body) @@ -231,8 +216,11 @@ def check_resumable_upload_status(client) puts 'No bytes uploaded yet.' end @upload_incomplete = true + when 499 + # Upload in canceled + @upload_incomplete = false when 200, 201 - puts 'Upload is complete.' + # Upload is complete. @upload_incomplete = false else puts "Unexpected response: #{response.code} - #{response.body}" @@ -240,10 +228,7 @@ def check_resumable_upload_status(client) end end - # Cancel and resumable upload - # storage = Google::Cloud::Storage.new(upload_chunk_size: chunk_size, upload_url: upload_url, delete_upload: true) - - + # Cancel resumable upload def cancel_resumable_upload(client) # Setting up request header request_header = header.dup @@ -252,9 +237,8 @@ def cancel_resumable_upload(client) response = client.delete(@upload_url, header: request_header, follow_redirect: true) case response.code.to_i when 499 - puts 'Resumable upload session canceled successfully.' - when 404 - puts 'Resumable upload session not found.' + @close_io_on_finish = true + @upload_incomplete = false else puts "Failed to cancel upload session. Response: #{response.code} - #{response.body}" end diff --git a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb index a0d2752c211..d9e2015ec25 100644 --- a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb +++ b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb @@ -129,6 +129,106 @@ end end + context('restart resumable upload with upload_url') do + let(:file) { StringIO.new('Hello world' * 3) } + let(:upload_url) { 'https://www.googleapis.com/zoo/animals' } + + before(:example) do + stub_request(:put, upload_url) + .with( + headers: { + 'Content-Length' => '0', + 'Content-Range' => 'bytes */33' + } + ) + .to_return( + status: [308, 'Resume Incomplete'], + headers: { 'Range' => 'bytes=0-21' }) + end + + before(:example) do + stub_request(:put, upload_url) + .with(headers: { 'Content-Range' => 'bytes 22-32/33' }) + .to_return(body: %(OK)) + end + + it 'should restart a resumable upload' do + command.options.upload_chunk_size = 11 + command.options.upload_url = upload_url + command.execute(client) + expect(a_request(:put, upload_url) + .with(body: 'Hello world')).to have_been_made + end + end + + context('should not restart resumable upload if upload is completed') do + let(:file) { StringIO.new('Hello world' * 3) } + let(:upload_url) { 'https://www.googleapis.com/zoo/animals' } + + before(:example) do + stub_request(:put, upload_url) + .with( + headers: { + 'Content-Length' => '0', + 'Content-Range' => 'bytes */33' + } + ) + .to_return(status: 200, headers: { 'Range' => 'bytes=0-32' }) + end + + before(:example) do + stub_request(:put, upload_url) + .with(headers: { 'Content-Range' => 'bytes */33' }) + .to_return(status: 200) + end + + it 'should not restart a upload' do + command.options.upload_chunk_size = 11 + command.options.upload_url = upload_url + command.execute(client) + expect(a_request(:put, upload_url) + .with(body: 'Hello world')).to have_not_been_made + end + end + + context('delete resumable upload with upload_url') do + let(:file) { StringIO.new('Hello world' * 3) } + let(:upload_url) { 'https://www.googleapis.com/zoo/animals' } + + before(:example) do + stub_request(:delete, upload_url) + .with(headers: { 'Content-Length' => '0' }) + .to_return(status: [499]) + end + + before(:example) do + stub_request(:put, upload_url) + .with( + headers: { + 'Content-Length' => '0', + 'Content-Range' => 'bytes */33' + } + ) + .to_return(status: [499]) + end + + it 'should cancel a resumable upload' do + command.options.upload_chunk_size = 11 + command.options.upload_url = upload_url + command.options.delete_upload = true + command.execute(client) + expect(a_request(:delete, upload_url)).to have_been_made + end + + it 'should not call resumable upload when delete upload is called' do + command.options.upload_chunk_size = 11 + command.options.upload_url = upload_url + command.execute(client) + expect(a_request(:put, upload_url) + .with(body: 'Hello world')).to have_not_been_made + end + end + context('with chunking disabled') do let!(:file) { StringIO.new("Hello world")} include_examples 'should upload' From 308ef36c52fdf92992ff040ebf40929b6cc19cab Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Tue, 25 Feb 2025 12:59:36 +0000 Subject: [PATCH 03/26] fix typo --- .../spec/google/apis/core/storage_upload_spec.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb index d9e2015ec25..c31ccd377d4 100644 --- a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb +++ b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb @@ -143,7 +143,8 @@ ) .to_return( status: [308, 'Resume Incomplete'], - headers: { 'Range' => 'bytes=0-21' }) + headers: { 'Range' => 'bytes=0-21' } + ) end before(:example) do @@ -220,7 +221,7 @@ expect(a_request(:delete, upload_url)).to have_been_made end - it 'should not call resumable upload when delete upload is called' do + it 'should not call resumable upload when upload is cancelled' do command.options.upload_chunk_size = 11 command.options.upload_url = upload_url command.execute(client) From 876827159b5433c01f889bed9f3f3cdc942be9c4 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 27 Feb 2025 05:21:21 +0000 Subject: [PATCH 04/26] returning upload_url will error --- google-apis-core/lib/google/apis/core/storage_upload.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 7cb49b5ae9d..1a0ef5c9ce7 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -132,6 +132,7 @@ def initiate_resumable_upload(client) body: body, header: request_header, follow_redirect: true) + result = process_response(response.status_code, response.header, response.body) success(result) rescue => e @@ -171,14 +172,14 @@ def send_upload_command(client) StringIO.new(upload_io.read(current_chunk_size)) end response = client.put(@upload_url, body: chunk_body, header: request_header, follow_redirect: true) - result = process_response(response.status_code, response.header, response.body) @upload_incomplete = false if response.status_code.eql? OK_STATUS @offset += current_chunk_size if @upload_incomplete success(result) - rescue => e - upload_io.pos = @offset - error(e, rethrow: true) + rescue => e + upload_io.pos = @offset + e.message = e.message + "Please save this upload_url==>#{@upload_url}" + error(e, rethrow: true) end # Check the to see if the upload is complete or needs to be resumed. From cb561a44a0a65731cde5fbb25642f27ac5301a79 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 27 Feb 2025 05:27:47 +0000 Subject: [PATCH 05/26] fix typo --- google-apis-core/lib/google/apis/core/storage_upload.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 1a0ef5c9ce7..a1fb39d4e28 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -178,7 +178,7 @@ def send_upload_command(client) success(result) rescue => e upload_io.pos = @offset - e.message = e.message + "Please save this upload_url==>#{@upload_url}" + e.message = e.message + ", Please save this upload_url: #{@upload_url}" error(e, rethrow: true) end From 1d26f0d55b8e9054e9a199128a66e0f8f0406363 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 27 Feb 2025 10:14:11 +0000 Subject: [PATCH 06/26] removing --- google-apis-core/lib/google/apis/core/storage_upload.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index a1fb39d4e28..6a58e43ddba 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -18,6 +18,7 @@ require 'stringio' require 'tempfile' require 'mini_mime' +require 'pry' module Google module Apis module Core @@ -178,7 +179,6 @@ def send_upload_command(client) success(result) rescue => e upload_io.pos = @offset - e.message = e.message + ", Please save this upload_url: #{@upload_url}" error(e, rethrow: true) end From fd8b6fda0e77429a7bc86c6e9a8595b88eae9922 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 27 Feb 2025 11:19:49 +0000 Subject: [PATCH 07/26] removing pry --- google-apis-core/lib/google/apis/core/storage_upload.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 6a58e43ddba..b87c37f9a48 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -18,7 +18,6 @@ require 'stringio' require 'tempfile' require 'mini_mime' -require 'pry' module Google module Apis module Core From 02be397cca4cbf2600b04c06589cf135873a24e7 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 27 Feb 2025 11:56:52 +0000 Subject: [PATCH 08/26] fix typo --- google-apis-core/lib/google/apis/core/storage_upload.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index b87c37f9a48..a4d688c9151 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -139,8 +139,7 @@ def initiate_resumable_upload(client) error(e, rethrow: true) end - # Initiating resumable upload - + # Restarting resumable upload def reinitiate_resumable_upload(client) logger.debug { sprintf('Restarting resumable upload command to %s', url) } @upload_url = options.upload_url unless options.upload_url.nil? From 5daa50a7e451207e763b0fa58fdf2896f4f95870 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 10 Mar 2025 18:33:37 +0000 Subject: [PATCH 09/26] changing approach --- .../lib/google/apis/storage_v1/service.rb | 12 ++++++- .../lib/google/apis/core/storage_upload.rb | 34 ++++++++++++++----- google-apis-core/lib/google/apis/options.rb | 5 ++- .../google/apis/core/storage_upload_spec.rb | 24 +++++++------ 4 files changed, 52 insertions(+), 23 deletions(-) diff --git a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb index 6cee008d917..e4d9db84d54 100644 --- a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb +++ b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb @@ -3848,7 +3848,17 @@ def delete_project_hmac_key(project_id, access_id, user_project: nil, fields: ni command.query['userIp'] = user_ip unless user_ip.nil? execute_or_queue_command(command, &block) end - + + # Deletes Resumable upload + def delete_ongoing_resumable_upload(bucket, upload_source, upload_id, options: nil) + command = make_storage_upload_command(:post, 'b/{bucket}/o', options) + command.upload_source = upload_source + command.upload_id = upload_id + command.params['bucket'] = bucket unless bucket.nil? + command.delete_upload = true + execute_or_queue_command(command) + end + # Retrieves an HMAC key's metadata # @param [String] project_id # Project ID owning the service account of the requested key. diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index a4d688c9151..65b2fabc0be 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -36,6 +36,14 @@ class StorageUploadCommand < ApiCommand # @return [String, File, #read] attr_accessor :upload_source + # Unique upload_id of a resumable upload + # @return [String] + attr_accessor :upload_id + + # Boolean Value to specify is a resumable upload is to be deleted or not + # @return [Boolean] + attr_accessor :delete_upload + # Content type of the upload material # @return [String] attr_accessor :upload_content_type @@ -95,13 +103,14 @@ def execute(client) prepare! opencensus_begin_span @upload_chunk_size = options.upload_chunk_size - if options.upload_url.nil? + if upload_id.nil? do_retry :initiate_resumable_upload, client - elsif options.delete_upload && !options.upload_url.nil? - @upload_url = options.upload_url + elsif delete_upload && !upload_id.nil? + make_resumabple_upload_url cancel_resumable_upload(client) else - do_retry :reinitiate_resumable_upload, client + make_resumabple_upload_url + reinitiate_resumable_upload(client) end while @upload_incomplete @@ -139,16 +148,23 @@ def initiate_resumable_upload(client) error(e, rethrow: true) end - # Restarting resumable upload + # Reinitiating resumable upload + def reinitiate_resumable_upload(client) logger.debug { sprintf('Restarting resumable upload command to %s', url) } - @upload_url = options.upload_url unless options.upload_url.nil? check_resumable_upload_status client upload_io.pos = @offset rescue => e error(e, rethrow: true) end + def make_resumabple_upload_url + query_params = query.dup + query_params['uploadType'] = RESUMABLE + query_params['upload_id'] = upload_id + resumable_upload_params = query_params.map { |key, value| "#{key}=#{value}" }.join('&') + @upload_url = "#{url}&#{resumable_upload_params}" + end # Send the actual content # # @param [HTTPClient] client @@ -175,9 +191,9 @@ def send_upload_command(client) @upload_incomplete = false if response.status_code.eql? OK_STATUS @offset += current_chunk_size if @upload_incomplete success(result) - rescue => e - upload_io.pos = @offset - error(e, rethrow: true) + rescue => e + upload_io.pos = @offset + error(e, rethrow: true) end # Check the to see if the upload is complete or needs to be resumed. diff --git a/google-apis-core/lib/google/apis/options.rb b/google-apis-core/lib/google/apis/options.rb index 7a3bc97dbec..f1ac437c9ed 100644 --- a/google-apis-core/lib/google/apis/options.rb +++ b/google-apis-core/lib/google/apis/options.rb @@ -41,9 +41,8 @@ module Apis :quota_project, :query, :add_invocation_id_header, - :upload_chunk_size, - :upload_url, - :delete_upload) + :upload_chunk_size + ) # General client options class ClientOptions diff --git a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb index c31ccd377d4..6eda8b216f1 100644 --- a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb +++ b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb @@ -119,7 +119,7 @@ stub_request(:put, 'https://www.googleapis.com/zoo/animals') .with(headers: { 'Content-Range' => 'bytes 11-21/22' }) .to_return(body: %(OK)) - end + end it 'should make requests multiple times' do command.options.upload_chunk_size = 11 @@ -131,7 +131,8 @@ context('restart resumable upload with upload_url') do let(:file) { StringIO.new('Hello world' * 3) } - let(:upload_url) { 'https://www.googleapis.com/zoo/animals' } + let(:upload_id) { 'TestId' } + let(:upload_url) { "https://www.googleapis.com/zoo/animals?uploadType=resumable&upload_id=#{upload_id}" } before(:example) do stub_request(:put, upload_url) @@ -155,7 +156,7 @@ it 'should restart a resumable upload' do command.options.upload_chunk_size = 11 - command.options.upload_url = upload_url + command.upload_id = upload_id command.execute(client) expect(a_request(:put, upload_url) .with(body: 'Hello world')).to have_been_made @@ -164,7 +165,8 @@ context('should not restart resumable upload if upload is completed') do let(:file) { StringIO.new('Hello world' * 3) } - let(:upload_url) { 'https://www.googleapis.com/zoo/animals' } + let(:upload_id) {"TestId"} + let(:upload_url) { "https://www.googleapis.com/zoo/animals?uploadType=resumable&upload_id=#{upload_id}" } before(:example) do stub_request(:put, upload_url) @@ -185,16 +187,18 @@ it 'should not restart a upload' do command.options.upload_chunk_size = 11 - command.options.upload_url = upload_url + command.upload_id = upload_id command.execute(client) expect(a_request(:put, upload_url) .with(body: 'Hello world')).to have_not_been_made end end - context('delete resumable upload with upload_url') do + context('delete resumable upload with upload_id') do let(:file) { StringIO.new('Hello world' * 3) } - let(:upload_url) { 'https://www.googleapis.com/zoo/animals' } + let(:upload_id) { 'TestId' } + let(:upload_url) { "https://www.googleapis.com/zoo/animals?uploadType=resumable&upload_id=#{upload_id}" } + before(:example) do stub_request(:delete, upload_url) @@ -215,15 +219,15 @@ it 'should cancel a resumable upload' do command.options.upload_chunk_size = 11 - command.options.upload_url = upload_url - command.options.delete_upload = true + command.upload_id = upload_id + command.delete_upload = true command.execute(client) expect(a_request(:delete, upload_url)).to have_been_made end it 'should not call resumable upload when upload is cancelled' do command.options.upload_chunk_size = 11 - command.options.upload_url = upload_url + command.upload_id = upload_id command.execute(client) expect(a_request(:put, upload_url) .with(body: 'Hello world')).to have_not_been_made From fa2b1aeee5b879ccb580e0e8c3a34c24aaadf6fb Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 10 Mar 2025 20:09:58 +0000 Subject: [PATCH 10/26] adding upload_id param --- .../lib/google/apis/storage_v1/service.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb index e4d9db84d54..0d2d5b834ed 100644 --- a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb +++ b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb @@ -2870,7 +2870,7 @@ def get_object_iam_policy(bucket, object, generation: nil, user_project: nil, fi # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification # @raise [Google::Apis::AuthorizationError] Authorization is required - def insert_object(bucket, object_object = nil, content_encoding: nil, if_generation_match: nil, if_generation_not_match: nil, if_metageneration_match: nil, if_metageneration_not_match: nil, kms_key_name: nil, name: nil, predefined_acl: nil, projection: nil, user_project: nil, fields: nil, quota_user: nil, user_ip: nil, upload_source: nil, content_type: nil, options: nil, &block) + def insert_object(bucket, object_object = nil, content_encoding: nil, if_generation_match: nil, if_generation_not_match: nil, if_metageneration_match: nil, if_metageneration_not_match: nil, kms_key_name: nil, name: nil, predefined_acl: nil, projection: nil, user_project: nil, fields: nil, quota_user: nil, user_ip: nil, upload_source: nil, content_type: nil, upload_id: nil, options: nil, &block) if upload_source.nil? command = make_simple_command(:post, 'b/{bucket}/o', options) @@ -2878,6 +2878,7 @@ def insert_object(bucket, object_object = nil, content_encoding: nil, if_generat command = make_storage_upload_command(:post, 'b/{bucket}/o', options) command.upload_source = upload_source command.upload_content_type = content_type + command.upload_id = upload_id end command.request_representation = Google::Apis::StorageV1::Object::Representation command.request_object = object_object From e891f3d31fdba9f887b7564690262518581dc517 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 10 Mar 2025 20:13:30 +0000 Subject: [PATCH 11/26] updating --- .../lib/google/apis/storage_v1/service.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb index 0d2d5b834ed..29dd3c4f4ab 100644 --- a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb +++ b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb @@ -2858,6 +2858,8 @@ def get_object_iam_policy(bucket, object, generation: nil, user_project: nil, fi # IO stream or filename containing content to upload # @param [String] content_type # Content type of the uploaded content. + # @param @param [IO, String] upload_id + # Unique upload Id for ongoing resumable upload # @param [Google::Apis::RequestOptions] options # Request-specific options # From 0cc8b2b9d5a7f9e6f7a15dcbdd2d745c3732ce7d Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 13 Mar 2025 10:04:04 +0000 Subject: [PATCH 12/26] code refactoring --- .../lib/google/apis/storage_v1/service.rb | 14 +------- .../lib/google/apis/core/storage_upload.rb | 32 +++++++++---------- google-apis-core/lib/google/apis/options.rb | 6 +++- .../google/apis/core/storage_upload_spec.rb | 10 +++--- 4 files changed, 27 insertions(+), 35 deletions(-) diff --git a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb index 29dd3c4f4ab..148258fed2b 100644 --- a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb +++ b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb @@ -2858,8 +2858,6 @@ def get_object_iam_policy(bucket, object, generation: nil, user_project: nil, fi # IO stream or filename containing content to upload # @param [String] content_type # Content type of the uploaded content. - # @param @param [IO, String] upload_id - # Unique upload Id for ongoing resumable upload # @param [Google::Apis::RequestOptions] options # Request-specific options # @@ -2872,7 +2870,7 @@ def get_object_iam_policy(bucket, object, generation: nil, user_project: nil, fi # @raise [Google::Apis::ServerError] An error occurred on the server and the request can be retried # @raise [Google::Apis::ClientError] The request is invalid and should not be retried without modification # @raise [Google::Apis::AuthorizationError] Authorization is required - def insert_object(bucket, object_object = nil, content_encoding: nil, if_generation_match: nil, if_generation_not_match: nil, if_metageneration_match: nil, if_metageneration_not_match: nil, kms_key_name: nil, name: nil, predefined_acl: nil, projection: nil, user_project: nil, fields: nil, quota_user: nil, user_ip: nil, upload_source: nil, content_type: nil, upload_id: nil, options: nil, &block) + def insert_object(bucket, object_object = nil, content_encoding: nil, if_generation_match: nil, if_generation_not_match: nil, if_metageneration_match: nil, if_metageneration_not_match: nil, kms_key_name: nil, name: nil, predefined_acl: nil, projection: nil, user_project: nil, fields: nil, quota_user: nil, user_ip: nil, upload_source: nil, content_type: nil, options: nil, &block) if upload_source.nil? command = make_simple_command(:post, 'b/{bucket}/o', options) @@ -2880,7 +2878,6 @@ def insert_object(bucket, object_object = nil, content_encoding: nil, if_generat command = make_storage_upload_command(:post, 'b/{bucket}/o', options) command.upload_source = upload_source command.upload_content_type = content_type - command.upload_id = upload_id end command.request_representation = Google::Apis::StorageV1::Object::Representation command.request_object = object_object @@ -3852,15 +3849,6 @@ def delete_project_hmac_key(project_id, access_id, user_project: nil, fields: ni execute_or_queue_command(command, &block) end - # Deletes Resumable upload - def delete_ongoing_resumable_upload(bucket, upload_source, upload_id, options: nil) - command = make_storage_upload_command(:post, 'b/{bucket}/o', options) - command.upload_source = upload_source - command.upload_id = upload_id - command.params['bucket'] = bucket unless bucket.nil? - command.delete_upload = true - execute_or_queue_command(command) - end # Retrieves an HMAC key's metadata # @param [String] project_id diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 65b2fabc0be..536de6e450a 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -36,14 +36,6 @@ class StorageUploadCommand < ApiCommand # @return [String, File, #read] attr_accessor :upload_source - # Unique upload_id of a resumable upload - # @return [String] - attr_accessor :upload_id - - # Boolean Value to specify is a resumable upload is to be deleted or not - # @return [Boolean] - attr_accessor :delete_upload - # Content type of the upload material # @return [String] attr_accessor :upload_content_type @@ -102,15 +94,17 @@ def release! def execute(client) prepare! opencensus_begin_span + upload_id = options.upload_id + delete_upload = options.delete_upload @upload_chunk_size = options.upload_chunk_size if upload_id.nil? do_retry :initiate_resumable_upload, client elsif delete_upload && !upload_id.nil? - make_resumabple_upload_url - cancel_resumable_upload(client) + make_resumable_upload_url upload_id + do_retry :cancel_resumable_upload, client else - make_resumabple_upload_url - reinitiate_resumable_upload(client) + make_resumable_upload_url upload_id + do_retry :reinitiate_resumable_upload,client end while @upload_incomplete @@ -149,7 +143,6 @@ def initiate_resumable_upload(client) end # Reinitiating resumable upload - def reinitiate_resumable_upload(client) logger.debug { sprintf('Restarting resumable upload command to %s', url) } check_resumable_upload_status client @@ -158,13 +151,15 @@ def reinitiate_resumable_upload(client) error(e, rethrow: true) end - def make_resumabple_upload_url + # Making resumable upload url from upload_id + def make_resumable_upload_url(upload_id) query_params = query.dup query_params['uploadType'] = RESUMABLE query_params['upload_id'] = upload_id resumable_upload_params = query_params.map { |key, value| "#{key}=#{value}" }.join('&') @upload_url = "#{url}&#{resumable_upload_params}" end + # Send the actual content # # @param [HTTPClient] client @@ -192,6 +187,9 @@ def send_upload_command(client) @offset += current_chunk_size if @upload_incomplete success(result) rescue => e + logger.warn { + "error occured please use uploadId-#{response.headers['X-GUploader-UploadID']} to resume your upload" + } unless response.nil? upload_io.pos = @offset error(e, rethrow: true) end @@ -221,6 +219,7 @@ def check_resumable_upload_status(client) request_header[CONTENT_LENGTH_HEADER] = '0' # Initiating call response = client.put(@upload_url, header: request_header, follow_redirect: true) + case response.code.to_i when 308 if response.headers['Range'] @@ -231,8 +230,8 @@ def check_resumable_upload_status(client) puts 'No bytes uploaded yet.' end @upload_incomplete = true - when 499 - # Upload in canceled + when 400..499 + # Upload is canceled @upload_incomplete = false when 200, 201 # Upload is complete. @@ -250,6 +249,7 @@ def cancel_resumable_upload(client) request_header[CONTENT_LENGTH_HEADER] = '0' # Initiating call response = client.delete(@upload_url, header: request_header, follow_redirect: true) + case response.code.to_i when 499 @close_io_on_finish = true diff --git a/google-apis-core/lib/google/apis/options.rb b/google-apis-core/lib/google/apis/options.rb index f1ac437c9ed..3df2ea83c51 100644 --- a/google-apis-core/lib/google/apis/options.rb +++ b/google-apis-core/lib/google/apis/options.rb @@ -41,7 +41,9 @@ module Apis :quota_project, :query, :add_invocation_id_header, - :upload_chunk_size + :upload_chunk_size, + :upload_id, + :delete_upload ) # General client options @@ -140,5 +142,7 @@ def merge(options) RequestOptions.default.quota_project = nil RequestOptions.default.add_invocation_id_header = false RequestOptions.default.upload_chunk_size = 100 * 1024 * 1024 # 100 MB + RequestOptions.default.upload_id = nil + RequestOptions.default.delete_upload = false end end diff --git a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb index 6eda8b216f1..26364da4f38 100644 --- a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb +++ b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb @@ -156,7 +156,7 @@ it 'should restart a resumable upload' do command.options.upload_chunk_size = 11 - command.upload_id = upload_id + command.options.upload_id = upload_id command.execute(client) expect(a_request(:put, upload_url) .with(body: 'Hello world')).to have_been_made @@ -187,7 +187,7 @@ it 'should not restart a upload' do command.options.upload_chunk_size = 11 - command.upload_id = upload_id + command.options.upload_id = upload_id command.execute(client) expect(a_request(:put, upload_url) .with(body: 'Hello world')).to have_not_been_made @@ -219,15 +219,15 @@ it 'should cancel a resumable upload' do command.options.upload_chunk_size = 11 - command.upload_id = upload_id - command.delete_upload = true + command.options.upload_id = upload_id + command.options.delete_upload = true command.execute(client) expect(a_request(:delete, upload_url)).to have_been_made end it 'should not call resumable upload when upload is cancelled' do command.options.upload_chunk_size = 11 - command.upload_id = upload_id + command.options.upload_id = upload_id command.execute(client) expect(a_request(:put, upload_url) .with(body: 'Hello world')).to have_not_been_made From de2358342dd631223f8756fced6eef777cb291d6 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 13 Mar 2025 11:26:34 +0000 Subject: [PATCH 13/26] fixing comment --- google-apis-core/lib/google/apis/core/storage_upload.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 536de6e450a..f1c52bf1fcd 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -225,7 +225,7 @@ def check_resumable_upload_status(client) if response.headers['Range'] range = response.headers['Range'] @offset = range ? range.split('-').last.to_i + 1 : 0 - puts "Upload is incomplete. Bytes uploaded: #{response.headers['Range']}" + puts "Upload is incomplete. Bytes uploaded so far: #{response.headers['Range']}" else puts 'No bytes uploaded yet.' end From 872802b26029e96d1e1aeb54b1d431b0f0670139 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 21 Apr 2025 12:44:17 +0000 Subject: [PATCH 14/26] wip - implementation 2 --- .../lib/google/apis/core/base_service.rb | 17 +++++++++++++++++ .../lib/google/apis/core/storage_upload.rb | 14 +++++++++----- google-apis-core/lib/google/apis/options.rb | 6 +----- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/base_service.rb b/google-apis-core/lib/google/apis/core/base_service.rb index 22ee6553ce4..25f013cbd01 100644 --- a/google-apis-core/lib/google/apis/core/base_service.rb +++ b/google-apis-core/lib/google/apis/core/base_service.rb @@ -350,6 +350,23 @@ def verify_universe_domain! true end + # Restarts Or Deletes An Ongoing Resumable upload + # @param [String] bucket + # Name of the bucket where the upload is being performed. + # @param [IO, String] upload_source + # IO stream or filename containing content to upload + # @param [IO, String] upload_id + # unique id generated for an ongoing upload + + def restart_delete_ongoing_resumable_upload(bucket, upload_source, upload_id, options: nil) + command = make_storage_upload_command(:post, 'b/{bucket}/o', options) + command.upload_source = upload_source + command.upload_id = upload_id + command.params['bucket'] = bucket unless bucket.nil? + command.delete_upload = options[:delete_upload ] unless options[:delete_upload].nil? + execute_or_queue_command(command) + end + protected # Create a new upload command. diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index f1c52bf1fcd..25b9726ae67 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -48,6 +48,14 @@ class StorageUploadCommand < ApiCommand # @return [Integer] attr_accessor :upload_chunk_size + # Unique upload_id of a resumable upload + # @return [String] + attr_accessor :upload_id + + # Boolean Value to specify is a resumable upload is to be deleted or not + # @return [Boolean] + attr_accessor :delete_upload + # Ensure the content is readable and wrapped in an IO instance. # # @return [void] @@ -94,8 +102,6 @@ def release! def execute(client) prepare! opencensus_begin_span - upload_id = options.upload_id - delete_upload = options.delete_upload @upload_chunk_size = options.upload_chunk_size if upload_id.nil? do_retry :initiate_resumable_upload, client @@ -171,7 +177,6 @@ def send_upload_command(client) remaining_content_size = upload_io.size - @offset current_chunk_size = get_current_chunk_size remaining_content_size - request_header = header.dup request_header[CONTENT_RANGE_HEADER] = get_content_range_header current_chunk_size request_header[CONTENT_LENGTH_HEADER] = current_chunk_size @@ -249,9 +254,8 @@ def cancel_resumable_upload(client) request_header[CONTENT_LENGTH_HEADER] = '0' # Initiating call response = client.delete(@upload_url, header: request_header, follow_redirect: true) - case response.code.to_i - when 499 + when 400..499 @close_io_on_finish = true @upload_incomplete = false else diff --git a/google-apis-core/lib/google/apis/options.rb b/google-apis-core/lib/google/apis/options.rb index 3df2ea83c51..f1ac437c9ed 100644 --- a/google-apis-core/lib/google/apis/options.rb +++ b/google-apis-core/lib/google/apis/options.rb @@ -41,9 +41,7 @@ module Apis :quota_project, :query, :add_invocation_id_header, - :upload_chunk_size, - :upload_id, - :delete_upload + :upload_chunk_size ) # General client options @@ -142,7 +140,5 @@ def merge(options) RequestOptions.default.quota_project = nil RequestOptions.default.add_invocation_id_header = false RequestOptions.default.upload_chunk_size = 100 * 1024 * 1024 # 100 MB - RequestOptions.default.upload_id = nil - RequestOptions.default.delete_upload = false end end From b92dd7f5b6bc8cf02c63c2ded61df1889b68baeb Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Thu, 24 Apr 2025 11:04:55 +0000 Subject: [PATCH 15/26] new changes working --- .../lib/google/apis/core/base_service.rb | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/base_service.rb b/google-apis-core/lib/google/apis/core/base_service.rb index 25f013cbd01..68b0b54d204 100644 --- a/google-apis-core/lib/google/apis/core/base_service.rb +++ b/google-apis-core/lib/google/apis/core/base_service.rb @@ -350,7 +350,7 @@ def verify_universe_domain! true end - # Restarts Or Deletes An Ongoing Resumable upload + # Restarts An Ongoing Resumable upload # @param [String] bucket # Name of the bucket where the upload is being performed. # @param [IO, String] upload_source @@ -358,7 +358,24 @@ def verify_universe_domain! # @param [IO, String] upload_id # unique id generated for an ongoing upload - def restart_delete_ongoing_resumable_upload(bucket, upload_source, upload_id, options: nil) + def restart_resumable_upload(bucket, upload_source, upload_id, options: nil) + command = make_storage_upload_command(:post, 'b/{bucket}/o', options) + command.upload_source = upload_source + command.upload_id = upload_id + command.params['bucket'] = bucket unless bucket.nil? + command.delete_upload = options[:delete_upload ] unless options[:delete_upload].nil? + execute_or_queue_command(command) + end + + # Deletes An Ongoing Resumable upload + # @param [String] bucket + # Name of the bucket where the upload is being performed. + # @param [IO, String] upload_source + # IO stream or filename containing content to upload + # @param [IO, String] upload_id + # unique id generated for an ongoing upload + + def delete_resumable_upload(bucket, upload_source, upload_id, options: nil) command = make_storage_upload_command(:post, 'b/{bucket}/o', options) command.upload_source = upload_source command.upload_id = upload_id From 6fbf86aa39634d4aff9faac3b72c6a003d2c2c7e Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Tue, 29 Apr 2025 08:12:20 +0000 Subject: [PATCH 16/26] rewriting unit test cases --- .../lib/google/apis/core/base_service.rb | 1 - .../spec/google/apis/core/service_spec.rb | 89 +++++++++++++++++++ .../google/apis/core/storage_upload_spec.rb | 12 +-- 3 files changed, 96 insertions(+), 6 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/base_service.rb b/google-apis-core/lib/google/apis/core/base_service.rb index 68b0b54d204..025007038a7 100644 --- a/google-apis-core/lib/google/apis/core/base_service.rb +++ b/google-apis-core/lib/google/apis/core/base_service.rb @@ -363,7 +363,6 @@ def restart_resumable_upload(bucket, upload_source, upload_id, options: nil) command.upload_source = upload_source command.upload_id = upload_id command.params['bucket'] = bucket unless bucket.nil? - command.delete_upload = options[:delete_upload ] unless options[:delete_upload].nil? execute_or_queue_command(command) end diff --git a/google-apis-core/spec/google/apis/core/service_spec.rb b/google-apis-core/spec/google/apis/core/service_spec.rb index 53167a12700..5184dcb48f2 100644 --- a/google-apis-core/spec/google/apis/core/service_spec.rb +++ b/google-apis-core/spec/google/apis/core/service_spec.rb @@ -225,6 +225,95 @@ include_examples 'with options' end + context 'when making restart resumable upload' do + let(:bucket_name) { 'test_bucket' } + let(:file) { StringIO.new('Hello world' * 3) } + + let(:upload_id) { 'foo' } + let(:command) do + service.send( + :restart_resumable_upload, + bucket_name, file, upload_id, + options: { upload_chunk_size: 11} + ) + end + let(:upload_url) { "https://www.googleapis.com/upload/b/#{bucket_name}/o?uploadType=resumable&upload_id=#{upload_id}"} + context 'should complete the upload' do + before(:example) do + stub_request(:put, upload_url) + .with( + headers: { + 'Content-Length' => '0', + 'Content-Range' => 'bytes */33' + } + ) + .to_return( + status: [308, 'Resume Incomplete'], + headers: { 'Range' => 'bytes=0-21' } + ) + end + + before(:example) do + stub_request(:put, upload_url) + .with(headers: { 'Content-Range' => 'bytes 22-32/33' }) + .to_return(body: %(OK)) + end + + it 'should send request to upload url multiple times' do + command + expect(a_request(:put, upload_url)).to have_been_made.twice + end + end + context 'not restart resumable upload if upload is completed' do + before(:example) do + stub_request(:put, upload_url) + .with( + headers: { + 'Content-Length' => '0', + 'Content-Range' => 'bytes */33' + } + ) + .to_return(status: 200, headers: { 'Range' => 'bytes=0-32' }) + end + + before(:example) do + stub_request(:put, upload_url) + .with(headers: { 'Content-Range' => 'bytes */33' }) + .to_return(status: 200) + end + + it 'should not restart a upload' do + command + expect(a_request(:put, upload_url)).to have_been_made + end + end + end + + context 'delete resumable upload with upload_id' do + let(:bucket_name) { 'test_bucket' } + let(:file) { StringIO.new('Hello world' * 3) } + let(:upload_id) { 'foo' } + let(:command) do + service.send( + :delete_resumable_upload, + bucket_name, file, upload_id, + options: { upload_chunk_size: 11, delete_upload: true } + ) + end + + let(:upload_url) { "https://www.googleapis.com/upload/b/#{bucket_name}/o?uploadType=resumable&upload_id=#{upload_id}" } + before(:example) do + stub_request(:delete, upload_url) + .with(headers: { 'Content-Length' => '0' }) + .to_return(status: [499]) + end + + it 'should cancel a resumable upload' do + command + expect(a_request(:delete, upload_url)).to have_been_made + end + end + context 'with batch' do before(:example) do response = < Date: Wed, 30 Apr 2025 11:05:00 +0000 Subject: [PATCH 17/26] adding test cases --- google-apis-core/lib/google/apis/core/storage_upload.rb | 9 +++++---- google-apis-core/spec/google/apis/core/service_spec.rb | 1 + .../spec/google/apis/core/storage_upload_spec.rb | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 25b9726ae67..66d4ab9aeae 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -104,13 +104,13 @@ def execute(client) opencensus_begin_span @upload_chunk_size = options.upload_chunk_size if upload_id.nil? - do_retry :initiate_resumable_upload, client + res = do_retry :initiate_resumable_upload, client elsif delete_upload && !upload_id.nil? make_resumable_upload_url upload_id - do_retry :cancel_resumable_upload, client + res = do_retry :cancel_resumable_upload, client else make_resumable_upload_url upload_id - do_retry :reinitiate_resumable_upload,client + res = do_retry :reinitiate_resumable_upload, client end while @upload_incomplete @@ -174,7 +174,6 @@ def make_resumable_upload_url(upload_id) # @raise [Google::Apis::ServerError] Unable to send the request def send_upload_command(client) logger.debug { sprintf('Sending upload command to %s', @upload_url) } - remaining_content_size = upload_io.size - @offset current_chunk_size = get_current_chunk_size remaining_content_size request_header = header.dup @@ -258,8 +257,10 @@ def cancel_resumable_upload(client) when 400..499 @close_io_on_finish = true @upload_incomplete = false + true # method returns true if upload is sucessfully cancelled else puts "Failed to cancel upload session. Response: #{response.code} - #{response.body}" + false # method returns false if upload is not cancelled end end diff --git a/google-apis-core/spec/google/apis/core/service_spec.rb b/google-apis-core/spec/google/apis/core/service_spec.rb index 5184dcb48f2..ff8ccae40f1 100644 --- a/google-apis-core/spec/google/apis/core/service_spec.rb +++ b/google-apis-core/spec/google/apis/core/service_spec.rb @@ -311,6 +311,7 @@ it 'should cancel a resumable upload' do command expect(a_request(:delete, upload_url)).to have_been_made + expect(command).to be_truthy end end diff --git a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb index bd311b1b9a0..f6fc4e7b786 100644 --- a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb +++ b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb @@ -165,7 +165,7 @@ context('should not restart resumable upload if upload is completed') do let(:file) { StringIO.new('Hello world' * 3) } - let(:upload_id) {"TestId"} + let(:upload_id) { 'TestId' } let(:upload_url) { "https://www.googleapis.com/zoo/animals?uploadType=resumable&upload_id=#{upload_id}" } before(:example) do @@ -224,6 +224,7 @@ command.delete_upload = true command.execute(client) expect(a_request(:delete, upload_url)).to have_been_made + expect(command).to be_truthy end it 'should not call resumable upload when upload is cancelled' do From 8d956f231b28aed3ab2e79fb2db5608c9c8092df Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Wed, 30 Apr 2025 11:21:10 +0000 Subject: [PATCH 18/26] fix typo --- .../lib/google/apis/storage_v1/service.rb | 1 - google-apis-core/lib/google/apis/core/storage_upload.rb | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb index 148258fed2b..afb9775e301 100644 --- a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb +++ b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb @@ -3849,7 +3849,6 @@ def delete_project_hmac_key(project_id, access_id, user_project: nil, fields: ni execute_or_queue_command(command, &block) end - # Retrieves an HMAC key's metadata # @param [String] project_id # Project ID owning the service account of the requested key. diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 66d4ab9aeae..c4365e74903 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -18,6 +18,7 @@ require 'stringio' require 'tempfile' require 'mini_mime' + module Google module Apis module Core @@ -114,7 +115,6 @@ def execute(client) end while @upload_incomplete - res = do_retry :send_upload_command, client end res @@ -141,7 +141,6 @@ def initiate_resumable_upload(client) body: body, header: request_header, follow_redirect: true) - result = process_response(response.status_code, response.header, response.body) success(result) rescue => e @@ -174,8 +173,10 @@ def make_resumable_upload_url(upload_id) # @raise [Google::Apis::ServerError] Unable to send the request def send_upload_command(client) logger.debug { sprintf('Sending upload command to %s', @upload_url) } + remaining_content_size = upload_io.size - @offset current_chunk_size = get_current_chunk_size remaining_content_size + request_header = header.dup request_header[CONTENT_RANGE_HEADER] = get_content_range_header current_chunk_size request_header[CONTENT_LENGTH_HEADER] = current_chunk_size @@ -185,7 +186,9 @@ def send_upload_command(client) else StringIO.new(upload_io.read(current_chunk_size)) end + response = client.put(@upload_url, body: chunk_body, header: request_header, follow_redirect: true) + result = process_response(response.status_code, response.header, response.body) @upload_incomplete = false if response.status_code.eql? OK_STATUS @offset += current_chunk_size if @upload_incomplete From a1df39055e36a292e0f58102f6e50d51ffbf579f Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Wed, 30 Apr 2025 11:23:16 +0000 Subject: [PATCH 19/26] fix typo --- google-apis-core/lib/google/apis/core/storage_upload.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index c4365e74903..66abc2e9d15 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -173,10 +173,10 @@ def make_resumable_upload_url(upload_id) # @raise [Google::Apis::ServerError] Unable to send the request def send_upload_command(client) logger.debug { sprintf('Sending upload command to %s', @upload_url) } - + remaining_content_size = upload_io.size - @offset current_chunk_size = get_current_chunk_size remaining_content_size - + request_header = header.dup request_header[CONTENT_RANGE_HEADER] = get_content_range_header current_chunk_size request_header[CONTENT_LENGTH_HEADER] = current_chunk_size @@ -188,7 +188,7 @@ def send_upload_command(client) end response = client.put(@upload_url, body: chunk_body, header: request_header, follow_redirect: true) - + result = process_response(response.status_code, response.header, response.body) @upload_incomplete = false if response.status_code.eql? OK_STATUS @offset += current_chunk_size if @upload_incomplete From ebb5d6cd4aca72b1f3b4d3f092ef64805ea7e59d Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Wed, 30 Apr 2025 11:39:17 +0000 Subject: [PATCH 20/26] removing unwanted changes --- .../lib/google/apis/storage_v1/service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb index afb9775e301..6cee008d917 100644 --- a/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb +++ b/generated/google-apis-storage_v1/lib/google/apis/storage_v1/service.rb @@ -3848,7 +3848,7 @@ def delete_project_hmac_key(project_id, access_id, user_project: nil, fields: ni command.query['userIp'] = user_ip unless user_ip.nil? execute_or_queue_command(command, &block) end - + # Retrieves an HMAC key's metadata # @param [String] project_id # Project ID owning the service account of the requested key. From 6fc5733b1214626935d1718be9ad089234c5a0f6 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Wed, 7 May 2025 05:47:49 +0000 Subject: [PATCH 21/26] fixing PR comments --- .../lib/google/apis/core/base_service.rb | 9 ++- .../lib/google/apis/core/storage_upload.rb | 62 ++++++++++--------- .../spec/google/apis/core/service_spec.rb | 3 +- 3 files changed, 38 insertions(+), 36 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/base_service.rb b/google-apis-core/lib/google/apis/core/base_service.rb index 025007038a7..e1adccef86d 100644 --- a/google-apis-core/lib/google/apis/core/base_service.rb +++ b/google-apis-core/lib/google/apis/core/base_service.rb @@ -350,7 +350,7 @@ def verify_universe_domain! true end - # Restarts An Ongoing Resumable upload + # Restarts An interrupted Resumable upload # @param [String] bucket # Name of the bucket where the upload is being performed. # @param [IO, String] upload_source @@ -366,7 +366,7 @@ def restart_resumable_upload(bucket, upload_source, upload_id, options: nil) execute_or_queue_command(command) end - # Deletes An Ongoing Resumable upload + # Deletes An interrupted Resumable upload # @param [String] bucket # Name of the bucket where the upload is being performed. # @param [IO, String] upload_source @@ -374,12 +374,11 @@ def restart_resumable_upload(bucket, upload_source, upload_id, options: nil) # @param [IO, String] upload_id # unique id generated for an ongoing upload - def delete_resumable_upload(bucket, upload_source, upload_id, options: nil) + def delete_resumable_upload(bucket, upload_id, options: nil) command = make_storage_upload_command(:post, 'b/{bucket}/o', options) - command.upload_source = upload_source command.upload_id = upload_id command.params['bucket'] = bucket unless bucket.nil? - command.delete_upload = options[:delete_upload ] unless options[:delete_upload].nil? + command.delete_upload = options[:delete_upload] unless options[:delete_upload].nil? execute_or_queue_command(command) end diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 66abc2e9d15..9e0f013111e 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -69,7 +69,6 @@ def prepare! # asserting that it already has a body. Form encoding is never used # by upload requests. self.body = '' unless self.body - super if streamable?(upload_source) self.upload_io = upload_source @@ -81,6 +80,8 @@ def prepare! self.upload_content_type = type&.content_type end @close_io_on_finish = true + elsif !upload_id.nil? && delete_upload + @close_io_on_finish = false else fail Google::Apis::ClientError, 'Invalid upload source' end @@ -88,7 +89,7 @@ def prepare! # Close IO stream when command done. Only closes the stream if it was opened by the command. def release! - upload_io.close if @close_io_on_finish + upload_io.close if @close_io_on_finish && !upload_io.nil? end # Execute the command, retrying as necessary @@ -150,10 +151,8 @@ def initiate_resumable_upload(client) # Reinitiating resumable upload def reinitiate_resumable_upload(client) logger.debug { sprintf('Restarting resumable upload command to %s', url) } - check_resumable_upload_status client + check_resumable_upload client upload_io.pos = @offset - rescue => e - error(e, rethrow: true) end # Making resumable upload url from upload_id @@ -219,22 +218,45 @@ def process_response(status, header, body) super(status, header, body) end - def check_resumable_upload_status(client) + def check_resumable_upload(client) # Setting up request header request_header = header.dup request_header[CONTENT_RANGE_HEADER] = "bytes */#{upload_io.size}" request_header[CONTENT_LENGTH_HEADER] = '0' # Initiating call response = client.put(@upload_url, header: request_header, follow_redirect: true) + handle_resumable_upload_http_response_codes(response) + end + + # Cancel resumable upload + def cancel_resumable_upload(client) + # Setting up request header + request_header = header.dup + request_header[CONTENT_LENGTH_HEADER] = '0' + # Initiating call + response = client.delete(@upload_url, header: request_header, follow_redirect: true) + handle_resumable_upload_http_response_codes(response) + + if !@upload_incomplete && (400..499).include?(response.code.to_i) + @close_io_on_finish = true + true # method returns true if upload is successfully cancelled + else + logger.debug { sprintf("Failed to cancel upload session. Response: #{response.code} - #{response.body}") } + end + + end - case response.code.to_i + def handle_resumable_upload_http_response_codes(response) + code = response.code.to_i + + case code when 308 if response.headers['Range'] range = response.headers['Range'] - @offset = range ? range.split('-').last.to_i + 1 : 0 - puts "Upload is incomplete. Bytes uploaded so far: #{response.headers['Range']}" + @offset = range.split('-').last.to_i + 1 + logger.debug { sprintf("Upload is incomplete. Bytes uploaded so far: #{response.headers['Range']}") } else - puts 'No bytes uploaded yet.' + logger.debug { sprintf('No bytes uploaded yet.') } end @upload_incomplete = true when 400..499 @@ -244,29 +266,11 @@ def check_resumable_upload_status(client) # Upload is complete. @upload_incomplete = false else - puts "Unexpected response: #{response.code} - #{response.body}" + logger.debug { sprintf("Unexpected response: #{response.code} - #{response.body}") } @upload_incomplete = true end end - # Cancel resumable upload - def cancel_resumable_upload(client) - # Setting up request header - request_header = header.dup - request_header[CONTENT_LENGTH_HEADER] = '0' - # Initiating call - response = client.delete(@upload_url, header: request_header, follow_redirect: true) - case response.code.to_i - when 400..499 - @close_io_on_finish = true - @upload_incomplete = false - true # method returns true if upload is sucessfully cancelled - else - puts "Failed to cancel upload session. Response: #{response.code} - #{response.body}" - false # method returns false if upload is not cancelled - end - end - def streamable?(upload_source) upload_source.is_a?(IO) || upload_source.is_a?(StringIO) || upload_source.is_a?(Tempfile) end diff --git a/google-apis-core/spec/google/apis/core/service_spec.rb b/google-apis-core/spec/google/apis/core/service_spec.rb index ff8ccae40f1..0710d30a0d8 100644 --- a/google-apis-core/spec/google/apis/core/service_spec.rb +++ b/google-apis-core/spec/google/apis/core/service_spec.rb @@ -291,12 +291,11 @@ context 'delete resumable upload with upload_id' do let(:bucket_name) { 'test_bucket' } - let(:file) { StringIO.new('Hello world' * 3) } let(:upload_id) { 'foo' } let(:command) do service.send( :delete_resumable_upload, - bucket_name, file, upload_id, + bucket_name, upload_id, options: { upload_chunk_size: 11, delete_upload: true } ) end From 422dc99694cc37bcdf51d496d53776f5c975abae Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Tue, 13 May 2025 11:08:20 +0530 Subject: [PATCH 22/26] Update google-apis-core/lib/google/apis/core/storage_upload.rb Co-authored-by: Neha Bajaj --- google-apis-core/lib/google/apis/core/storage_upload.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 9e0f013111e..a21d93fb58d 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -156,7 +156,7 @@ def reinitiate_resumable_upload(client) end # Making resumable upload url from upload_id - def make_resumable_upload_url(upload_id) + def construct_resumable_upload_url(upload_id) query_params = query.dup query_params['uploadType'] = RESUMABLE query_params['upload_id'] = upload_id From 238a1e25f867aac22134656adb5df03d13cf889c Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Tue, 13 May 2025 08:27:13 +0000 Subject: [PATCH 23/26] :q --- google-apis-core/lib/google/apis/core/storage_upload.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index a21d93fb58d..47e6d9d0c0b 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -108,10 +108,10 @@ def execute(client) if upload_id.nil? res = do_retry :initiate_resumable_upload, client elsif delete_upload && !upload_id.nil? - make_resumable_upload_url upload_id + construct_resumable_upload_url upload_id res = do_retry :cancel_resumable_upload, client else - make_resumable_upload_url upload_id + construct_resumable_upload_url upload_id res = do_retry :reinitiate_resumable_upload, client end From 89e281d91808853f352ba3568751a7082bceb449 Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Tue, 13 May 2025 20:31:49 +0000 Subject: [PATCH 24/26] updating requests --- google-apis-core/lib/google/apis/core/base_service.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/base_service.rb b/google-apis-core/lib/google/apis/core/base_service.rb index e1adccef86d..168db6a3e2e 100644 --- a/google-apis-core/lib/google/apis/core/base_service.rb +++ b/google-apis-core/lib/google/apis/core/base_service.rb @@ -359,7 +359,7 @@ def verify_universe_domain! # unique id generated for an ongoing upload def restart_resumable_upload(bucket, upload_source, upload_id, options: nil) - command = make_storage_upload_command(:post, 'b/{bucket}/o', options) + command = make_storage_upload_command(:put, 'b/{bucket}/o', options) command.upload_source = upload_source command.upload_id = upload_id command.params['bucket'] = bucket unless bucket.nil? @@ -375,7 +375,7 @@ def restart_resumable_upload(bucket, upload_source, upload_id, options: nil) # unique id generated for an ongoing upload def delete_resumable_upload(bucket, upload_id, options: nil) - command = make_storage_upload_command(:post, 'b/{bucket}/o', options) + command = make_storage_upload_command(:delete, 'b/{bucket}/o', options) command.upload_id = upload_id command.params['bucket'] = bucket unless bucket.nil? command.delete_upload = options[:delete_upload] unless options[:delete_upload].nil? From 53b8d502c58c449e19e1e1109fbc169b0169a65b Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 19 May 2025 11:32:43 +0530 Subject: [PATCH 25/26] Update google-apis-core/lib/google/apis/core/storage_upload.rb Co-authored-by: Neha Bajaj --- google-apis-core/lib/google/apis/core/storage_upload.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index 47e6d9d0c0b..365beaf9e5f 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -254,7 +254,7 @@ def handle_resumable_upload_http_response_codes(response) if response.headers['Range'] range = response.headers['Range'] @offset = range.split('-').last.to_i + 1 - logger.debug { sprintf("Upload is incomplete. Bytes uploaded so far: #{response.headers['Range']}") } + logger.debug { sprintf("Upload is incomplete. Bytes uploaded so far: #{range}") } else logger.debug { sprintf('No bytes uploaded yet.') } end From 0a9bb3f9562ab8a5a77f008198891c81f5f0c46f Mon Sep 17 00:00:00 2001 From: Shubhangi Singh Date: Mon, 19 May 2025 16:42:07 +0530 Subject: [PATCH 26/26] Update base_service.rb --- google-apis-core/lib/google/apis/core/base_service.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/google-apis-core/lib/google/apis/core/base_service.rb b/google-apis-core/lib/google/apis/core/base_service.rb index 168db6a3e2e..c19a27fd02b 100644 --- a/google-apis-core/lib/google/apis/core/base_service.rb +++ b/google-apis-core/lib/google/apis/core/base_service.rb @@ -369,8 +369,6 @@ def restart_resumable_upload(bucket, upload_source, upload_id, options: nil) # Deletes An interrupted Resumable upload # @param [String] bucket # Name of the bucket where the upload is being performed. - # @param [IO, String] upload_source - # IO stream or filename containing content to upload # @param [IO, String] upload_id # unique id generated for an ongoing upload