diff --git a/google-apis-core/lib/google/apis/core/storage_upload.rb b/google-apis-core/lib/google/apis/core/storage_upload.rb index da31878d3a3..923826f3563 100644 --- a/google-apis-core/lib/google/apis/core/storage_upload.rb +++ b/google-apis-core/lib/google/apis/core/storage_upload.rb @@ -176,6 +176,10 @@ def send_upload_command(client) request_header = header.dup request_header[CONTENT_RANGE_HEADER] = get_content_range_header current_chunk_size request_header[CONTENT_LENGTH_HEADER] = current_chunk_size.to_s + last_chunk = remaining_content_size <= current_chunk_size + formatted_string = formatted_checksum_header + request_header['X-Goog-Hash'] = formatted_string if (last_chunk && !formatted_string.empty?) + chunk_body = if @upload_chunk_size == 0 upload_io @@ -191,7 +195,7 @@ def send_upload_command(client) success(result) rescue => e logger.warn { - "error occured please use uploadId-#{response.headers['X-GUploader-UploadID']} to resume your upload" + "error occurred please use uploadId-#{response.headers['X-GUploader-UploadID']} to resume your upload" } unless response.nil? upload_io.pos = @offset error(e, rethrow: true) @@ -290,6 +294,30 @@ def get_content_range_header current_chunk_size end sprintf('bytes %s/%d', numerator, upload_io.size) end + + # Generates a formatted checksum header string from the request body. + # + # Parses the body as JSON and extracts checksum values for the keys "crc32c", "md5Hash", and "md5". + # The "md5Hash" key is renamed to "md5" in the output. + # Returns a comma-separated string in the format "key=value" for each present checksum. + # + # @example + # If the body contains: + # { "md5Hash": "1B2M2Y8AsgTpgAmY7PhCfg==", + # "crc32c": "AAAAAA==" } + # The method returns: + # "crc32c=AAAAAA==,md5=1B2M2Y8AsgTpgAmY7PhCfg==" + # @return [String] the formatted checksum header, or an empty string if no relevant keys are present + def formatted_checksum_header + hash_data = body.to_s.empty? ? {} : JSON.parse(body) + target_keys = ["crc32c", "md5Hash", "md5"] + selected_keys = hash_data.slice(*target_keys) + formatted_string = selected_keys.map do |key, value| + output_key = (key == "md5Hash") ? "md5" : key + "#{output_key}=#{value}" + end.join(',') + formatted_string + end end end end diff --git a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb index 7435f9cafe8..550ba987d3b 100644 --- a/google-apis-core/spec/google/apis/core/storage_upload_spec.rb +++ b/google-apis-core/spec/google/apis/core/storage_upload_spec.rb @@ -307,4 +307,224 @@ expect { command.execute(client) }.to raise_error Google::Apis::ServerError end end + context 'when uploading with md5 checksum' do + + let(:file) { StringIO.new(file_content) } + let(:md5_checksum) {"md5_checksum" } + let(:body_with_md5) { { "md5Hash" => md5_checksum }.to_json } + + context 'with single shot upload' do + let(:file_content) { "Hello world" } + + before(:example) do + command.body = body_with_md5 + allow(command).to receive(:formatted_checksum_header).and_call_original + + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" } + .to_return(body: %(OK)) + end + + it 'should not include X-Goog-Hash header during initiation' do + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to_not have_been_made + end + + it 'calls formatted_checksum_header and returns correct value' do + expect(command.formatted_checksum_header).to eq("md5=#{md5_checksum}") + end + end + + context 'with chunked upload' do + let(:file_content) { "Hello world" * 2 } + + before(:example) do + command.body = body_with_md5 + allow(command).to receive(:formatted_checksum_header).and_call_original + + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { 'Content-Range' => 'bytes 0-10/22' }) + .to_return(status: [308, 'Resume Incomplete']) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { + 'Content-Range' => 'bytes 11-21/22', + 'X-Goog-Hash' => "md5=#{md5_checksum}" + }) + .to_return(body: %(OK)) + end + + it 'should not include X-Goog-Hash header during initiation' do + command.options.upload_chunk_size = 11 + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to_not have_been_made + end + + it 'includes md5 checksum in X-Goog-Hash header only in the last chunk' do + command.options.upload_chunk_size = 11 + command.execute(client) + + # First chunk should NOT have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 0-10/22' && !req.headers.key?('X-Goog-Hash') }).to have_been_made + + # Last chunk should have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 11-21/22' && req.headers['X-Goog-Hash'] == "md5=#{md5_checksum}" }).to have_been_made + end + end + end + + context 'when uploading with crc32c checksum' do + + let(:file) { StringIO.new(file_content) } + let(:crc32c_checksum) { "abc_checksum" } + let(:body_with_crc32c) { { "crc32c" => crc32c_checksum }.to_json } + + context 'with single shot upload' do + let(:file_content) { "Hello world" } + + before(:example) do + command.body = body_with_crc32c + allow(command).to receive(:formatted_checksum_header).and_call_original + + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" } + .to_return(body: %(OK)) + end + + it 'should not include X-Goog-Hash header during initiation' do + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to_not have_been_made + end + + it 'calls formatted_checksum_header and returns correct value' do + expect(command.formatted_checksum_header).to eq("crc32c=#{crc32c_checksum}") + end + end + + context 'with chunked upload' do + let(:file_content) { "Hello world" * 2 } + + before(:example) do + command.body = body_with_crc32c + allow(command).to receive(:formatted_checksum_header).and_call_original + + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { 'Content-Range' => 'bytes 0-10/22' }) + .to_return(status: [308, 'Resume Incomplete']) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { + 'Content-Range' => 'bytes 11-21/22', + 'X-Goog-Hash' => "crc32c=#{crc32c_checksum}" + }) + .to_return(body: %(OK)) + end + + it 'should not include X-Goog-Hash header during initiation' do + command.options.upload_chunk_size = 11 + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to_not have_been_made + end + + it 'includes md5 checksum in X-Goog-Hash header only in the last chunk' do + command.options.upload_chunk_size = 11 + command.execute(client) + + # First chunk should NOT have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 0-10/22' && !req.headers.key?('X-Goog-Hash') }).to have_been_made + + # Last chunk should have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 11-21/22' && req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum}" }).to have_been_made + end + + end + end + + context 'when uploading with md5 and crc32c checksum' do + let(:file) { StringIO.new(file_content) } + let(:md5_checksum) { "md5_checksum"} + let(:crc32c_checksum) { "crc32c_checksum" } + let(:body_with_md5_crc32c) { { "md5Hash" => md5_checksum, "crc32c" => crc32c_checksum }.to_json } + + context 'with single shot upload' do + let(:file_content) { "Hello world" } + + before(:example) do + command.body = body_with_md5_crc32c + allow(command).to receive(:formatted_checksum_header).and_call_original + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" } + .to_return(body: %(OK)) + end + + it 'should not include X-Goog-Hash header during initiation' do + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to_not have_been_made + end + + it 'calls formatted_checksum_header and returns correct value' do + expect(command.formatted_checksum_header).to eq("crc32c=#{crc32c_checksum},md5=#{md5_checksum}") + end + end + + context 'with chunked upload' do + let(:file_content) { "Hello world" * 2 } + + before(:example) do + command.body = body_with_md5_crc32c + allow(command).to receive(:formatted_checksum_header).and_call_original + + stub_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .to_return(headers: { 'Location' => 'https://www.googleapis.com/zoo/animals' }, body: %(OK)) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { 'Content-Range' => 'bytes 0-10/22' }) + .to_return(status: [308, 'Resume Incomplete']) + stub_request(:put, 'https://www.googleapis.com/zoo/animals') + .with(headers: { + 'Content-Range' => 'bytes 11-21/22', + 'X-Goog-Hash' => "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" + }) + .to_return(body: %(OK)) + end + + it 'should not includeX-Goog-Hash header during initiation' do + command.options.upload_chunk_size = 11 + command.execute(client) + expect(a_request(:post, 'https://www.googleapis.com/zoo/animals?uploadType=resumable') + .with { |req| req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to_not have_been_made + end + + it 'includes md5 and crc32c checksum in X-Goog-Hash header only in the last chunk' do + command.options.upload_chunk_size = 11 + command.execute(client) + + # First chunk should NOT have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 0-10/22' && !req.headers.key?('X-Goog-Hash') }).to have_been_made + + # Last chunk should have the X-Goog-Hash header + expect(a_request(:put, 'https://www.googleapis.com/zoo/animals') + .with { |req| req.headers['Content-Range'] == 'bytes 11-21/22' && req.headers['X-Goog-Hash'] == "crc32c=#{crc32c_checksum},md5=#{md5_checksum}" }).to have_been_made + end + + end + end + end