diff --git a/google-cloud-storage/lib/google/cloud/storage/bucket.rb b/google-cloud-storage/lib/google/cloud/storage/bucket.rb index ca8461354bb7..e0cd98b00300 100644 --- a/google-cloud-storage/lib/google/cloud/storage/bucket.rb +++ b/google-cloud-storage/lib/google/cloud/storage/bucket.rb @@ -1805,6 +1805,7 @@ def create_file file, path ||= file.path if file.respond_to? :path path ||= file if file.is_a? String raise ArgumentError, "must provide path" if path.nil? + checksum = :crc32c if checksum.nil? && crc32c.nil? && md5.nil? crc32c = crc32c_for file, checksum, crc32c md5 = md5_for file, checksum, md5 diff --git a/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb b/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb index dcfefc6c616d..c65f33bb7c08 100644 --- a/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb +++ b/google-cloud-storage/lib/google/cloud/storage/file/verifier.rb @@ -49,29 +49,47 @@ def self.verify_crc32c gcloud_file, local_file gcloud_file.crc32c == crc32c_for(local_file) end + # Calculates MD5 digest using either file path or open stream. def self.md5_for local_file - if local_file.respond_to? :to_path - ::File.open Pathname(local_file).to_path, "rb" do |f| - ::Digest::MD5.file(f).base64digest - end - else # StringIO - local_file.rewind - md5 = ::Digest::MD5.base64digest local_file.read - local_file.rewind - md5 - end + _digest_for local_file, ::Digest::MD5 end + # Calculates CRC32c digest using either file path or open stream. def self.crc32c_for local_file - if local_file.respond_to? :to_path + _digest_for local_file, ::Digest::CRC32c + end + + # @private + # Computes a base64-encoded digest for a local file or IO stream. + # + # This method handles two types of inputs for `local_file`: + # 1. A file path (String or Pathname): It efficiently streams the file + # to compute the digest without loading the entire file into memory. + # 2. An IO-like stream (e.g., File, StringIO): It reads the stream's + # content to compute the digest. The stream is rewound before and after + # reading to ensure its position is not permanently changed. + # + # @param local_file [String, Pathname, IO] The local file path or IO + # stream for which to compute the digest. + # @param digest_class [Class] The digest class to use for the + # calculation (e.g., `Digest::MD5`). It must respond to `.file` and + # `.base64digest`. + # + # @return [String] The base64-encoded digest of the file's content. + # + def self._digest_for local_file, digest_class + + if local_file.respond_to?(:to_path) || local_file.is_a?(String) + # Case 1: Input is a file path (String, Pathname, or object that responds to :to_path). ::File.open Pathname(local_file).to_path, "rb" do |f| - ::Digest::CRC32c.file(f).base64digest + digest_class.file(f).base64digest end - else # StringIO + else + # Case 2: Input is an open stream (File or StringIO). local_file.rewind - crc32c = ::Digest::CRC32c.base64digest local_file.read + digest = digest_class.base64digest local_file.read local_file.rewind - crc32c + digest end end end diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb index ce617da110e6..7fe89376cfd2 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_encryption_test.rb @@ -131,6 +131,10 @@ def empty_file_gapi cache_control: nil, content_disposition: nil, content_encoding: nil, content_language: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, storage_class: nil + # Set crc32c if both md5 and crc32c are not provided + if md5.nil? && crc32c.nil? + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for(StringIO.new("Hello world")) + end params = { cache_control: cache_control, content_type: content_type, content_disposition: content_disposition, md5_hash: md5, diff --git a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb index 203e423467e5..7b9685f1dc08 100644 --- a/google-cloud-storage/test/google/cloud/storage/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/bucket_test.rb @@ -101,6 +101,40 @@ _(bucket_complete.autoclass_enabled).must_equal bucket_autoclass_enabled _(bucket_complete.autoclass_terminal_storage_class).must_equal bucket_autoclass_terminal_storage_class end + + it "creates a file with checksum: :crc32c by default" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile + + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + bucket.create_file tmpfile, new_file_name + + mock.verify + end + end + + it "creates a file with a StringIO and checksum: :crc32c by default" do + new_file_name = random_file_path + new_file_contents = StringIO.new "Hello world" + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0}) + + bucket.service.mocked_service = mock + + bucket.create_file new_file_contents, new_file_name + + mock.verify + end it "returns frozen cors" do bucket_complete.cors.each do |cors| @@ -595,9 +629,11 @@ new_file_name = random_file_path Tempfile.create ["google-cloud", ".txt"] do |tmpfile| + + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket_user_project.name, new_file_name), - [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) bucket_user_project.service.mocked_service = mock @@ -610,7 +646,7 @@ it "creates an file with a StringIO" do new_file_name = random_file_path - new_file_contents = StringIO.new + new_file_contents = StringIO.new("Hello world") mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), @@ -1417,6 +1453,12 @@ def empty_file_gapi cache_control: nil, content_disposition: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, storage_class: nil, temporary_hold: nil, event_based_hold: nil + + # Set crc32c if both md5 and crc32c are not provided + if md5.nil? && crc32c.nil? + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for(StringIO.new("Hello world")) + end + params = { cache_control: cache_control, content_type: content_type, content_disposition: content_disposition, md5_hash: md5, diff --git a/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb b/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb index c304f54b3838..5f530634f16b 100644 --- a/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb +++ b/google-cloud-storage/test/google/cloud/storage/lazy/bucket_test.rb @@ -244,6 +244,25 @@ mock.verify end end + + it "creates a file with checksum: :crc32c by default" do + new_file_name = random_file_path + + Tempfile.open ["google-cloud", ".txt"] do |tmpfile| + tmpfile.write "Hello world!" + tmpfile.rewind + + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile + mock = Minitest::Mock.new + mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) + + bucket.service.mocked_service = mock + bucket.create_file tmpfile, new_file_name + + mock.verify + end + end it "creates a file with attributes" do new_file_name = random_file_path @@ -340,9 +359,10 @@ new_file_name = random_file_path Tempfile.create ["google-cloud", ".txt"] do |tmpfile| + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile mock = Minitest::Mock.new mock.expect :insert_object, create_file_gapi(bucket_user_project.name, new_file_name), - [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) + [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) bucket_user_project.service.mocked_service = mock @@ -1091,6 +1111,10 @@ def empty_file_gapi cache_control: nil, content_disposition: nil, content_encoding: nil, content_language: nil, content_type: nil, crc32c: nil, md5: nil, metadata: nil, storage_class: nil + # Set crc32c if both md5 and crc32c are not provided + if md5.nil? && crc32c.nil? + crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for(StringIO.new("Hello world")) + end params = { cache_control: cache_control, content_type: content_type, content_disposition: content_disposition, md5_hash: md5,