diff --git a/google-cloud-storage_batch_operations/samples/.rubocop.yml b/google-cloud-storage_batch_operations/samples/.rubocop.yml new file mode 100644 index 000000000000..b3a280358647 --- /dev/null +++ b/google-cloud-storage_batch_operations/samples/.rubocop.yml @@ -0,0 +1,24 @@ +inherit_gem: + google-style: google-style.yml + +Lint/EmptyBlock: + Enabled: false +Lint/ShadowingOuterLocalVariable: + Enabled: false +Metrics/AbcSize: + Enabled: false +Metrics/BlockLength: + Enabled: false +Metrics/CyclomaticComplexity: + Max: 12 +Metrics/MethodLength: + Enabled: false +Naming/AccessorMethodName: + Enabled: false +Style/GlobalVars: + Exclude: + - "acceptance/helper.rb" +Style/Next: + Enabled: false +Style/SymbolProc: + Enabled: false diff --git a/google-cloud-storage_batch_operations/samples/Gemfile b/google-cloud-storage_batch_operations/samples/Gemfile new file mode 100644 index 000000000000..c8bf8442a31b --- /dev/null +++ b/google-cloud-storage_batch_operations/samples/Gemfile @@ -0,0 +1,30 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source "https://rubygems.org" + +if ENV["GOOGLE_CLOUD_SAMPLES_TEST"] == "master" + gem "google-cloud-storage", group: :test, path: "../../google-cloud-storage" + gem "google-cloud-storage_batch_operations", group: :test, path: "../../google-cloud-storage_batch_operations" +else + gem "google-cloud-storage" + gem "google-cloud-storage_batch_operations" +end +group :test do + gem "google-style", "~> 1.30.0" + gem "minitest", "~> 5.14" + gem "minitest-focus", "~> 1.1" + gem "minitest-hooks", "~> 1.5" + gem "rake" +end diff --git a/google-cloud-storage_batch_operations/samples/Rakefile b/google-cloud-storage_batch_operations/samples/Rakefile new file mode 100644 index 000000000000..fae40276bdec --- /dev/null +++ b/google-cloud-storage_batch_operations/samples/Rakefile @@ -0,0 +1,20 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "rake/testtask" + +Rake::TestTask.new "test" do |t| + t.test_files = FileList["**/*_test.rb"] + t.warning = false +end diff --git a/google-cloud-storage_batch_operations/samples/acceptance/batch_job_test.rb b/google-cloud-storage_batch_operations/samples/acceptance/batch_job_test.rb new file mode 100644 index 000000000000..5645295499a6 --- /dev/null +++ b/google-cloud-storage_batch_operations/samples/acceptance/batch_job_test.rb @@ -0,0 +1,90 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require_relative "helper" +require "google/cloud/storage" +require_relative "../storage_batch_create_job" +require_relative "../storage_batch_delete_job" +require_relative "../storage_batch_cancel_job" +require_relative "../storage_batch_list_job" +require_relative "../storage_batch_get_job" +require 'pry' + +describe "Batch jobs Snippets" do + let(:storage) { Google::Cloud::Storage.new} + let(:project_name) { storage.project } + let(:bucket) { @bucket } + let(:file_content) { "some content" } + let(:remote_file_name) { "ruby_file_#{SecureRandom.hex}" } + + before :all do + @bucket = create_bucket_helper random_bucket_name + end + + after :all do + delete_bucket_helper @bucket.name + end + + def create_test_job my_job + bucket.create_file StringIO.new(file_content), remote_file_name + create_job bucket_name: bucket.name, prefix: "ruby_file", job_name: my_job, project_name: project_name + end + + describe "storage batch manage operations" do + before do + @job_name = "ruby-sbo-job-#{SecureRandom.hex}" + create_test_job @job_name + end + + it "lists jobs and includes the created job" do + out, _err = capture_io { list_job project_name: project_name } + assert_includes out, @job_name, "Expected job name not found in the result list" + end + + it "fetches the details of a job" do + result = get_job project_name: project_name, job_name: @job_name + assert_includes result, @job_name, "Expected job name not found in the result" + end + + it "cancels a job" do + assert_output "The #{@job_name} is canceled.\n" do + cancel_job project_name: project_name, job_name: @job_name + end + end + end + + describe "Delete storage batch operation" do + before do + @job_name = "ruby-sbo-job-#{SecureRandom.hex}" + create_test_job @job_name + end + it "deletes a job" do + retry_job_status do + get_job project_name: project_name, job_name: @job_name + end + assert_output "The #{@job_name} is deleted.\n" do + delete_job project_name: project_name, job_name: @job_name + end + end + end + + describe "creates a storage batch operation" do + it "creates a job" do + @job_name = "ruby-sbo-job-#{SecureRandom.hex}" + assert_output "The #{@job_name} is created.\n" do + create_test_job @job_name + end + end + end +end diff --git a/google-cloud-storage_batch_operations/samples/acceptance/helper.rb b/google-cloud-storage_batch_operations/samples/acceptance/helper.rb new file mode 100644 index 000000000000..2e5b28157562 --- /dev/null +++ b/google-cloud-storage_batch_operations/samples/acceptance/helper.rb @@ -0,0 +1,68 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +require "google/cloud/errors" +require "google/cloud/storage" +require "minitest/autorun" +require "minitest/focus" +require "minitest/hooks/default" +require "net/http" +require "time" +require "securerandom" + +def create_bucket_helper bucket_name + storage_client = Google::Cloud::Storage.new + retry_resource_exhaustion do + storage_client.create_bucket bucket_name + end +end + +def delete_bucket_helper bucket_name + storage_client = Google::Cloud::Storage.new + retry_resource_exhaustion do + bucket = storage_client.bucket bucket_name + return unless bucket + + bucket.files.each(&:delete) + bucket.delete + end +end + +def retry_job_status + 5.times do + status = yield[/job_status-\s*(\w+)/, 1] + break unless status == "RUNNING" + + puts "Job in Running status. Gonna try again" + sleep rand(10..16) + end +end + +def retry_resource_exhaustion + 5.times do + return yield + rescue Google::Cloud::ResourceExhaustedError => e + puts "\n#{e} Gonna try again" + sleep rand(10..16) + rescue StandardError => e + puts "\n#{e}" + raise e + end + raise Google::Cloud::ResourceExhaustedError, "Maybe take a break from creating and deleting buckets for a bit" +end + +def random_bucket_name + t = Time.now.utc.iso8601.gsub ":", "-" + "ruby-storage-samples-test-#{t}-#{SecureRandom.hex 4}".downcase +end diff --git a/google-cloud-storage_batch_operations/samples/storage_batch_cancel_job.rb b/google-cloud-storage_batch_operations/samples/storage_batch_cancel_job.rb new file mode 100644 index 000000000000..6d097edb21d1 --- /dev/null +++ b/google-cloud-storage_batch_operations/samples/storage_batch_cancel_job.rb @@ -0,0 +1,38 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require "google/cloud/storage_batch_operations" + +# [START storage_batch_cancel_job] +def cancel_job project_name:, job_name: + # The Name/ID of your project + # project_name = "your-project-id" + + # The name of your Storage batch operation job + # job_name = "your-job-name" + + client = Google::Cloud::StorageBatchOperations.storage_batch_operations + parent = "projects/#{project_name}/locations/global" + + request = Google::Cloud::StorageBatchOperations::V1::CancelJobRequest.new name: "#{parent}/jobs/#{job_name}" + result = client.cancel_job request + message = if result.is_a? Google::Cloud::StorageBatchOperations::V1::CancelJobResponse + "The #{job_name} is canceled." + else + "The #{job_name} is not canceled." + end + puts message +end +# [END storage_batch_cancel_job] + +cancel_job project_name: ARGV.shift, job_name: ARGV.shift if $PROGRAM_NAME == __FILE__ diff --git a/google-cloud-storage_batch_operations/samples/storage_batch_create_job.rb b/google-cloud-storage_batch_operations/samples/storage_batch_create_job.rb new file mode 100644 index 000000000000..2a29bd58ac81 --- /dev/null +++ b/google-cloud-storage_batch_operations/samples/storage_batch_create_job.rb @@ -0,0 +1,72 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require "google/cloud/storage_batch_operations" + +# [START storage_batch_create_job] +def create_job bucket_name:, prefix:, job_name:, project_name: + # The ID of your GCS bucket + # bucket_name = "your-unique-bucket-name" + + # Prefix is the first part of filename on which job has to be executed + # prefix = 'test' + + # The Name/ID of your project + # project_name = "your-project-id" + + # The name of your Storage batch operation job + # job_name = "your-job-name" + + client = Google::Cloud::StorageBatchOperations.storage_batch_operations + + parent = "projects/#{project_name}/locations/global" + + prefix_list = Google::Cloud::StorageBatchOperations::V1::PrefixList.new( + included_object_prefixes: [prefix] + ) + + bucket = Google::Cloud::StorageBatchOperations::V1::BucketList::Bucket.new( + bucket: bucket_name, + prefix_list: prefix_list + ) + + bucket_list = Google::Cloud::StorageBatchOperations::V1::BucketList.new( + buckets: [bucket] + ) + + # Define the delete operation + delete_object = Google::Cloud::StorageBatchOperations::V1::DeleteObject.new( + permanent_object_deletion_enabled: false + ) + + # Build the job + job = Google::Cloud::StorageBatchOperations::V1::Job.new( + name: job_name, + bucket_list: bucket_list, + delete_object: delete_object + ) + + request = Google::Cloud::StorageBatchOperations::V1::CreateJobRequest.new parent: parent, job_id: job_name, job: job + result = client.create_job request + + puts result.is_a?(Gapic::Operation) ? "The #{job_name} is created." : "The #{job_name} is not created." +end +# [END storage_batch_create_job] + +if $PROGRAM_NAME == __FILE__ + create_job( + bucket_name: ARGV.shift, + prefix: ARGV.shift, + job_name: ARGV.shift, + project_name: ARGV.shift ) +end diff --git a/google-cloud-storage_batch_operations/samples/storage_batch_delete_job.rb b/google-cloud-storage_batch_operations/samples/storage_batch_delete_job.rb new file mode 100644 index 000000000000..8c0908930b4b --- /dev/null +++ b/google-cloud-storage_batch_operations/samples/storage_batch_delete_job.rb @@ -0,0 +1,32 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require "google/cloud/storage_batch_operations" + +# [START storage_batch_delete_job] +def delete_job project_name:, job_name: + # The Name/ID of your project + # project_name = "your-project-id" + + # The name of your Storage batch operation job + # job_name = "your-job-name" + + client = Google::Cloud::StorageBatchOperations.storage_batch_operations + parent = "projects/#{project_name}/locations/global" + request = Google::Cloud::StorageBatchOperations::V1::DeleteJobRequest.new name: "#{parent}/jobs/#{job_name}" + result = client.delete_job request + puts result.is_a?(Google::Protobuf::Empty) ? "The #{job_name} is deleted." : "The #{job_name} is not deleted." +end +# [END storage_batch_delete_job] + +delete_job project_name: ARGV.shift, job_name: ARGV.shift if $PROGRAM_NAME == __FILE__ diff --git a/google-cloud-storage_batch_operations/samples/storage_batch_get_job.rb b/google-cloud-storage_batch_operations/samples/storage_batch_get_job.rb new file mode 100644 index 000000000000..1d92e8e8cad9 --- /dev/null +++ b/google-cloud-storage_batch_operations/samples/storage_batch_get_job.rb @@ -0,0 +1,33 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START storage_batch_get_job] +def get_job project_name:, job_name: + # The Name/ID of your project + # project_name = "your-project-id" + + # The name of your Storage batch operation job + # job_name = "your-job-name" + + require "google/cloud/storage_batch_operations" + + client = Google::Cloud::StorageBatchOperations.storage_batch_operations + parent = "projects/#{project_name}/locations/global" + request = Google::Cloud::StorageBatchOperations::V1::GetJobRequest.new name: "#{parent}/jobs/#{job_name}" + result = client.get_job request + "job_name- #{result.name}, job_status- #{result.state}" +end +# [END storage_batch_get_job] + +get_job project_name: ARGV.shift, job_name: ARGV.shift if $PROGRAM_NAME == __FILE__ diff --git a/google-cloud-storage_batch_operations/samples/storage_batch_list_job.rb b/google-cloud-storage_batch_operations/samples/storage_batch_list_job.rb new file mode 100644 index 000000000000..6df9722bd350 --- /dev/null +++ b/google-cloud-storage_batch_operations/samples/storage_batch_list_job.rb @@ -0,0 +1,30 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +require "google/cloud/storage_batch_operations" + +# [START storage_batch_list_job] +def list_job project_name: + # The Name/ID of your project + # project_name = "your-project-id" + client = Google::Cloud::StorageBatchOperations.storage_batch_operations + parent = "projects/#{project_name}/locations/global" + request = Google::Cloud::StorageBatchOperations::V1::ListJobsRequest.new parent: parent, page_size: 10 + result = client.list_jobs request + result.each do |job| + puts "Job name: #{job.name}" + end +end +# [END storage_batch_list_job] + +list_job project_name: ARGV.shift if $PROGRAM_NAME == __FILE__ \ No newline at end of file