From 8ce521ed29cd8a94f14670da3ec259e33d718059 Mon Sep 17 00:00:00 2001 From: Seth Boyles Date: Wed, 16 Feb 2022 17:38:26 +0000 Subject: [PATCH 1/3] Bump fog-azure-rm * we have to use a special branch of azure-storage-ruby for now until its merged in Co-authored-by: Seth Boyles Co-authored-by: Tom Viehman --- Gemfile | 6 ++++-- Gemfile.lock | 26 ++++++++++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index 5ab2c2ba629..6da8011cfce 100644 --- a/Gemfile +++ b/Gemfile @@ -53,12 +53,14 @@ gem 'activemodel', '~> 6.1.4' gem 'railties', '~> 6.1.4' # Blobstore and Bits Service Dependencies -gem 'azure-storage', '0.14.0.preview' # https://github.com/Azure/azure-storage-ruby/issues/122 gem 'bits_service_client', '~> 3.3', '>= 3.3.0' + +gem 'azure-storage-blob', git: 'https://github.com/sethboyles/azure-storage-ruby.git', branch: 'x-ms-blob-content-type-fix-1.1' + gem 'fog-aliyun' gem 'fog-aws' gem 'fog-azure-rm', git: 'https://github.com/fog/fog-azure-rm.git', branch: 'fog-arm-cf' -gem 'fog-google' +gem 'fog-google', '~> 1.18.0' gem 'fog-local' gem 'fog-openstack' diff --git a/Gemfile.lock b/Gemfile.lock index 143467ceaf0..1d46e85cff6 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -7,7 +7,7 @@ GIT GIT remote: https://github.com/fog/fog-azure-rm.git - revision: 79a738331538c6bdba806774cb11af8977979f8e + revision: 69ba1d7e9f82b7a71214b92fa6fa95169b539ed3 branch: fog-arm-cf specs: fog-azure-rm (0.4.7) @@ -25,6 +25,19 @@ GIT mime-types (~> 3.0) vhd (= 0.0.4) +GIT + remote: https://github.com/sethboyles/azure-storage-ruby.git + revision: d4737cdfcef3f3c4c2463a54690b6b73c0f6f707 + branch: x-ms-blob-content-type-fix-1.1 + specs: + azure-storage-blob (1.1.0) + azure-core (~> 0.1.13) + azure-storage-common (~> 1.0) + nokogiri (~> 1.6, >= 1.6.8) + azure-storage-common (1.1.0) + azure-core (~> 0.1.13) + nokogiri (~> 1.6, >= 1.6.8) + GIT remote: https://github.com/sporkrb/spork revision: 224df492657e617a0c93c0319e78f0eefee5b636 @@ -142,8 +155,8 @@ GEM eventmachine (>= 1.0.0.beta.4) erubi (1.10.0) eventmachine (1.2.7) - excon (0.90.0) - faraday (0.15.4) + excon (0.91.0) + faraday (0.17.4) multipart-post (>= 1.2, < 3) faraday-cookie_jar (0.0.6) faraday (>= 0.7.4) @@ -500,7 +513,8 @@ GEM crack (>= 0.3.2) hashdiff (>= 0.4.0, < 2.0.0) webrick (1.7.0) - xml-simple (1.1.5) + xml-simple (1.1.9) + rexml yajl-ruby (1.4.1) yard (0.9.27) webrick (~> 1.7.0) @@ -520,7 +534,7 @@ DEPENDENCIES addressable allowy (>= 2.1.0) awesome_print - azure-storage (= 0.14.0.preview) + azure-storage-blob! bits_service_client (~> 3.3, >= 3.3.0) byebug cf-copilot (= 0.0.14) @@ -535,7 +549,7 @@ DEPENDENCIES fog-aliyun fog-aws fog-azure-rm! - fog-google + fog-google (~> 1.7.1) fog-local fog-openstack googleapis-common-protos (>= 1.3.12) From 49677502e558fe6dda95075e8410c23bb1ca41bd Mon Sep 17 00:00:00 2001 From: Seth Boyles Date: Wed, 16 Feb 2022 17:43:28 +0000 Subject: [PATCH 2/3] Revert "Revert "Merge pull request #2478 from cloudfoundry/dependabot/bundler/fog-aws-3.12.0"" This reverts commit 197a975ca4671d5a7d38d8c8e65a3c7a4f7e4a1f. --- Gemfile | 1 + Gemfile.lock | 68 +++++++++++-------- .../blobstore/fog/fog_client.rb | 4 +- .../blobstore/fog/providers.rb | 13 ++++ .../blobstore/fog/fog_client_spec.rb | 6 +- 5 files changed, 58 insertions(+), 34 deletions(-) diff --git a/Gemfile b/Gemfile index 6da8011cfce..22f3852df78 100644 --- a/Gemfile +++ b/Gemfile @@ -63,6 +63,7 @@ gem 'fog-azure-rm', git: 'https://github.com/fog/fog-azure-rm.git', branch: 'fog gem 'fog-google', '~> 1.18.0' gem 'fog-local' gem 'fog-openstack' +gem 'fog-core', '~> 2.1.2' gem 'cf-uaa-lib', '~> 4.0.1' gem 'vcap-concurrency', git: 'https://github.com/cloudfoundry/vcap-concurrency.git', ref: '2a5b0179' diff --git a/Gemfile.lock b/Gemfile.lock index 1d46e85cff6..de770c7730f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -11,7 +11,8 @@ GIT branch: fog-arm-cf specs: fog-azure-rm (0.4.7) - azure-storage (>= 0.11.5.preview, < 1.0) + azure-storage-blob (~> 1.1) + azure-storage-common (~> 1.1) azure_mgmt_compute (~> 0.9.0) azure_mgmt_dns (~> 0.9.0) azure_mgmt_key_vault (~> 0.9.0) @@ -20,8 +21,8 @@ GIT azure_mgmt_sql (~> 0.9.0) azure_mgmt_storage (~> 0.9.0) azure_mgmt_traffic_manager (~> 0.9.0) - fog-core (~> 1.43) - fog-json (~> 1.0.2) + fog-core (>= 1.43) + fog-json (>= 1.0.2) mime-types (~> 3.0) vhd (= 0.0.4) @@ -79,7 +80,7 @@ GEM i18n ast (2.4.2) awesome_print (1.9.2) - azure-core (0.1.14) + azure-core (0.1.15) faraday (~> 0.9) faraday_middleware (~> 0.10) nokogiri (~> 1.6) @@ -88,6 +89,13 @@ GEM faraday (~> 0.9) faraday_middleware (~> 0.10) nokogiri (~> 1.6, >= 1.6.8) + azure-storage-blob (1.1.0) + azure-core (~> 0.1.13) + azure-storage-common (~> 1.0) + nokogiri (~> 1.6, >= 1.6.8) + azure-storage-common (1.1.0) + azure-core (~> 0.1.13) + nokogiri (~> 1.6, >= 1.6.8) azure_mgmt_compute (0.9.0) ms_rest_azure (~> 0.7.0) azure_mgmt_dns (0.9.0) @@ -107,7 +115,7 @@ GEM backport (1.2.0) beefcake (1.0.0) benchmark (0.2.0) - bit-struct (0.16) + bit-struct (0.17) bits_service_client (3.4.0) activesupport statsd-ruby (~> 1.4.0) @@ -136,8 +144,7 @@ GEM debase (0.2.5.beta2) debase-ruby_core_source (>= 0.10.12) debase-ruby_core_source (0.10.12) - declarative (0.0.10) - declarative-option (0.1.0) + declarative (0.0.20) delayed_job (4.1.9) activesupport (>= 3.0, < 6.2) diff-lcs (1.5.0) @@ -158,10 +165,10 @@ GEM excon (0.91.0) faraday (0.17.4) multipart-post (>= 1.2, < 3) - faraday-cookie_jar (0.0.6) - faraday (>= 0.7.4) + faraday-cookie_jar (0.0.7) + faraday (>= 0.8.0) http-cookie (~> 1.0.0) - faraday_middleware (0.12.2) + faraday_middleware (0.14.0) faraday (>= 0.7.4, < 1.0) ffi (1.15.5) ffi-compiler (1.0.1) @@ -175,30 +182,31 @@ GEM fog-json ipaddress (~> 0.8) xml-simple (~> 1.1) - fog-aws (2.0.1) - fog-core (~> 1.38) - fog-json (~> 1.0) + fog-aws (3.12.0) + fog-core (~> 2.1) + fog-json (~> 1.1) fog-xml (~> 0.1) ipaddress (~> 0.8) - fog-core (1.45.0) + fog-core (2.1.2) builder excon (~> 0.58) formatador (~> 0.2) + mime-types fog-google (1.7.1) fog-core fog-json fog-xml google-api-client (~> 0.23.0) - fog-json (1.0.2) - fog-core (~> 1.0) + fog-json (1.2.0) + fog-core multi_json (~> 1.10) fog-local (0.8.0) fog-core (>= 1.27, < 3.0) - fog-openstack (0.3.10) - fog-core (>= 1.45, <= 2.1.0) + fog-openstack (1.0.11) + fog-core (~> 2.1) fog-json (>= 1.0) ipaddress (>= 0.8) - fog-xml (0.1.3) + fog-xml (0.1.4) fog-core nokogiri (>= 1.5.11, < 2.0.0) formatador (0.3.0) @@ -262,7 +270,7 @@ GEM json_pure (2.6.1) jsonpath (1.1.0) multi_json - jwt (2.2.1) + jwt (2.3.0) kramdown (2.3.1) rexml kramdown-parser-gfm (1.1.0) @@ -286,12 +294,12 @@ GEM nokogiri (>= 1.5.9) machinist (1.0.6) membrane (1.1.0) - memoist (0.16.0) + memoist (0.16.2) method_source (1.0.0) middleware (0.1.0) mime-types (3.4.1) mime-types-data (~> 3.2015) - mime-types-data (3.2021.1115) + mime-types-data (3.2022.0105) mini_portile2 (2.7.1) minitest (5.15.0) ms_rest (0.6.4) @@ -323,7 +331,7 @@ GEM nokogiri (1.13.1-x86_64-linux) racc (~> 1.4) oj (3.13.11) - os (1.0.1) + os (1.1.4) palm_civet (1.1.0) parallel (1.21.0) parallel_tests (3.7.3) @@ -368,9 +376,9 @@ GEM ffi (~> 1.0) recursive-open-struct (1.1.3) regexp_parser (2.2.0) - representable (3.0.4) + representable (3.1.1) declarative (< 0.1.0) - declarative-option (< 0.2.0) + trailblazer-option (>= 0.1.1, < 0.2.0) uber (< 0.2.0) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) @@ -442,9 +450,9 @@ GEM pg (>= 0.18.0, != 1.2.0) sequel (>= 4.38.0) sexp_processor (4.7.0) - signet (0.11.0) - addressable (~> 2.3) - faraday (~> 0.9) + signet (0.16.0) + addressable (~> 2.8) + faraday (>= 0.17.3, < 2.0) jwt (>= 1.5, < 3.0) multi_json (~> 1.10) simplecov (0.13.0) @@ -497,7 +505,8 @@ GEM thread_safe (0.3.6) tilt (2.0.10) timecop (0.9.4) - timeliness (0.3.10) + timeliness (0.4.4) + trailblazer-option (0.1.2) tzinfo (2.0.4) concurrent-ruby (~> 1.0) uber (0.1.0) @@ -550,6 +559,7 @@ DEPENDENCIES fog-aws fog-azure-rm! fog-google (~> 1.7.1) + fog-core (~> 2.1.2) fog-local fog-openstack googleapis-common-protos (>= 1.3.12) diff --git a/lib/cloud_controller/blobstore/fog/fog_client.rb b/lib/cloud_controller/blobstore/fog/fog_client.rb index 894ce624f81..d2fdfe04e86 100644 --- a/lib/cloud_controller/blobstore/fog/fog_client.rb +++ b/lib/cloud_controller/blobstore/fog/fog_client.rb @@ -114,7 +114,7 @@ def blob(key) end def files_for(prefix, _ignored_directory_prefixes=[]) - if connection.is_a? Fog::Storage::Local::Real + if connection.is_a? Fog::Local::Storage::Real directory = connection.directories.get(File.join(dir.key, prefix || '')) directory ? directory.files : [] else @@ -126,7 +126,7 @@ def ensure_bucket_exists return if local? options = { max_keys: 1 } - options['limit'] = 1 if connection.service == Fog::Storage::OpenStack + options['limit'] = 1 if connection.service == Fog::OpenStack::Storage connection.directories.get(@directory_key, options) || connection.directories.create(key: @directory_key, public: false) end diff --git a/lib/cloud_controller/blobstore/fog/providers.rb b/lib/cloud_controller/blobstore/fog/providers.rb index ad21e49b0ad..a6b23fecd12 100644 --- a/lib/cloud_controller/blobstore/fog/providers.rb +++ b/lib/cloud_controller/blobstore/fog/providers.rb @@ -1,6 +1,19 @@ +require 'fog/core' + +# This can be removed once fog-google fixes +# https://github.com/fog/fog-google/issues/421, which will allow us to upgrade +# fog-core to 2.2.4. +original = Fog::Logger[:deprecation] +Fog::Logger[:deprecation] = nil + require 'fog/aliyun' +# Hack until https://github.com/fog/fog-aliyun/pull/155 is merged and released +Fog::Aliyun::Compute = Fog::Compute::Aliyun + require 'fog/aws' require 'fog/local' require 'fog/google' require 'fog/azurerm' require 'fog/openstack' + +Fog::Logger[:deprecation] = original diff --git a/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb b/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb index 82272304887..82398dbd4af 100644 --- a/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb +++ b/spec/unit/lib/cloud_controller/blobstore/fog/fog_client_spec.rb @@ -729,21 +729,21 @@ def upload_tmpfile(client, key='abcdef') end it 'gets the bucket' do - expect_any_instance_of(Fog::Storage::AWS::Directories).to receive(:get).with(directory_key, max_keys: 1) + expect_any_instance_of(Fog::AWS::Storage::Directories).to receive(:get).with(directory_key, max_keys: 1) subject.ensure_bucket_exists end context 'the bucket exists' do it 'does not create the bucket' do subject.ensure_bucket_exists - expect_any_instance_of(Fog::Storage::AWS::Directories).not_to receive(:create).with(key: directory_key, public: false) + expect_any_instance_of(Fog::AWS::Storage::Directories).not_to receive(:create).with(key: directory_key, public: false) subject.ensure_bucket_exists end end context 'the bucket does not exist' do it 'creates the bucket' do - expect_any_instance_of(Fog::Storage::AWS::Directories).to receive(:create).with(key: directory_key, public: false) + expect_any_instance_of(Fog::AWS::Storage::Directories).to receive(:create).with(key: directory_key, public: false) subject.ensure_bucket_exists end end From ffc7e67346ec1b756a697f8b6629a0b30d7c133e Mon Sep 17 00:00:00 2001 From: Seth Boyles Date: Wed, 16 Feb 2022 19:47:39 +0000 Subject: [PATCH 3/3] Bump fog-google Authored-by: Seth Boyles --- Gemfile.lock | 67 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index de770c7730f..5992f8ef933 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -84,18 +84,6 @@ GEM faraday (~> 0.9) faraday_middleware (~> 0.10) nokogiri (~> 1.6) - azure-storage (0.14.0.preview) - azure-core (~> 0.1) - faraday (~> 0.9) - faraday_middleware (~> 0.10) - nokogiri (~> 1.6, >= 1.6.8) - azure-storage-blob (1.1.0) - azure-core (~> 0.1.13) - azure-storage-common (~> 1.0) - nokogiri (~> 1.6, >= 1.6.8) - azure-storage-common (1.1.0) - azure-core (~> 0.1.13) - nokogiri (~> 1.6, >= 1.6.8) azure_mgmt_compute (0.9.0) ms_rest_azure (~> 0.7.0) azure_mgmt_dns (0.9.0) @@ -192,11 +180,18 @@ GEM excon (~> 0.58) formatador (~> 0.2) mime-types - fog-google (1.7.1) - fog-core - fog-json - fog-xml - google-api-client (~> 0.23.0) + fog-google (1.18.0) + fog-core (< 2.3) + fog-json (~> 1.2) + fog-xml (~> 0.1.0) + google-apis-compute_v1 (~> 0.14) + google-apis-dns_v1 (~> 0.12) + google-apis-iamcredentials_v1 (~> 0.6) + google-apis-monitoring_v3 (~> 0.12) + google-apis-pubsub_v1 (~> 0.7) + google-apis-sqladmin_v1beta4 (~> 0.13) + google-apis-storage_v1 (~> 0.6) + google-cloud-env (~> 1.2) fog-json (1.2.0) fog-core multi_json (~> 1.10) @@ -210,14 +205,31 @@ GEM fog-core nokogiri (>= 1.5.11, < 2.0.0) formatador (0.3.0) - google-api-client (0.23.9) + google-apis-compute_v1 (0.25.0) + google-apis-core (>= 0.4, < 2.a) + google-apis-core (0.4.2) addressable (~> 2.5, >= 2.5.1) - googleauth (>= 0.5, < 0.7.0) - httpclient (>= 2.8.1, < 3.0) - mime-types (~> 3.0) + googleauth (>= 0.16.2, < 2.a) + httpclient (>= 2.8.1, < 3.a) + mini_mime (~> 1.0) representable (~> 3.0) - retriable (>= 2.0, < 4.0) - signet (~> 0.9) + retriable (>= 2.0, < 4.a) + rexml + webrick + google-apis-dns_v1 (0.19.0) + google-apis-core (>= 0.4, < 2.a) + google-apis-iamcredentials_v1 (0.10.0) + google-apis-core (>= 0.4, < 2.a) + google-apis-monitoring_v3 (0.21.0) + google-apis-core (>= 0.4, < 2.a) + google-apis-pubsub_v1 (0.12.0) + google-apis-core (>= 0.4, < 2.a) + google-apis-sqladmin_v1beta4 (0.24.0) + google-apis-core (>= 0.4, < 2.a) + google-apis-storage_v1 (0.11.0) + google-apis-core (>= 0.4, < 2.a) + google-cloud-env (1.5.0) + faraday (>= 0.17.3, < 2.0) google-protobuf (3.19.2) google-protobuf (3.19.2-x86_64-darwin) google-protobuf (3.19.2-x86_64-linux) @@ -227,13 +239,13 @@ GEM grpc (~> 1.27) googleapis-common-protos-types (1.3.0) google-protobuf (~> 3.14) - googleauth (0.6.7) - faraday (~> 0.12) + googleauth (1.1.1) + faraday (>= 0.17.3, < 2.0) jwt (>= 1.4, < 3.0) memoist (~> 0.16) multi_json (~> 1.11) os (>= 0.9, < 2.0) - signet (~> 0.7) + signet (>= 0.16, < 2.a) grpc (1.42.0) google-protobuf (~> 3.18) googleapis-common-protos-types (~> 1.0) @@ -300,6 +312,7 @@ GEM mime-types (3.4.1) mime-types-data (~> 3.2015) mime-types-data (3.2022.0105) + mini_mime (1.1.2) mini_portile2 (2.7.1) minitest (5.15.0) ms_rest (0.6.4) @@ -558,8 +571,8 @@ DEPENDENCIES fog-aliyun fog-aws fog-azure-rm! - fog-google (~> 1.7.1) fog-core (~> 2.1.2) + fog-google (~> 1.18.0) fog-local fog-openstack googleapis-common-protos (>= 1.3.12)