[DRAFT] initial refactor to make integration-e2e run

.gitignore

@@ -45,3 +45,7 @@ deprecations.txt
 
 # test cache files
 spec/cache/
+
+# local/ci execution
+lib/gooddata/cloud_resources/*/*/*.jar
+ci/*/target/

docker-compose.lcm.yml

@@ -1,12 +1,11 @@
-version: '2'
+version: '3.8'
 services:
   appstore:
     image: gooddata/appstore
     build:
       context: .
       dockerfile: Dockerfile.jruby
-    links:
-     - localstack
+    working_dir: /src
     environment:
      - GD_ENV
      - GDC_LOG_LEVEL
@@ -57,16 +56,17 @@ services:
      - GD_STG_DEFAULT_PASSWORD
     volumes:
       - .:/src
-    volumes_from:
-      - bundle
-    mem_limit: 2500m
+      - bundle_cache:/src/bundle
+    deploy:
+      resources:
+        limits:
+          memory: 2500m
   localstack:
     image: hahihula/localstack_dockerfile
     expose:
      - "4572"
     environment:
      - SERVICES=s3:4572
-  bundle:
-    image: busybox
-    volumes:
-      - /bundle
+
+volumes:
+  bundle_cache:

gooddata.gemspec

@@ -56,7 +56,7 @@ Gem::Specification.new do |s|
     s.add_dependency 'unf', '~> 0.1.4'
   end
   s.add_development_dependency 'simplecov', '~> 0.12'
-  s.add_development_dependency 'webmock', '~> 2.3.1'
+  s.add_development_dependency 'webmock', '~> 3.26.0'
   s.add_development_dependency 'yard', '~> 0.9.11'
   s.add_development_dependency 'yard-rspec', '~> 0.1'
   s.add_development_dependency 'pry'
@@ -68,7 +68,12 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'vcr', '5.0.0'
   s.add_development_dependency 'hashdiff', '~> 0.4'
 
-  s.add_development_dependency 'sqlite3' if RUBY_PLATFORM != 'java'
+  if RUBY_PLATFORM == 'java'
+    s.add_development_dependency 'activerecord-jdbcsqlite3-adapter'
+    s.add_development_dependency 'activerecord'
+  else
+    s.add_development_dependency 'sqlite3'
+  end
 
   if RUBY_VERSION >= '2.8'
     s.add_dependency 'activesupport', '>= 6.0.3.1'

lib/gooddata/rest/connection.rb

@@ -177,6 +177,8 @@ def initialize(opts)
 
       # Connect using username and password
       def connect(username, password, options = {})
+        Psych::Parser.code_point_limit = 10_000_000
+
         server = options[:server] || Helpers::AuthHelper.read_server
         options = DEFAULT_LOGIN_PAYLOAD.merge(options)
         headers = options[:headers] || {}

spec/environment/environment.rb

@@ -49,6 +49,7 @@ def initial_secrets(env)
           s3_bucket_name: ENV['RT_S3_BUCKET_NAME'],
           s3_access_key_id: ENV['RT_S3_ACCESS_KEY'],
           s3_secret_access_key: ENV['RT_S3_SECRET_KEY'],
+          s3_session_token: ENV['RT_S3_SESSION_TOKEN'],
           redshift_password: ENV['REDSHIFT_PASSWORD'],
           redshift_access_key: ENV['REDSHIFT_ACCESS_KEY'],
           redshift_secret_key: ENV['REDSHIFT_SECRET_KEY'],

spec/lcm/integration/support/in_memory_ads.rb

@@ -1,11 +1,26 @@
-require 'sqlite3' unless RUBY_PLATFORM == 'java'
+if RUBY_PLATFORM == 'java'
+  require 'active_record'
+  require 'activerecord-jdbcsqlite3-adapter'
+else
+  require 'sqlite3'
+end
 
 module Support
   class InMemoryAds
     def initialize
-      db = SQLite3::Database.new(':memory:')
-      db.results_as_hash = true
-      @db = db
+      if RUBY_PLATFORM == 'java'
+        # Use ActiveRecord with JDBC adapter for JRuby
+        ActiveRecord::Base.establish_connection(
+          adapter: 'sqlite3',
+          database: ':memory:'
+        )
+        @db = ActiveRecord::Base.connection
+      else
+        # Use sqlite3 gem for MRI
+        db = SQLite3::Database.new(':memory:')
+        db.results_as_hash = true
+        @db = db
+      end
     end
 
     def data
@@ -33,12 +48,30 @@ def execute(*args, &block)
     end
 
     def execute_with_headers(*args)
-      res = @db.execute(*args)
-      res.map do |row|
-        # sqlite3 returns hash with both column names and numbers, we want only names
-        res = GoodData::Helpers.symbolize_keys(row.reject { |k, _| k.is_a? Integer })
-        yield res if block_given?
-        res
+      if RUBY_PLATFORM == 'java'
+        # ActiveRecord JDBC adapter
+        result = @db.exec_query(*args)
+        # ActiveRecord returns arrays of arrays, convert to hash format
+        columns = result.columns if result.respond_to?(:columns)
+        result.map do |row|
+          if row.is_a?(Array) && columns
+            # Convert array to hash
+            row_hash = columns.zip(row).to_h
+            res = GoodData::Helpers.symbolize_keys(row_hash)
+          else
+            res = GoodData::Helpers.symbolize_keys(row.is_a?(Hash) ? row : {})
+          end
+          yield res if block_given?
+          res
+        end
+      else
+        res = @db.execute(*args)
+        res.map do |row|
+          # sqlite3 returns hash with both column names and numbers, we want only names
+          res = GoodData::Helpers.symbolize_keys(row.reject { |k, _| k.is_a? Integer })
+          yield res if block_given?
+          res
+        end
       end
     end
 

spec/lcm/integration/support/s3_helper.rb

@@ -1,7 +1,7 @@
 module Support
   class S3Helper
     LOCALSTACK_ENDPOINT = 'http://localstack:4572'.freeze
-    S3_ENDPOINT = 'https://s3.amazonaws.com'.freeze
+    S3_ENDPOINT = 'https://%s.s3.amazonaws.com'.freeze
     USER_FILTERS_KEY = 'user_filters'
     USERS_KEY = 'users_brick_input'
     REGION = 'us-east-1'
@@ -14,15 +14,16 @@ def upload_file(file, object_name)
         s3_endpoint = if GoodData::Environment::LOCALSTACK
                         Support::S3Helper::LOCALSTACK_ENDPOINT
                       else
-                        Support::S3Helper::S3_ENDPOINT
+                        Support::S3Helper::S3_ENDPOINT % bucket_name
                       end
 
         s3 = Aws::S3::Resource.new(
           access_key_id: SECRETS[:s3_access_key_id],
           secret_access_key: SECRETS[:s3_secret_access_key],
+          session_token: SECRETS[:s3_session_token],
           endpoint: s3_endpoint,
           region: REGION,
-          force_path_style: true
+          force_path_style: true,
         )
 
         bucket = s3.bucket(bucket_name)
@@ -34,7 +35,8 @@ def upload_file(file, object_name)
           s3_bucket: bucket_name,
           s3_endpoint: s3_endpoint,
           s3_access_key: SECRETS[:s3_access_key_id],
-          s3_secret_access_key: SECRETS[:s3_secret_access_key]
+          s3_secret_access_key: SECRETS[:s3_secret_access_key],
+          s3_session_token: SECRETS[:s3_session_token]
         }
       end
     end

spec/vcr_configurer.rb

@@ -43,8 +43,8 @@ def self.setup
           vcr_config.allow_http_connections_when_no_cassette = true
           vcr_config.configure_rspec_metadata!
 
-          vcr_config.ignore_request do
-            @ignore_vcr_requests
+          vcr_config.ignore_request do |request|
+            @ignore_vcr_requests || request.uri.include?('s3.amazonaws.com')
           end
 
           vcr_config.default_cassette_options = {