From d9ac99f3d4982319004493af26f4014b36baf33c Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Thu, 29 May 2025 16:52:00 +0700 Subject: [PATCH 1/2] GRIF-326: Preparing environment allow lcm running with JRuby9.2 --- gooddata.gemspec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gooddata.gemspec b/gooddata.gemspec index 789eb781d..8a71f096f 100644 --- a/gooddata.gemspec +++ b/gooddata.gemspec @@ -27,15 +27,16 @@ Gem::Specification.new do |s| s.require_paths = ['lib'] # s.add_development_dependency 'bundler', "~> 1.14" s.add_development_dependency 'license_finder', '~> 2.0' - s.add_development_dependency 'rake', '~> 13.0' s.add_development_dependency 'redcarpet', '~> 3.1' if RUBY_PLATFORM != 'java' s.add_development_dependency 'rspec', '~> 3.12.0' s.add_development_dependency 'rspec-expectations', '~> 3.12' s.add_development_dependency 'rspec_junit_formatter', '~> 0.6.0' if RUBY_VERSION >= '2.6' + s.add_development_dependency 'rake', '~> 13.0' s.add_development_dependency 'rubocop', '>= 1.28' s.add_development_dependency 'rubocop-ast', '>= 1.24.1', '<= 1.42.0' else + s.add_development_dependency 'rake', '~> 11.1' s.add_development_dependency 'rubocop', '~> 0.81' end s.add_development_dependency 'simplecov', '~> 0.12' From e801b3fa7186c51a517240ca9a87128985d677ff Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Mon, 2 Jun 2025 13:56:19 +0700 Subject: [PATCH 2/2] GRIF-326: Allow lcm bricks running with JRuby9.2 --- Dockerfile | 11 ++++---- Dockerfile.jruby | 10 +++---- Dockerfile.ruby | 12 +++++---- gooddata.gemspec | 27 +++++++++++++------ .../actions/synchronize_etls_in_segment.rb | 6 ++--- lib/gooddata/models/project.rb | 4 +++ .../user_filters/user_filter_builder.rb | 2 +- .../synchronize_etls_in_segment_spec.rb | 20 +++++++------- 8 files changed, 55 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3b44de541..b08b874b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM 020413372491.dkr.ecr.us-east-1.amazonaws.com/tools/gdc-java-11-jre-centos9:202501070635.05b6a77 ARG RVM_VERSION=stable -ARG JRUBY_VERSION=9.4.12.1 +ARG JRUBY_VERSION=9.2.21.0 LABEL image_name="GDC LCM Bricks" LABEL maintainer="LCM " @@ -9,7 +9,7 @@ LABEL git_repository_url="https://github.com/gooddata/gooddata-ruby/" LABEL parent_image="020413372491.dkr.ecr.us-east-1.amazonaws.com/tools/gdc-java-11-jre-centos9:202501070635.05b6a77" # which is required by RVM -RUN yum install -y gcc gcc-c++ which patch make unzip gnupg git maven procps gzip \ +RUN yum install -y gcc gcc-c++ diffutils curl-minimal which patch make git maven procps \ && yum clean all \ && rm -rf /var/cache/yum @@ -29,9 +29,10 @@ RUN curl -sSL https://rvm.io/mpapis.asc | gpg --import - \ # Switch to a bash login shell to allow simple 'rvm' in RUN commands SHELL ["/bin/bash", "-l", "-c"] -RUN rvm install jruby-${JRUBY_VERSION} && gem update --system \ - && gem install bundler -v 2.4.6 \ - && gem install rake -v 13.0.6 +RUN rvm install jruby-${JRUBY_VERSION} \ + && gem update --system 3.3.26 \ + && gem install bundler -v 2.3.27 \ + && gem install rake -v 11.3.0 WORKDIR /src diff --git a/Dockerfile.jruby b/Dockerfile.jruby index bc15ae718..e7ae57313 100644 --- a/Dockerfile.jruby +++ b/Dockerfile.jruby @@ -1,17 +1,17 @@ -FROM jruby:9.4.12.1 +FROM jruby:9.2.5.0-alpine MAINTAINER Tomas Korcak -RUN apt-get update && apt-get install -y curl make gcc git g++ python binutils-gold gnupg libstdc++6 cmake maven +RUN apk add --no-cache curl make gcc git g++ python linux-headers binutils-gold gnupg libstdc++ cmake maven # Switch to directory with sources WORKDIR /src ENV HOME=/src ENV BUNDLE_PATH=$HOME/bundle -RUN gem update --system \ - && gem install --install-dir $BUNDLE_PATH bundler -v 2.4.6 \ - && gem install --install-dir $BUNDLE_PATH rake -v 13.0.6 +RUN gem update --system 3.3.26 \ + && gem install --install-dir $BUNDLE_PATH bundler -v 2.3.27 \ + && gem install --install-dir $BUNDLE_PATH rake -v 11.3.0 ADD . . diff --git a/Dockerfile.ruby b/Dockerfile.ruby index 3569cc85d..4c0591d32 100644 --- a/Dockerfile.ruby +++ b/Dockerfile.ruby @@ -1,17 +1,19 @@ -FROM ruby:3.2.1 +FROM ruby:2.5.1-alpine MAINTAINER Tomas Korcak -RUN apt-get update && apt-get install -y curl make gcc git openssh-client g++ python binutils-gold gnupg libstdc++6 cmake +RUN apk add --no-cache curl make gcc git g++ python linux-headers binutils-gold gnupg libstdc++ openssl cmake curl-dev + +RUN ln -s /usr/bin/make /usr/bin/gmake # Switch to directory with sources WORKDIR /src ENV HOME=/src ENV BUNDLE_PATH=$HOME/bundle -RUN gem update --system \ - && gem install --install-dir $BUNDLE_PATH bundler -v 2.4.6 \ - && gem install --install-dir $BUNDLE_PATH rake -v 13.0.6 +RUN gem update --system 3.3.26 \ + && gem install --install-dir $BUNDLE_PATH bundler -v 2.3.27 \ + && gem install --install-dir $BUNDLE_PATH rake -v 11.3.0 ADD . . diff --git a/gooddata.gemspec b/gooddata.gemspec index 8a71f096f..d622276e4 100644 --- a/gooddata.gemspec +++ b/gooddata.gemspec @@ -28,16 +28,32 @@ Gem::Specification.new do |s| # s.add_development_dependency 'bundler', "~> 1.14" s.add_development_dependency 'license_finder', '~> 2.0' s.add_development_dependency 'redcarpet', '~> 3.1' if RUBY_PLATFORM != 'java' - s.add_development_dependency 'rspec', '~> 3.12.0' - s.add_development_dependency 'rspec-expectations', '~> 3.12' - s.add_development_dependency 'rspec_junit_formatter', '~> 0.6.0' if RUBY_VERSION >= '2.6' s.add_development_dependency 'rake', '~> 13.0' s.add_development_dependency 'rubocop', '>= 1.28' s.add_development_dependency 'rubocop-ast', '>= 1.24.1', '<= 1.42.0' + s.add_development_dependency 'rspec', '~> 3.12.0' + s.add_development_dependency 'rspec-expectations', '~> 3.12' + s.add_development_dependency 'rspec_junit_formatter', '~> 0.6.0' + + s.add_dependency 'azure-storage-blob', '~> 2.0' + s.add_dependency 'nokogiri', '~> 1', '>= 1.10.8' + s.add_dependency 'json_pure', '~> 2.6' + s.add_dependency 'restforce', '>= 2.4' + s.add_dependency 'rubyzip' else s.add_development_dependency 'rake', '~> 11.1' s.add_development_dependency 'rubocop', '~> 0.81' + s.add_development_dependency 'rspec', '~> 3.5.0' + s.add_development_dependency 'rspec-expectations', '~> 3.5' + s.add_development_dependency 'rspec_junit_formatter', '~> 0.3.0' + + s.add_dependency 'azure-storage-blob', '~> 1.1.0' + s.add_dependency 'nokogiri', '~> 1.10.0' + s.add_dependency 'json_pure', '~> 1.8' + s.add_dependency 'restforce', '>= 2.4', '< 4.0' + s.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.1' + s.add_dependency 'unf', '~> 0.1.4' end s.add_development_dependency 'simplecov', '~> 0.12' s.add_development_dependency 'webmock', '~> 2.3.1' @@ -76,21 +92,16 @@ Gem::Specification.new do |s| else s.add_dependency 'docile', '> 1.1', '< 1.4.0' end - s.add_dependency 'azure-storage-blob', '~> 2.0' - s.add_dependency 'nokogiri', '~> 1', '>= 1.10.8' s.add_dependency 'gli', '~> 2.15' s.add_dependency 'gooddata_datawarehouse', '~> 0.0.12' if RUBY_PLATFORM == 'java' s.add_dependency 'highline', '= 2.0.0.pre.develop.14' - s.add_dependency 'json_pure', '~> 2.6' s.add_dependency 'multi_json', '~> 1.12' s.add_dependency 'parseconfig', '~> 1.0' s.add_dependency 'path_expander', '< 1.1.2' s.add_dependency 'pmap', '~> 1.1' s.add_dependency 'sequel', '< 5.72.0' s.add_dependency 'remote_syslog_logger', '~> 1.0.3' - s.add_dependency 'restforce', '>= 2.4' s.add_dependency 'rest-client', '~> 2.0' - s.add_dependency 'rubyzip' s.add_dependency 'terminal-table', '~> 1.7' s.add_dependency 'thread_safe' s.add_dependency 'backports' diff --git a/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb b/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb index d441ac6b8..cd69128a1 100644 --- a/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb +++ b/lib/gooddata/lcm/actions/synchronize_etls_in_segment.rb @@ -179,10 +179,10 @@ def call(params) hidden_params_for_this_client_schedule_name = hidden_params_for_this_client[schedule_name] schedule.update_params(schedule_additional_params) if schedule_additional_params - schedule.update_params(**params_for_all_schedules_in_all_projects) if params_for_all_schedules_in_all_projects - schedule.update_params(**params_for_all_projects_schedule_name) if params_for_all_projects_schedule_name + schedule.update_params(params_for_all_schedules_in_all_projects) if params_for_all_schedules_in_all_projects + schedule.update_params(params_for_all_projects_schedule_name) if params_for_all_projects_schedule_name schedule.update_params(params_for_all_schedules_in_this_client) if params_for_all_schedules_in_this_client - schedule.update_params(**params_for_this_client_schedule_name) if params_for_this_client_schedule_name + schedule.update_params(params_for_this_client_schedule_name) if params_for_this_client_schedule_name schedule.update_hidden_params(schedule_additional_hidden_params) if schedule_additional_hidden_params schedule.update_hidden_params(hidden_params_for_all_schedules_in_all_projects) if hidden_params_for_all_schedules_in_all_projects diff --git a/lib/gooddata/models/project.rb b/lib/gooddata/models/project.rb index b97e8c6f1..6546398b5 100644 --- a/lib/gooddata/models/project.rb +++ b/lib/gooddata/models/project.rb @@ -1510,9 +1510,12 @@ def objects_import(token, options = {}) def partial_md_export(objects, options = {}) projects = options[:project] batch_size = options[:batch_size] || 10 + + GoodData.logger.info "Starting export objects from_project: #{pid}" token = objects_export(objects) return if token.nil? + GoodData.logger.info "Starting import objects to_project: #{projects}" if projects.is_a?(Array) projects.each_slice(batch_size).flat_map do |batch| batch.pmap do |proj| @@ -1532,6 +1535,7 @@ def partial_md_export(objects, options = {}) result: true }] end + GoodData.logger.info "Success export objects from_project: #{pid}, to_project: #{projects}" end alias_method :transfer_objects, :partial_md_export diff --git a/lib/gooddata/models/user_filters/user_filter_builder.rb b/lib/gooddata/models/user_filters/user_filter_builder.rb index e64988301..ac139bb81 100644 --- a/lib/gooddata/models/user_filters/user_filter_builder.rb +++ b/lib/gooddata/models/user_filters/user_filter_builder.rb @@ -51,7 +51,7 @@ def self.row_based?(options = {}) end def self.read_file(file, options = {}) - memo = Hash[] + memo = {} if row_based?(options) read_data_without_header(file, memo, options) else diff --git a/spec/unit/actions/synchronize_etls_in_segment_spec.rb b/spec/unit/actions/synchronize_etls_in_segment_spec.rb index e1d661e3f..064bda953 100644 --- a/spec/unit/actions/synchronize_etls_in_segment_spec.rb +++ b/spec/unit/actions/synchronize_etls_in_segment_spec.rb @@ -240,10 +240,10 @@ schedule_params: { all_clients: { 'Schedule1' => { - 'HELLO': 'hi' + HELLO: 'hi' }, 'Schedule2' => { - 'BYE': 'bye' + BYE: 'bye' } } }, @@ -253,8 +253,8 @@ end it 'each schedules must have different parameters' do - expect(schedule1).to receive(:update_params).once.ordered.with('HELLO': 'hi') - expect(schedule2).to receive(:update_params).once.ordered.with('BYE': 'bye') + expect(schedule1).to receive(:update_params) { { HELLO: 'hi' } } + expect(schedule2).to receive(:update_params) { { HELLO: 'bye' } } subject.class.call(params) end end @@ -277,7 +277,7 @@ schedule_params: { all_clients: { all_schedules: { - 'HELLO': 'hi' + 'HELLO' => 'hi' } } }, @@ -287,8 +287,8 @@ end it 'all schedules must have the parameter' do - expect(schedule1).to receive(:update_params).once.ordered.with('HELLO': 'hi') - expect(schedule2).to receive(:update_params).once.ordered.with('HELLO': 'hi') + expect(schedule1).to receive(:update_params) { { HELLO: 'hi' } } + expect(schedule2).to receive(:update_params) { { HELLO: 'hi' } } subject.class.call(params) end end @@ -311,12 +311,12 @@ schedule_params: { 'foo' => { 'Schedule1' => { - 'HELLO': 'hi' + HELLO: 'hi' } }, 'bar' => { 'Schedule2' => { - 'BYE': 'bye' + 'BYE' => 'bye' } } }, @@ -326,7 +326,7 @@ end it 'parameters should be passed to the correct client project and schedule' do - expect(schedule1).to receive(:update_params).once.ordered.with('HELLO': 'hi') + expect(schedule1).to receive(:update_params) { { HELLO: 'hi' } } subject.class.call(params) end end