From 5c8458b33119819702cf7fd3ffa1b3cb866d9494 Mon Sep 17 00:00:00 2001 From: Milan Dufek Date: Tue, 21 Jan 2025 10:40:23 +0100 Subject: [PATCH 1/5] chore: postgresql driver version update --- ci/postgresql/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/postgresql/pom.xml b/ci/postgresql/pom.xml index 2b6c9632f..4c0451870 100644 --- a/ci/postgresql/pom.xml +++ b/ci/postgresql/pom.xml @@ -12,7 +12,7 @@ org.postgresql postgresql - 42.2.19 + 42.5.0 org.slf4j From f38b0dee97374a405ec07b763156262f6b4d9433 Mon Sep 17 00:00:00 2001 From: Milan Dufek Date: Tue, 21 Jan 2025 10:40:23 +0100 Subject: [PATCH 2/5] test: integration tests for postgresql driver JIRA: GRIF-15 --- .../drivers/postgresql_client_test.rb | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 spec/integration/drivers/postgresql_client_test.rb diff --git a/spec/integration/drivers/postgresql_client_test.rb b/spec/integration/drivers/postgresql_client_test.rb new file mode 100644 index 000000000..7d1a65740 --- /dev/null +++ b/spec/integration/drivers/postgresql_client_test.rb @@ -0,0 +1,97 @@ +# spec/lib/gooddata/cloud_resources/postgresql/postgresql_client_spec.rb + +require 'spec_helper' +require 'gooddata/cloud_resources/postgresql/postgresql_client' +require 'rspec' +require 'csv' +require 'benchmark' + +describe GoodData::CloudResources::PostgresClient do + before do + @postresql_host = ENV['POSTGRES_HOST'] || 'localhost' + @postresql_port = ENV['POSTGRES_PORT'] || 5432 + @postresql_database = ENV['POSTGRES_DB'] || 'postgres' + @postgres_schema = ENV['POSTGRES_SCHEMA'] || 'public' + @postresql_user = ENV['POSTGRES_USER'] || 'postgres' + @postresql_password = ENV['POSTGRES_SECRET'] || 'postgres' + end + + let(:valid_options) do + { + 'postgresql_client' => { + 'connection' => { + 'database' => @postresql_database, + 'schema' => @postgres_schema, + 'authentication' => { + 'basic' => { + 'userName' => @postresql_user, + 'password' => @postresql_password + } + }, + 'sslMode' => 'prefer', + 'url' => "jdbc:postgresql://#{@postresql_host}:#{@postresql_port}/#{@postresql_database}" + } + } + } + end + + describe '.accept?' do + it 'returns true for postgresql type' do + expect(GoodData::CloudResources::PostgresClient.accept?('postgresql')).to be true + end + + it 'returns false for other types' do + expect(GoodData::CloudResources::PostgresClient.accept?('mysql')).to be false + end + end + + describe '#initialize' do + it 'raises an error if postgresql_client is missing' do + options = {} + expect { GoodData::CloudResources::PostgresClient.new(options) }.to raise_error(RuntimeError, "Data Source needs a client to Postgres to be able to query the storage but 'postgresql_client' is empty.") + end + + it 'raises an error if connection info is missing' do + options = { 'postgresql_client' => {} } + expect { GoodData::CloudResources::PostgresClient.new(options) }.to raise_error(RuntimeError, 'Missing connection info for Postgres client') + end + + it 'raises an error if sslMode is invalid' do + invalid_options = valid_options.dup + invalid_options['postgresql_client']['connection']['sslMode'] = 'invalid' + expect { GoodData::CloudResources::PostgresClient.new(invalid_options) }.to raise_error(RuntimeError, 'SSL Mode should be prefer, require and verify-full') + end + + it 'initializes with valid options' do + client = GoodData::CloudResources::PostgresClient.new(valid_options) + expect(client).not_to be_nil + end + end + + describe '#build_url' do + it 'builds the correct URL' do + client = GoodData::CloudResources::PostgresClient.new(valid_options) + expected_url = "jdbc:postgresql://#{@postresql_host}:#{@postresql_port}/#{@postresql_database}?sslmode=prefer" + expect(client.send(:build_url, "jdbc:postgresql://#{@postresql_host}:#{@postresql_port}")).to eq(expected_url) + end + end + + describe '#connect' do + it 'sets up the connection' do + client = GoodData::CloudResources::PostgresClient.new(valid_options) + client.connect + expect(client.instance_variable_get(:@connection)).not_to be_nil + end + end + + describe '#realize_query_version' do + it 'executes the query with postgres version and writes results to a CSV file' do + client = GoodData::CloudResources::PostgresClient.new(valid_options) + client.connect + output_file = client.realize_query('SELECT version();', {}) + expect(File).to exist(output_file) + expect(File.read(output_file)).to include('PostgreSQL') + File.delete(output_file) + end + end +end From df19d9cfcd468706da017346b967302bae50debb Mon Sep 17 00:00:00 2001 From: Milan Dufek Date: Tue, 21 Jan 2025 11:14:08 +0100 Subject: [PATCH 3/5] test: integration tests for mysql driver JIRA: GRIF-15 --- spec/integration/drivers/mysql_client_test.rb | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 spec/integration/drivers/mysql_client_test.rb diff --git a/spec/integration/drivers/mysql_client_test.rb b/spec/integration/drivers/mysql_client_test.rb new file mode 100644 index 000000000..544c69241 --- /dev/null +++ b/spec/integration/drivers/mysql_client_test.rb @@ -0,0 +1,141 @@ +# /Users/milandufek/dev/gooddata/gooddata-ruby/spec/integration/drivers/mysql_client_test.rb + +require 'spec_helper' +require 'gooddata/cloud_resources/mysql/mysql_client' +require 'rspec' +require 'csv' +require 'benchmark' + +describe GoodData::CloudResources::MysqlClient do + before do + @mysql_host = ENV['MYSQL_HOST'] || 'localhost' + @mysql_port = ENV['MYSQL_PORT'] || 3306 + @mysql_database = ENV['MYSQL_DB'] || 'mysql' + @mysql_user = ENV['MYSQL_USER'] || 'root' + @mysql_password = ENV['MYSQL_SECRET'] || 'root' + end + + let(:valid_options) do + { + 'mysql_client' => { + 'connection' => { + 'database' => @mysql_database, + 'schema' => 'public', + 'authentication' => { + 'basic' => { + 'userName' => @mysql_user, + 'password' => @mysql_password + } + }, + 'sslMode' => 'prefer', + 'url' => "jdbc:mysql://#{@mysql_host}:#{@mysql_port}/#{@mysql_database}" + } + } + } + end + + describe '.accept?' do + it 'returns true for mysql type' do + expect(GoodData::CloudResources::MysqlClient.accept?('mysql')).to be true + end + + it 'returns false for other types' do + expect(GoodData::CloudResources::MysqlClient.accept?('postgresql')).to be false + end + end + + describe '#initialize' do + it 'raises an error if mysql_client is missing' do + options = {} + expect { GoodData::CloudResources::MysqlClient.new(options) }.to raise_error(RuntimeError, "Data Source needs a client to Mysql to be able to query the storage but 'mysql_client' is empty.") + end + + it 'raises an error if connection info is missing' do + options = { 'mysql_client' => {} } + expect { GoodData::CloudResources::MysqlClient.new(options) }.to raise_error(RuntimeError, 'Missing connection info for Mysql client') + end + + it 'raises an error if sslMode is invalid' do + invalid_options = valid_options.dup + invalid_options['mysql_client']['connection']['sslMode'] = 'invalid' + expect { GoodData::CloudResources::MysqlClient.new(invalid_options) }.to raise_error(RuntimeError, 'SSL Mode should be prefer, require and verify-full') + end + + it 'initializes with valid options' do + client = GoodData::CloudResources::MysqlClient.new(valid_options) + expect(client).not_to be_nil + end + end + + describe '#build_url' do + it 'builds the correct URL' do + client = GoodData::CloudResources::MysqlClient.new(valid_options) + expected_url = "jdbc:mysql://#{@mysql_host}:#{@mysql_port}/#{@mysql_database}?&useSSL=true&requireSSL=false&verifyServerCertificate=false&useCursorFetch=true&enabledTLSProtocols=TLSv1.2" + expect(client.send(:build_url, "jdbc:mysql://#{@mysql_host}:#{@mysql_port}")).to eq(expected_url) + end + end + + describe '#connect' do + it 'sets up the connection' do + client = GoodData::CloudResources::MysqlClient.new(valid_options) + client.connect + expect(client.instance_variable_get(:@connection)).not_to be_nil + end + end + + describe '#realize_query' do + it 'executes the query and writes results to a CSV file' do + client = GoodData::CloudResources::MysqlClient.new(valid_options) + client.connect + output_file = client.realize_query('SELECT 123;', {}) + expect(File).to exist(output_file) + expect(File.read(output_file)).to include('123') + File.delete(output_file) + end + end + + describe '#fetch_size' do + it 'returns the correct fetch size for MySQL' do + client = GoodData::CloudResources::MysqlClient.new(valid_options) + expect(client.fetch_size).to eq(GoodData::CloudResources::MysqlClient::MYSQL_FETCH_SIZE) + end + + it 'returns the correct fetch size for MongoDB BI Connector' do + mongo_options = valid_options.dup + mongo_options['mysql_client']['connection']['databaseType'] = GoodData::CloudResources::MysqlClient::MONGO_BI_TYPE + client = GoodData::CloudResources::MysqlClient.new(mongo_options) + expect(client.fetch_size).to eq(GoodData::CloudResources::MysqlClient::MONGO_BI_FETCH_SIZE) + end + end + + describe '#get_ssl_mode' do + it 'returns the correct SSL mode for prefer' do + client = GoodData::CloudResources::MysqlClient.new(valid_options) + expect(client.send(:get_ssl_mode, 'prefer')).to eq(GoodData::CloudResources::MysqlClient::PREFER) + end + + it 'returns the correct SSL mode for require' do + client = GoodData::CloudResources::MysqlClient.new(valid_options) + expect(client.send(:get_ssl_mode, 'require')).to eq(GoodData::CloudResources::MysqlClient::REQUIRE) + end + + it 'returns the correct SSL mode for verify-full' do + client = GoodData::CloudResources::MysqlClient.new(valid_options) + expect(client.send(:get_ssl_mode, 'verify-full')).to eq(GoodData::CloudResources::MysqlClient::VERIFY_FULL) + end + end + + describe '#add_extended' do + it 'returns the correct extended parameters for MongoDB BI Connector' do + mongo_options = valid_options.dup + mongo_options['mysql_client']['connection']['databaseType'] = GoodData::CloudResources::MysqlClient::MONGO_BI_TYPE + client = GoodData::CloudResources::MysqlClient.new(mongo_options) + expect(client.send(:add_extended)).to eq('&authenticationPlugins=org.mongodb.mongosql.auth.plugin.MongoSqlAuthenticationPlugin&useLocalTransactionState=true') + end + + it 'returns an empty string for MySQL' do + client = GoodData::CloudResources::MysqlClient.new(valid_options) + expect(client.send(:add_extended)).to eq('') + end + end +end From eb1994b01d14dbfa67c5bb2ade4e178836e8678e Mon Sep 17 00:00:00 2001 From: Milan Dufek Date: Tue, 21 Jan 2025 13:07:59 +0100 Subject: [PATCH 4/5] test: integration tests for mssql driver JIRA: GRIF-15 --- spec/integration/drivers/mssql_client_test.rb | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 spec/integration/drivers/mssql_client_test.rb diff --git a/spec/integration/drivers/mssql_client_test.rb b/spec/integration/drivers/mssql_client_test.rb new file mode 100644 index 000000000..ec080d135 --- /dev/null +++ b/spec/integration/drivers/mssql_client_test.rb @@ -0,0 +1,70 @@ +require 'spec_helper' +require 'gooddata/cloud_resources/mssql/mssql_client' +require 'rspec' + +describe GoodData::CloudResources::MSSQLClient do + before do + @mssql_host = ENV['MSSQL_HOST'] || 'localhost' + @mssql_port = ENV['MSSQL_PORT'] || 1433 + @mssql_database = ENV['MSSQL_DB'] || 'master' + @mssql_user = ENV['MSSQL_USER'] || 'sa' + @mssql_password = ENV['MSSQL_PASSWORD'] || 'Password123' + end + + let(:valid_options) do + { + 'mssql_client' => { + 'connection' => { + 'database' => @mssql_database, + 'schema' => 'dbo', + 'authentication' => { + 'basic' => { + 'userName' => @mssql_user, + 'password' => @mssql_password + } + }, + 'sslMode' => 'prefer', + 'url' => "jdbc:sqlserver://#{@mssql_host}:#{@mssql_port}" + } + } + } + end + + describe '.accept?' do + it 'returns true for mssql type' do + expect(GoodData::CloudResources::MSSQLClient.accept?('mssql')).to be true + end + + it 'returns false for other types' do + expect(GoodData::CloudResources::MSSQLClient.accept?('mysql')). to be false + end + end + + describe '#build_connection_string' do + it 'builds a valid connection string' do + client = GoodData::CloudResources::MSSQLClient.new(valid_options) + connection_string = client.send(:build_connection_string) + expect(connection_string).to eq("jdbc:sqlserver://#{@mssql_host}:#{@mssql_port};database=#{@mssql_database};"\ + "encrypt=false;trustServerCertificate=false;loginTimeout=30;") + end + end + + describe '#initialize' do + it 'raises an error if mssql_client is missing' do + expect { GoodData::CloudResources::MSSQLClient.new({}) }.to raise_error(RuntimeError, "Data Source needs a client to MSSQL to be able to query the storage but 'mssql_client' is empty.") + end + + it 'initializes with valid options' do + client = GoodData::CloudResources::MSSQLClient.new(valid_options) + expect(client).to be_an_instance_of(GoodData::CloudResources::MSSQLClient) + end + + it 'connects to SQL Server and selects the version' do + client = GoodData::CloudResources::MSSQLClient.new(valid_options) + version_csv = client.realize_query('SELECT @@VERSION', nil) + expect(File).to exist(version_csv) + expect(File.read(version_csv)).to include('Microsoft SQL Server') + File.delete(version_csv) + end + end +end From 4d794027bacdb61df47ec698f9eccc4a5b770809 Mon Sep 17 00:00:00 2001 From: Milan Dufek Date: Wed, 29 Jan 2025 15:53:42 +0100 Subject: [PATCH 5/5] test: install maven dependencies for DB drivers JIRA: GRIF-15 --- Dockerfile.jruby | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Dockerfile.jruby b/Dockerfile.jruby index 48c02aa1c..e02d0818f 100644 --- a/Dockerfile.jruby +++ b/Dockerfile.jruby @@ -2,7 +2,7 @@ FROM jruby:9.4.1.0 MAINTAINER Tomas Korcak -RUN apt-get update && apt-get install -y curl make gcc git g++ python binutils-gold gnupg libstdc++6 cmake +RUN apt-get update && apt-get install -y curl make gcc git g++ python binutils-gold gnupg libstdc++6 cmake maven # Switch to directory with sources WORKDIR /src @@ -15,6 +15,18 @@ RUN gem update --system \ ADD . . +# build postgresql dependencies +RUN mvn -f ci/postgresql/pom.xml clean install -P binary-packaging \ + && cp -rf ci/postgresql/target/*.jar ./lib/gooddata/cloud_resources/postgresql/drivers/ + +# build mssql dependencies +RUN mvn -f ci/mssql/pom.xml clean install -P binary-packaging \ + && cp -rf ci/mssql/target/*.jar ./lib/gooddata/cloud_resources/mssql/drivers/ + +# build mysql dependencies +RUN mvn -f ci/mysql/pom.xml clean install -P binary-packaging \ + && cp -rf ci/mysql/target/*.jar ./lib/gooddata/cloud_resources/mysql/drivers/ + # Import GoodData certificate to Java. This is needed for connection to ADS. # https://jira.intgdc.com/browse/TMA-300 RUN keytool -importcert -alias gooddata-2008 -file "./data/2008.crt" -keystore $JAVA_HOME/lib/security/cacerts -trustcacerts -storepass 'changeit' -noprompt