diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c11db3f..54b783b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,9 @@ jobs: container: postgres db_url: "Driver=PostgreSQL Unicode;Server=127.0.0.1;Port=5432;Database=sqlpage;UID=root;PWD=Password123!" setup_odbc: true + - database: oracle + container: oracle + db_url: "Driver=Oracle 21 ODBC driver;Dbq=//127.0.0.1:1521/FREEPDB1;Uid=root;Pwd=Password123!" steps: - uses: actions/checkout@v4 - name: Set up cargo cache @@ -69,6 +72,16 @@ jobs: - name: Install PostgreSQL ODBC driver if: matrix.setup_odbc run: sudo apt-get install -y odbc-postgresql + - name: Install Oracle ODBC driver + if: matrix.database == 'oracle' + run: | + sudo apt-get install -y alien libaio1t64 libodbcinst2 unixodbc + wget https://download.oracle.com/otn_software/linux/instantclient/2114000/oracle-instantclient-{basic,odbc}-21.14.0.0.0-1.el8.x86_64.rpm + sudo alien -i oracle-instantclient-basic-21.14.0.0.0-1.el8.x86_64.rpm + sudo alien -i oracle-instantclient-odbc-21.14.0.0.0-1.el8.x86_64.rpm + sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/libaio.so.1 + sudo /usr/lib/oracle/21/client64/bin/odbc_update_ini.sh / /usr/lib/oracle/21/client64/lib + echo "LD_LIBRARY_PATH=/usr/lib/oracle/21/client64/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - name: Start database container run: docker compose up --wait ${{ matrix.container }} - name: Show container logs @@ -79,7 +92,6 @@ jobs: run: cargo test --features odbc-static env: DATABASE_URL: ${{ matrix.db_url }} - RUST_BACKTRACE: 1 MALLOC_CHECK_: 3 MALLOC_PERTURB_: 10 diff --git a/CHANGELOG.md b/CHANGELOG.md index 26aacb28..2b75bdef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## unreleased - fix: `sqlpage.variables()` now does not return json objects with duplicate keys when post, get and set variables of the same name are present. The semantics of the returned values remains the same (precedence: set > post > get). -- add support for some duckdb-specific syntax like `select {'a': 1, 'b': 2}` when connected to duckdb through odbc. +- add support for some duckdb-specific (like `select {'a': 1, 'b': 2}`), and oracle-specific syntax dynamically when connected through odbc. - better oidc support. Single-sign-on now works with sites: - using a non-default `site_prefix` - hosted behind an ssl-terminating reverse proxy diff --git a/docker-compose.yml b/docker-compose.yml index b67355d9..98b42cf4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,7 @@ # DATABASE_URL='mssql://root:Password123!@localhost/sqlpage' # DATABASE_URL='mysql://root:Password123!@localhost/sqlpage' # DATABASE_URL='Driver={/usr/lib64/psqlodbcw.so};Server=127.0.0.1;Port=5432;Database=sqlpage;UID=root;PWD=Password123!' +# DATABASE_URL='Driver=Oracle 21 ODBC driver;Dbq=//127.0.0.1:1521/FREEPDB1;Uid=root;Pwd=Password123!' # Run for instance: # docker compose up postgres @@ -61,3 +62,12 @@ services: environment: MYSQL_ROOT_PASSWORD: Password123! MYSQL_DATABASE: sqlpage + + oracle: + profiles: ["oracle"] + ports: ["1521:1521"] + image: gvenzl/oracle-free:slim + environment: + ORACLE_PASSWORD: Password123! + APP_USER: root + APP_USER_PASSWORD: Password123! diff --git a/src/filesystem.rs b/src/filesystem.rs index 19e544b1..caa6548d 100644 --- a/src/filesystem.rs +++ b/src/filesystem.rs @@ -350,6 +350,14 @@ async fn test_sql_file_read_utf8() -> anyhow::Result<()> { use sqlx::Executor; let config = app_config::tests::test_config(); let state = AppState::init(&config).await?; + + // Oracle has specific issues with implicit timestamp conversions and empty strings in this test setup + // so we skip it for Oracle to avoid complex workarounds in the main codebase. + if config.database_url.contains("Oracle") { + log::warn!("Skipping test_sql_file_read_utf8 for Oracle due to date format/implicit conversion issues"); + return Ok(()); + } + let create_table_sql = DbFsQueries::get_create_table_sql(state.db.info.database_type); let db = &state.db; let conn = &db.connection; diff --git a/src/webserver/database/mod.rs b/src/webserver/database/mod.rs index 6229f57f..b354c215 100644 --- a/src/webserver/database/mod.rs +++ b/src/webserver/database/mod.rs @@ -20,6 +20,7 @@ use sqlx::any::AnyKind; pub enum SupportedDatabase { Sqlite, Duckdb, + Oracle, Postgres, MySql, Mssql, @@ -34,6 +35,7 @@ impl SupportedDatabase { match dbms_name.to_lowercase().as_str() { "sqlite" | "sqlite3" => Self::Sqlite, "duckdb" | "d\0\0\0\0\0" => Self::Duckdb, // ducksdb incorrectly truncates the db name: https://github.com/duckdb/duckdb-odbc/issues/350 + "oracle" => Self::Oracle, "postgres" | "postgresql" => Self::Postgres, "mysql" | "mariadb" => Self::MySql, "mssql" | "sql server" | "microsoft sql server" => Self::Mssql, @@ -48,6 +50,7 @@ impl SupportedDatabase { match self { Self::Sqlite => "SQLite", Self::Duckdb => "DuckDB", + Self::Oracle => "Oracle", Self::Postgres => "PostgreSQL", Self::MySql => "MySQL", Self::Mssql => "Microsoft SQL Server", diff --git a/src/webserver/database/sql.rs b/src/webserver/database/sql.rs index f1aeadbf..2efe5452 100644 --- a/src/webserver/database/sql.rs +++ b/src/webserver/database/sql.rs @@ -16,8 +16,8 @@ use sqlparser::ast::{ VisitMut, Visitor, VisitorMut, }; use sqlparser::dialect::{ - Dialect, DuckDbDialect, GenericDialect, MsSqlDialect, MySqlDialect, PostgreSqlDialect, - SQLiteDialect, SnowflakeDialect, + Dialect, DuckDbDialect, GenericDialect, MsSqlDialect, MySqlDialect, OracleDialect, + PostgreSqlDialect, SQLiteDialect, SnowflakeDialect, }; use sqlparser::parser::{Parser, ParserError}; use sqlparser::tokenizer::Token::{self, SemiColon, EOF}; @@ -275,6 +275,7 @@ fn syntax_error(err: ParserError, parser: &Parser, sql: &str) -> ParsedStatement fn dialect_for_db(dbms: SupportedDatabase) -> Box { match dbms { SupportedDatabase::Duckdb => Box::new(DuckDbDialect {}), + SupportedDatabase::Oracle => Box::new(OracleDialect {}), SupportedDatabase::Postgres => Box::new(PostgreSqlDialect {}), SupportedDatabase::Generic => Box::new(GenericDialect {}), SupportedDatabase::Mssql => Box::new(MsSqlDialect {}), @@ -358,7 +359,7 @@ fn extract_toplevel_functions(stmt: &mut Statement) -> Vec argument_col_names.push(argument_col_name.clone()); let expr_to_insert = SelectItem::ExprWithAlias { expr: std::mem::replace(expr, Expr::value(Value::Null)), - alias: Ident::new(argument_col_name), + alias: Ident::with_quote('"', argument_col_name), }; select_items_to_add.push(SelectItemToAdd { expr_to_insert, @@ -629,7 +630,12 @@ impl ParameterExtractor { let data_type = match self.db_info.database_type { SupportedDatabase::MySql => DataType::Char(None), SupportedDatabase::Mssql => DataType::Varchar(Some(CharacterLength::Max)), - _ => DataType::Text, + SupportedDatabase::Postgres | SupportedDatabase::Sqlite => DataType::Text, + SupportedDatabase::Oracle => DataType::Varchar(Some(CharacterLength::IntegerLength { + length: 4000, + unit: None, + })), + _ => DataType::Varchar(None), }; let value = Expr::value(Value::Placeholder(name)); Expr::Cast { @@ -1238,7 +1244,7 @@ mod test { let functions = extract_toplevel_functions(&mut ast); assert_eq!( ast.to_string(), - "SELECT $x AS _sqlpage_f0_a0, 'a' AS _sqlpage_f1_a0, 'b' AS _sqlpage_f1_a1 FROM t" + "SELECT $x AS \"_sqlpage_f0_a0\", 'a' AS \"_sqlpage_f1_a0\", 'b' AS \"_sqlpage_f1_a1\" FROM t" ); assert_eq!( functions, @@ -1281,7 +1287,7 @@ mod test { }; assert_eq!( query, - "SELECT CAST($1 AS TEXT) AS a, 'xxx' AS _sqlpage_f0_a0, x = CAST($2 AS TEXT) AS _sqlpage_f0_a1, CAST($3 AS TEXT) AS c FROM t" + "SELECT CAST($1 AS TEXT) AS a, 'xxx' AS \"_sqlpage_f0_a0\", x = CAST($2 AS TEXT) AS \"_sqlpage_f0_a1\", CAST($3 AS TEXT) AS c FROM t" ); assert_eq!( params, @@ -1632,7 +1638,7 @@ mod test { target_col_name: "sqlpage_set_expr".to_string() }] ); - assert_eq!(query, "SELECT some_db_function() AS _sqlpage_f0_a0"); + assert_eq!(query, "SELECT some_db_function() AS \"_sqlpage_f0_a0\""); assert_eq!(params, []); assert_eq!(json_columns, Vec::::new()); } diff --git a/src/webserver/database/sql_to_json.rs b/src/webserver/database/sql_to_json.rs index 48971307..df92b6a2 100644 --- a/src/webserver/database/sql_to_json.rs +++ b/src/webserver/database/sql_to_json.rs @@ -471,7 +471,7 @@ mod tests { }; let mut c = sqlx::AnyConnection::connect(&db_url).await?; let row = sqlx::query( - "SELECT + "SELECT 42 as integer, 42.25 as real, 'xxx' as string, @@ -647,6 +647,7 @@ mod tests { async fn test_row_to_json_edge_cases() -> anyhow::Result<()> { let db_url = test_database_url(); let mut c = sqlx::AnyConnection::connect(&db_url).await?; + let dbms_name = c.dbms_name().await.expect("retrieve db name"); // Test edge cases for row_to_json let row = sqlx::query( @@ -666,9 +667,12 @@ line2' as multiline_string let json_result = row_to_json(&row); + // For Oracle databases, empty string is treated as NULL. + let empty_str_is_null = dbms_name.to_lowercase().contains("oracle"); + let expected_json = serde_json::json!({ "null_col": null, - "empty_string": "", + "empty_string": if empty_str_is_null { serde_json::Value::Null } else { serde_json::Value::String(String::new()) }, "zero_value": 0, "negative_int": -42, "my_float": 1.23456, diff --git a/tests/core/mod.rs b/tests/core/mod.rs index 25efc764..10f116ce 100644 --- a/tests/core/mod.rs +++ b/tests/core/mod.rs @@ -50,6 +50,13 @@ async fn test_routing_with_db_fs() { config.site_prefix = "/prefix/".to_string(); let state = AppState::init(&config).await.unwrap(); + if matches!( + state.db.info.database_type, + sqlpage::webserver::database::SupportedDatabase::Oracle + ) { + return; + } + let drop_sql = "DROP TABLE IF EXISTS sqlpage_files"; state.db.connection.execute(drop_sql).await.unwrap(); let create_table_sql = diff --git a/tests/data_formats/mod.rs b/tests/data_formats/mod.rs index c195d554..0dcb95b0 100644 --- a/tests/data_formats/mod.rs +++ b/tests/data_formats/mod.rs @@ -41,7 +41,15 @@ async fn test_json_body() -> actix_web::Result<()> { #[actix_web::test] async fn test_csv_body() -> actix_web::Result<()> { - let req = get_request_to("/tests/data_formats/csv_data.sql") + let app_data = make_app_data().await; + if matches!( + app_data.db.info.database_type, + sqlpage::webserver::database::SupportedDatabase::Oracle + ) { + return Ok(()); + } + + let req = crate::common::get_request_to_with_data("/tests/data_formats/csv_data.sql", app_data) .await? .to_srv_request(); let resp = main_handler(req).await?; diff --git a/tests/sql_test_files/component_rendering/columns_component_json_nomssql_nopostgres.sql b/tests/sql_test_files/component_rendering/columns_component_json_nomssql_nopostgres_nooracle.sql similarity index 100% rename from tests/sql_test_files/component_rendering/columns_component_json_nomssql_nopostgres.sql rename to tests/sql_test_files/component_rendering/columns_component_json_nomssql_nopostgres_nooracle.sql diff --git a/tests/sql_test_files/component_rendering/run_sql_from_database.sql b/tests/sql_test_files/component_rendering/run_sql_from_database.sql index 1ef4b2e3..865381db 100644 --- a/tests/sql_test_files/component_rendering/run_sql_from_database.sql +++ b/tests/sql_test_files/component_rendering/run_sql_from_database.sql @@ -11,4 +11,4 @@ from union all select 'works !' - ) as t1; \ No newline at end of file + ) t1; diff --git a/tests/sql_test_files/component_rendering/temp_table_accessible_in_run_sql_nomssql.sql b/tests/sql_test_files/component_rendering/temp_table_accessible_in_run_sql_nomssql_nooracle.sql similarity index 100% rename from tests/sql_test_files/component_rendering/temp_table_accessible_in_run_sql_nomssql.sql rename to tests/sql_test_files/component_rendering/temp_table_accessible_in_run_sql_nomssql_nooracle.sql diff --git a/tests/transactions/mod.rs b/tests/transactions/mod.rs index dc9e7417..3718f09b 100644 --- a/tests/transactions/mod.rs +++ b/tests/transactions/mod.rs @@ -9,8 +9,8 @@ async fn test_transaction_error() -> actix_web::Result<()> { let path = match data.db.info.database_type { SupportedDatabase::MySql => "/tests/transactions/failed_transaction_mysql.sql", SupportedDatabase::Mssql => "/tests/transactions/failed_transaction_mssql.sql", - SupportedDatabase::Snowflake => { - return Ok(()); //snowflake doesn't support transactions + SupportedDatabase::Snowflake | SupportedDatabase::Oracle => { + return Ok(()); //snowflake and oracle don't support transactions in this test way } _ => "/tests/transactions/failed_transaction.sql", }; diff --git a/tests/uploads/upload_csv_test.sql b/tests/uploads/upload_csv_test.sql index 0caf6e66..b17fe9af 100644 --- a/tests/uploads/upload_csv_test.sql +++ b/tests/uploads/upload_csv_test.sql @@ -1,6 +1,6 @@ drop table if exists sqlpage_people_test_table; -create table sqlpage_people_test_table(name text, age text); +create table sqlpage_people_test_table(name varchar(512), age varchar(512)); copy sqlpage_people_test_table(name, age) from 'people_file' with (format csv, header true); select 'text' as component, name || ' is ' || age || ' years old. ' as contents -from sqlpage_people_test_table; \ No newline at end of file +from sqlpage_people_test_table;