From 4d341de4a02ffa88a3df35d5810453424f61416d Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Fri, 1 Apr 2022 02:02:31 -0400 Subject: [PATCH 01/14] Changes for Tomcat 8 --- .../conifer/roles/conifer/templates/WDK/tomcat_context.xml.j2 | 2 +- Model/pom.xml | 2 +- Service/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Model/lib/conifer/roles/conifer/templates/WDK/tomcat_context.xml.j2 b/Model/lib/conifer/roles/conifer/templates/WDK/tomcat_context.xml.j2 index 90d94a07a7..6429616c2b 100644 --- a/Model/lib/conifer/roles/conifer/templates/WDK/tomcat_context.xml.j2 +++ b/Model/lib/conifer/roles/conifer/templates/WDK/tomcat_context.xml.j2 @@ -14,7 +14,7 @@ site_vars file: {{ site_vars }} docBase="{{ contextxml_docBase }}" privileged="{{ contextxml_privileged|default("false") }}" swallowOutput="{{ contextxml_swallowOutput|default("true") }}" - allowLinking="{{ contextxml_allowLinking|default("true") }}" > + diff --git a/Model/pom.xml b/Model/pom.xml index a843cabd8c..7ae0784a93 100644 --- a/Model/pom.xml +++ b/Model/pom.xml @@ -158,7 +158,7 @@ javax.servlet - servlet-api + javax.servlet-api provided diff --git a/Service/pom.xml b/Service/pom.xml index fa346246e2..d2161bff4f 100644 --- a/Service/pom.xml +++ b/Service/pom.xml @@ -108,7 +108,7 @@ javax.servlet - servlet-api + javax.servlet-api provided From c9631b638d7d660a1f9d0d3795e74fd505ae617d Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Fri, 28 Jun 2024 15:34:35 -0400 Subject: [PATCH 02/14] Always use https for redirect URLs --- .../java/org/gusdb/wdk/service/formatter/ProjectFormatter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java b/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java index d0abea4be9..6f3040138b 100644 --- a/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java +++ b/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java @@ -45,7 +45,8 @@ public static JSONObject getWdkProjectInfo(WdkModel wdkModel, String serviceEndp JSONObject authConfig = new JSONObject() .put(JsonKeys.AUTHENTICATION_METHOD, config.getAuthenticationMethodEnum().name()) .put(JsonKeys.OAUTH_URL, config.getOauthUrl()) - .put(JsonKeys.OAUTH_CLIENT_URL, serviceEndpoint) + // Always use HTTPS + .put(JsonKeys.OAUTH_CLIENT_URL, serviceEndpoint.replace("http","https")) .put(JsonKeys.OAUTH_CLIENT_ID, config.getOauthClientId()); // create profile property config sub-array From 41555ba2bebc41093a84d90570e16ab9990796a5 Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Tue, 2 Jul 2024 14:54:52 -0400 Subject: [PATCH 03/14] Add external OAuth URL option so we can use docker network URLs for direct access within the stack --- Model/lib/rng/wdkModel-config.rng | 3 +++ .../org/gusdb/wdk/model/config/ModelConfig.java | 12 +++++++++++- .../gusdb/wdk/model/config/ModelConfigBuilder.java | 13 +++++++++++++ .../wdk/service/formatter/ProjectFormatter.java | 3 ++- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/Model/lib/rng/wdkModel-config.rng b/Model/lib/rng/wdkModel-config.rng index e9d5e48b59..c452081aad 100644 --- a/Model/lib/rng/wdkModel-config.rng +++ b/Model/lib/rng/wdkModel-config.rng @@ -50,6 +50,9 @@ + + + diff --git a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java index af2b74436f..e41c0ccd11 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java +++ b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java @@ -129,6 +129,7 @@ public String getName() { */ private final AuthenticationMethod _authenticationMethod; private final String _oauthUrl; // needed if method is OAUTH2 + private final String _externalOauthUrl; // may be needed if method is OAUTH2 and internal URL is not available externally private final String _oauthClientId; // needed if method is OAUTH2 private final String _oauthClientSecret; // needed if method is OAUTH2 private final String _changePasswordUrl; // probably needed if method is OAUTH2 @@ -150,7 +151,7 @@ public ModelConfig(String modelName, String projectId, Path gusHome, boolean cac String emailContent, ModelConfigUserDB userDB, ModelConfigAppDB appDB, ModelConfigUserDatasetStore userDatasetStoreConfig, QueryMonitor queryMonitor, boolean monitorBlockedThreads, int blockedThreshold, AuthenticationMethod authenticationMethod, - String oauthUrl, String oauthClientId, String oauthClientSecret, String changePasswordUrl, + String oauthUrl, String externalOauthUrl, String oauthClientId, String oauthClientSecret, String changePasswordUrl, String keyStoreFile, String keyStorePassPhrase) { // basic model information @@ -195,6 +196,7 @@ public ModelConfig(String modelName, String projectId, Path gusHome, boolean cac // user authentication setup _authenticationMethod = authenticationMethod; _oauthUrl = oauthUrl; + _externalOauthUrl = externalOauthUrl; _oauthClientId = oauthClientId; _oauthClientSecret = oauthClientSecret; _changePasswordUrl = changePasswordUrl; @@ -338,6 +340,14 @@ public String getOauthUrl() { return _oauthUrl; } + /** + * @return base URL of OAuth2 server to use for authentication + * (called only if authentication method is OAUTH2) + */ + public String getExternalOauthUrl() { + return _externalOauthUrl == null || _externalOauthUrl.isBlank() ? _externalOauthUrl : _oauthUrl; + } + /** * @return OAuth2 client ID to use for authentication * (called only if authentication method is OAUTH2) diff --git a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java index ae414d5906..54f0bc250d 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java +++ b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java @@ -59,6 +59,7 @@ public class ModelConfigBuilder { // user authentication setup private AuthenticationMethod _authenticationMethod = AuthenticationMethod.USER_DB; private String _oauthUrl = ""; // needed if method is OAUTH2 + private String _externalOauthUrl = ""; // may be needed if method is OAUTH2 and internal URL is not available externally private String _oauthClientId = ""; // needed if method is OAUTH2 private String _oauthClientSecret = ""; // needed if method is OAUTH2 private String _changePasswordUrl = ""; // probably needed if method is OAUTH2 @@ -137,6 +138,7 @@ public ModelConfig build() throws WdkModelException { // user authentication setup _authenticationMethod, _oauthUrl, + _externalOauthUrl, _oauthClientId, _oauthClientSecret, _changePasswordUrl, @@ -269,6 +271,17 @@ public void setOauthUrl(String oauthUrl) { _oauthUrl = oauthUrl; } + /** + * @param externalOauthUrl base URL of OAuth2 server to use for authentication + * (used only if authentication method is OAUTH2). This may differ from the + * (internal) oauthUrl for some deployments. The external value is returned to + * external clients, telling them how to connect to OAuth. The internal value + * is what WDK actually uses to connect directly to OAuth. + */ + public void setExternalOauthUrl(String externalOauthUrl) { + _externalOauthUrl = externalOauthUrl; + } + /** * @param oauthClientId OAuth2 client ID to use for authentication * (used only if authentication method is OAUTH2) diff --git a/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java b/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java index 6f3040138b..03b5e4b3e2 100644 --- a/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java +++ b/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java @@ -44,7 +44,8 @@ public static JSONObject getWdkProjectInfo(WdkModel wdkModel, String serviceEndp // create authentication config sub-object JSONObject authConfig = new JSONObject() .put(JsonKeys.AUTHENTICATION_METHOD, config.getAuthenticationMethodEnum().name()) - .put(JsonKeys.OAUTH_URL, config.getOauthUrl()) + // Tell client to use external URL + .put(JsonKeys.OAUTH_URL, config.getExternalOauthUrl()) // Always use HTTPS .put(JsonKeys.OAUTH_CLIENT_URL, serviceEndpoint.replace("http","https")) .put(JsonKeys.OAUTH_CLIENT_ID, config.getOauthClientId()); From 2074d7d05d9d3e6a41fd5655a3d468a36836873c Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Wed, 3 Jul 2024 23:07:00 -0400 Subject: [PATCH 04/14] Fix bug --- Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java | 2 +- .../java/org/gusdb/wdk/model/config/ModelConfigBuilder.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java index e41c0ccd11..3a27c24d88 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java +++ b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java @@ -345,7 +345,7 @@ public String getOauthUrl() { * (called only if authentication method is OAUTH2) */ public String getExternalOauthUrl() { - return _externalOauthUrl == null || _externalOauthUrl.isBlank() ? _externalOauthUrl : _oauthUrl; + return _externalOauthUrl != null && !_externalOauthUrl.isBlank() ? _externalOauthUrl : _oauthUrl; } /** diff --git a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java index 54f0bc250d..5e11ce676e 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java +++ b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java @@ -59,7 +59,7 @@ public class ModelConfigBuilder { // user authentication setup private AuthenticationMethod _authenticationMethod = AuthenticationMethod.USER_DB; private String _oauthUrl = ""; // needed if method is OAUTH2 - private String _externalOauthUrl = ""; // may be needed if method is OAUTH2 and internal URL is not available externally + private String _externalOauthUrl = null; // may be needed if method is OAUTH2 and internal URL is not available externally private String _oauthClientId = ""; // needed if method is OAUTH2 private String _oauthClientSecret = ""; // needed if method is OAUTH2 private String _changePasswordUrl = ""; // probably needed if method is OAUTH2 From ef3cfb3fcac1830df80eda806eb272515e9bc61f Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Wed, 3 Jul 2024 23:40:08 -0400 Subject: [PATCH 05/14] Make sure redirects use https --- .../java/org/gusdb/wdk/service/formatter/ProjectFormatter.java | 2 +- .../main/java/org/gusdb/wdk/service/service/SessionService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java b/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java index 03b5e4b3e2..287bf0924f 100644 --- a/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java +++ b/Service/src/main/java/org/gusdb/wdk/service/formatter/ProjectFormatter.java @@ -47,7 +47,7 @@ public static JSONObject getWdkProjectInfo(WdkModel wdkModel, String serviceEndp // Tell client to use external URL .put(JsonKeys.OAUTH_URL, config.getExternalOauthUrl()) // Always use HTTPS - .put(JsonKeys.OAUTH_CLIENT_URL, serviceEndpoint.replace("http","https")) + .put(JsonKeys.OAUTH_CLIENT_URL, serviceEndpoint.replace("http://","https://")) .put(JsonKeys.OAUTH_CLIENT_ID, config.getOauthClientId()); // create profile property config sub-array diff --git a/Service/src/main/java/org/gusdb/wdk/service/service/SessionService.java b/Service/src/main/java/org/gusdb/wdk/service/service/SessionService.java index 8f52c70a19..bce1cc4d17 100644 --- a/Service/src/main/java/org/gusdb/wdk/service/service/SessionService.java +++ b/Service/src/main/java/org/gusdb/wdk/service/service/SessionService.java @@ -340,7 +340,7 @@ private static ResponseBuilder createJsonResponse(boolean success, String messag */ private static ResponseBuilder createRedirectResponse(String redirectUrl) throws WdkModelException { try { - return Response.temporaryRedirect(new URI(redirectUrl)); + return Response.temporaryRedirect(new URI(redirectUrl.replace("http://","https://"))); } catch (URISyntaxException e) { throw new WdkModelException("Redirect " + redirectUrl + " not a valid URI."); From 8984a4a06338b1780bf34184a69e89a3ccb1f8c8 Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Wed, 10 Jul 2024 07:14:51 -0400 Subject: [PATCH 06/14] Try to make sure redirects use https --- .../java/org/gusdb/wdk/service/service/SessionService.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Service/src/main/java/org/gusdb/wdk/service/service/SessionService.java b/Service/src/main/java/org/gusdb/wdk/service/service/SessionService.java index 8f52c70a19..14385cdd51 100644 --- a/Service/src/main/java/org/gusdb/wdk/service/service/SessionService.java +++ b/Service/src/main/java/org/gusdb/wdk/service/service/SessionService.java @@ -281,9 +281,12 @@ public Response processLogout() throws WdkModelException { User oldUser = getRequestingUser(); getTemporaryUserData().invalidate(); + // build context URI that always uses https + String contextUri = getRequest().getNoContextUri().replace("http://", "https://"); + // if user is already a guest, no need to log out if (oldUser.isGuest()) - return createRedirectResponse(getContextUri()).build(); + return createRedirectResponse(contextUri).build(); // get a new session and add new guest user to it TwoTuple newUser = getWdkModel().getUserFactory().createUnregisteredUser(); @@ -297,7 +300,7 @@ public Response processLogout() throws WdkModelException { logoutCookies.add(extraCookie); } - ResponseBuilder builder = createRedirectResponse(getContextUri()); + ResponseBuilder builder = createRedirectResponse(contextUri); for (CookieBuilder logoutCookie : logoutCookies) { builder.cookie(logoutCookie.toJaxRsCookie()); } From 0f0d09e71f8d01e18ffd7211629020f24a149bcc Mon Sep 17 00:00:00 2001 From: Dan Galdi Date: Wed, 14 Aug 2024 11:15:05 -0400 Subject: [PATCH 07/14] Postgres support (#100) --- Model/pom.xml | 5 +++ .../wdk/model/config/SecretKeyReader.java | 35 ------------------- .../gusdb/wdk/model/dbms/ResultFactory.java | 2 +- .../wdk/model/user/UserReferenceFactory.java | 34 ++++++++++-------- 4 files changed, 26 insertions(+), 50 deletions(-) delete mode 100644 Model/src/main/java/org/gusdb/wdk/model/config/SecretKeyReader.java diff --git a/Model/pom.xml b/Model/pom.xml index 2d79536b7f..6a1e0eca26 100644 --- a/Model/pom.xml +++ b/Model/pom.xml @@ -166,6 +166,11 @@ test + + org.postgresql + postgresql + + org.mockito mockito-core diff --git a/Model/src/main/java/org/gusdb/wdk/model/config/SecretKeyReader.java b/Model/src/main/java/org/gusdb/wdk/model/config/SecretKeyReader.java deleted file mode 100644 index 30af7de87a..0000000000 --- a/Model/src/main/java/org/gusdb/wdk/model/config/SecretKeyReader.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gusdb.wdk.model.config; - -import java.io.FileReader; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Optional; - -import org.gusdb.fgputil.EncryptionUtil; -import org.gusdb.fgputil.IoUtil; - -public class SecretKeyReader { - - public static void main(String[] args) { - if (args.length != 1) { - System.err.println("USAGE: readSecretKey "); - System.exit(1); - } - Path path = Paths.get(args[0]); - try { - System.out.print(readSecretKey(Optional.of(path))); - } - catch (Exception e) { - e.printStackTrace(System.err); - System.exit(2); - } - } - - public static String readSecretKey(Optional secretKeyFile) throws IOException { - try (FileReader in = new FileReader(secretKeyFile.get().toFile())) { - return EncryptionUtil.md5(IoUtil.readAllChars(in).strip()); - } - } - -} diff --git a/Model/src/main/java/org/gusdb/wdk/model/dbms/ResultFactory.java b/Model/src/main/java/org/gusdb/wdk/model/dbms/ResultFactory.java index dc63bb9428..c1b3c118e1 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/dbms/ResultFactory.java +++ b/Model/src/main/java/org/gusdb/wdk/model/dbms/ResultFactory.java @@ -162,7 +162,7 @@ private void insertInstanceRow(InstanceInfo instanceInfo) throws WdkModelExcepti Types.VARCHAR, // table name Types.VARCHAR, // query name Types.VARCHAR, // checksum - Types.CLOB // result message + Types.OTHER // result message }; try { diff --git a/Model/src/main/java/org/gusdb/wdk/model/user/UserReferenceFactory.java b/Model/src/main/java/org/gusdb/wdk/model/user/UserReferenceFactory.java index 2673c9cc65..d07a3ffa91 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/user/UserReferenceFactory.java +++ b/Model/src/main/java/org/gusdb/wdk/model/user/UserReferenceFactory.java @@ -6,6 +6,7 @@ import java.util.Optional; import org.gusdb.fgputil.Tuples.ThreeTuple; +import org.gusdb.fgputil.db.platform.DBPlatform; import org.gusdb.fgputil.db.pool.DatabaseInstance; import org.gusdb.fgputil.db.runner.SQLRunner; import org.gusdb.fgputil.db.runner.SQLRunnerException; @@ -31,20 +32,14 @@ class UserReferenceFactory { private static final String IS_GUEST_VALUE_MACRO = "$$IS_GUEST$$"; // SQL and types to insert previously unknown user refs into the users table - private static final String INSERT_USER_REF_SQL = - "insert" + - " when not exists (select 1 from " + USER_SCHEMA_MACRO + TABLE_USERS + " where " + COL_USER_ID + " = ?)" + - " then" + - " into " + USER_SCHEMA_MACRO + TABLE_USERS + " (" + COL_USER_ID + "," + COL_IS_GUEST + "," + COL_FIRST_ACCESS +")" + - " select ?, " + IS_GUEST_VALUE_MACRO + ", ? from dual"; - private static final Integer[] INSERT_USER_REF_PARAM_TYPES = { Types.BIGINT, Types.BIGINT, Types.TIMESTAMP }; + private static final Integer[] INSERT_USER_REF_PARAM_TYPES = { Types.BIGINT, Types.TIMESTAMP }; // SQL and types to select user ref by ID private static final String SELECT_USER_REF_BY_ID_SQL = "select " + COL_USER_ID + ", " + COL_IS_GUEST + ", " + COL_FIRST_ACCESS + - " from " + USER_SCHEMA_MACRO + TABLE_USERS + - " where " + COL_USER_ID + " = ?"; + " from " + USER_SCHEMA_MACRO + TABLE_USERS + + " where " + COL_USER_ID + " = ?"; private static final Integer[] SELECT_USER_REF_BY_ID_PARAM_TYPES = { Types.BIGINT }; @@ -60,10 +55,12 @@ public UserReference(Long userId, Boolean isGuest, Date firstAccess) { private final DatabaseInstance _userDb; private final String _userSchema; + private final DBPlatform _dbPlatform; public UserReferenceFactory(WdkModel wdkModel) { _userDb = wdkModel.getUserDb(); _userSchema = wdkModel.getModelConfig().getUserDB().getUserSchema(); + _dbPlatform = wdkModel.getUserDb().getPlatform(); } /** @@ -73,24 +70,33 @@ public UserReferenceFactory(WdkModel wdkModel) { * changed by this code. * * @param user user to add - * @throws WdkModelException + * @throws WdkModelException */ public int addUserReference(User user) throws WdkModelException { try { long userId = user.getUserId(); boolean isGuest = user.isGuest(); Timestamp insertedOn = new Timestamp(new Date().getTime()); - String sql = INSERT_USER_REF_SQL + String sql = getInsertUserRefSql() .replace(USER_SCHEMA_MACRO, _userSchema) .replace(IS_GUEST_VALUE_MACRO, _userDb.getPlatform().convertBoolean(isGuest).toString()); return new SQLRunner(_userDb.getDataSource(), sql, "insert-user-ref") - .executeUpdate(new Object[]{ userId, userId, insertedOn }, INSERT_USER_REF_PARAM_TYPES); + .executeUpdate(new Object[]{ userId, insertedOn }, INSERT_USER_REF_PARAM_TYPES); } catch (SQLRunnerException e) { throw WdkModelException.translateFrom(e); } } + private String getInsertUserRefSql() { + return "MERGE INTO " + USER_SCHEMA_MACRO + TABLE_USERS + " AS target " + + "USING (SELECT ? AS user_id, ? AS first_access, " + IS_GUEST_VALUE_MACRO + " AS is_guest" + this._dbPlatform.getDummyTable() + ") AS source (user_id, first_access, is_guest) " + + "ON (target." + COL_USER_ID + " = source.user_id) " + + "WHEN NOT MATCHED THEN " + + "INSERT (" + COL_USER_ID + ", " + COL_IS_GUEST + ", " + COL_FIRST_ACCESS + ") " + + "VALUES (source.user_id, source.is_guest, source.first_access)"; + } + // FIXME: see if this is actually needed anywhere? E.g. do we ever need to look up user refs by user ID to find last login? public Optional getUserReference(long userId) throws WdkModelException { try { @@ -100,8 +106,8 @@ public Optional getUserReference(long userId) throws WdkModelExce SELECT_USER_REF_BY_ID_PARAM_TYPES, rs -> !rs.next() - ? Optional.empty() - : Optional.of(new UserReference( + ? Optional.empty() + : Optional.of(new UserReference( rs.getLong(COL_USER_ID), rs.getBoolean(COL_IS_GUEST), new Date(rs.getTimestamp(COL_FIRST_ACCESS).getTime())))); From 4b1abe9984560a9fe001345b1f549dc2e3c6df3a Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Wed, 14 Aug 2024 12:02:18 -0400 Subject: [PATCH 08/14] Add back deleted file and remove postgres dep (driver lives in tomcat for now) --- Model/pom.xml | 5 --- .../wdk/model/config/SecretKeyReader.java | 35 +++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) create mode 100644 Model/src/main/java/org/gusdb/wdk/model/config/SecretKeyReader.java diff --git a/Model/pom.xml b/Model/pom.xml index 6a1e0eca26..2d79536b7f 100644 --- a/Model/pom.xml +++ b/Model/pom.xml @@ -166,11 +166,6 @@ test - - org.postgresql - postgresql - - org.mockito mockito-core diff --git a/Model/src/main/java/org/gusdb/wdk/model/config/SecretKeyReader.java b/Model/src/main/java/org/gusdb/wdk/model/config/SecretKeyReader.java new file mode 100644 index 0000000000..30af7de87a --- /dev/null +++ b/Model/src/main/java/org/gusdb/wdk/model/config/SecretKeyReader.java @@ -0,0 +1,35 @@ +package org.gusdb.wdk.model.config; + +import java.io.FileReader; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Optional; + +import org.gusdb.fgputil.EncryptionUtil; +import org.gusdb.fgputil.IoUtil; + +public class SecretKeyReader { + + public static void main(String[] args) { + if (args.length != 1) { + System.err.println("USAGE: readSecretKey "); + System.exit(1); + } + Path path = Paths.get(args[0]); + try { + System.out.print(readSecretKey(Optional.of(path))); + } + catch (Exception e) { + e.printStackTrace(System.err); + System.exit(2); + } + } + + public static String readSecretKey(Optional secretKeyFile) throws IOException { + try (FileReader in = new FileReader(secretKeyFile.get().toFile())) { + return EncryptionUtil.md5(IoUtil.readAllChars(in).strip()); + } + } + +} From 80d101e6f7e93c197f7682e9307a76482ab3639d Mon Sep 17 00:00:00 2001 From: Dan Galdi Date: Fri, 16 Aug 2024 09:26:25 -0400 Subject: [PATCH 09/14] Fix date postgres and make SMTP port configurable (#102) --- Model/lib/rng/wdkModel-config.rng | 11 ++++++++ .../java/org/gusdb/wdk/model/Utilities.java | 16 +++++++++--- .../gusdb/wdk/model/config/ModelConfig.java | 26 +++++++++++++++++-- .../wdk/model/config/ModelConfigBuilder.java | 12 +++++++++ .../gusdb/wdk/model/dbms/ResultFactory.java | 2 +- .../wdk/model/user/UserPasswordEmailer.java | 4 ++- .../wdk/model/user/UserReferenceFactory.java | 12 +++++---- 7 files changed, 70 insertions(+), 13 deletions(-) diff --git a/Model/lib/rng/wdkModel-config.rng b/Model/lib/rng/wdkModel-config.rng index 5661cb31a7..d953003992 100644 --- a/Model/lib/rng/wdkModel-config.rng +++ b/Model/lib/rng/wdkModel-config.rng @@ -13,6 +13,17 @@ + + + + + + + + + + + diff --git a/Model/src/main/java/org/gusdb/wdk/model/Utilities.java b/Model/src/main/java/org/gusdb/wdk/model/Utilities.java index ab6b8e2b1a..48fcce356d 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/Utilities.java +++ b/Model/src/main/java/org/gusdb/wdk/model/Utilities.java @@ -217,18 +217,18 @@ public static void sendEmail(String smtpServer, String sendTos, String reply, String subject, String content, String ccAddresses, Attachment[] attachments) throws WdkModelException { // call the 8 parameter one - sendEmail(smtpServer, null, null, sendTos, reply, subject, content, ccAddresses, null, attachments); + sendEmail(smtpServer, null, null, sendTos, reply, subject, content, ccAddresses, null, attachments, 25, false); } public static void sendEmail(String smtpServer, String sendTos, String reply, String subject, String content, String ccAddresses, String bccAddresses, Attachment[] attachments) throws WdkModelException { - sendEmail(smtpServer, null, null, sendTos, reply, subject, content, ccAddresses, bccAddresses, attachments); + sendEmail(smtpServer, null, null, sendTos, reply, subject, content, ccAddresses, bccAddresses, attachments, 25, false); } - // sendEmail() all 10 parameters + // sendEmail() all 12 parameters public static void sendEmail(String smtpServer, String username, String password, String sendTos, String reply, - String subject, String content, String ccAddresses, String bccAddresses, Attachment[] attachments) throws WdkModelException { + String subject, String content, String ccAddresses, String bccAddresses, Attachment[] attachments, int smtpPort, boolean tlsEnabled) throws WdkModelException { LOG.debug("Sending message to: " + sendTos + ", bcc to: " + bccAddresses + ",reply: " + reply + ", using SMPT: " + smtpServer); @@ -237,6 +237,14 @@ public static void sendEmail(String smtpServer, String username, String password Properties props = new Properties(); props.put("mail.smtp.host", smtpServer); props.put("mail.debug", "true"); + props.put("mail.smtp.port", smtpPort); + + if (tlsEnabled) { + props.put("mail.smtp.starttls.enable","true"); + props.put("mail.smtp.ssl.protocols", "TLSv1.2"); + props.put("mail.smtp.ssl.trust", smtpServer); + } + Authenticator auth = null; if (username != null && password != null) { diff --git a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java index 50adc3dc0b..5009304a28 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java +++ b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfig.java @@ -53,6 +53,17 @@ public String getName() { private final Optional _smtpPassword; + /** + * the SMTP port to connect to on SMTP server. + */ + private final int _smtpPort; + + /** + * Determines whether to use TLS when connecting to SMTP server. + */ + private final boolean _smtpTlsEnabled; + + /** * the reply of the registration & recover password emails. */ @@ -156,8 +167,8 @@ public String getName() { public ModelConfig(String modelName, String projectId, Path gusHome, boolean caching, boolean useWeights, String paramRegex, Optional secretKeyFile, String secretKey, Path wdkTempDir, String webServiceUrl, String assetsUrl, - String smtpServer, Optional smtpUser, Optional smtpPassword, String supportEmail, List adminEmails, String emailSubject, - String emailContent, ModelConfigUserDB userDB, ModelConfigAppDB appDB, + String smtpServer, Optional smtpUser, Optional smtpPassword, int smtpPort, boolean smtpTlsEnabled, String supportEmail, + List adminEmails, String emailSubject, String emailContent, ModelConfigUserDB userDB, ModelConfigAppDB appDB, ModelConfigUserDatasetStore userDatasetStoreConfig, QueryMonitor queryMonitor, boolean monitorBlockedThreads, int blockedThreshold, AuthenticationMethod authenticationMethod, String oauthUrl, String externalOauthUrl, String oauthClientId, String oauthClientSecret, String changePasswordUrl, @@ -186,6 +197,9 @@ public ModelConfig(String modelName, String projectId, Path gusHome, boolean cac _smtpServer = smtpServer; _smtpUserName = smtpUser; _smtpPassword = smtpPassword; + _smtpPort = smtpPort; + _smtpTlsEnabled = smtpTlsEnabled; + _supportEmail = supportEmail; _adminEmails = adminEmails; _emailSubject = emailSubject; @@ -279,6 +293,14 @@ public Optional getSmtpPassword() { return _smtpPassword; } + public int getSmtpPort() { + return _smtpPort; + } + + public boolean isSmtpTlsEnabled() { + return _smtpTlsEnabled; + } + /** * @return Returns the emailContent. */ diff --git a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java index 1cd9a5fe82..d8b9476af1 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java +++ b/Model/src/main/java/org/gusdb/wdk/model/config/ModelConfigBuilder.java @@ -40,6 +40,8 @@ public class ModelConfigBuilder { private String _smtpServer; private String _smtpUsername; private String _smtpPassword; + private int _smtpPort; + private boolean _smtpTlsEnabled; private String _supportEmail; private List _adminEmails = Collections.emptyList(); private String _emailSubject = ""; @@ -125,6 +127,8 @@ public ModelConfig build() throws WdkModelException { smtpServer, smtpUsername, smtpPassword, + _smtpPort, + _smtpTlsEnabled, _supportEmail, _adminEmails, _emailSubject, @@ -216,6 +220,14 @@ public void setSmtpPassword(String smtpPassword) { _smtpPassword = smtpPassword; } + public void setSmtpPort(int smtpPort) { + _smtpPort = smtpPort; + } + + public void setSmtpTlsEnabled(boolean smtpTlsEnabled) { + _smtpTlsEnabled = smtpTlsEnabled; + } + /** * @param emailContent * The emailContent to set. diff --git a/Model/src/main/java/org/gusdb/wdk/model/dbms/ResultFactory.java b/Model/src/main/java/org/gusdb/wdk/model/dbms/ResultFactory.java index c1b3c118e1..dc63bb9428 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/dbms/ResultFactory.java +++ b/Model/src/main/java/org/gusdb/wdk/model/dbms/ResultFactory.java @@ -162,7 +162,7 @@ private void insertInstanceRow(InstanceInfo instanceInfo) throws WdkModelExcepti Types.VARCHAR, // table name Types.VARCHAR, // query name Types.VARCHAR, // checksum - Types.OTHER // result message + Types.CLOB // result message }; try { diff --git a/Model/src/main/java/org/gusdb/wdk/model/user/UserPasswordEmailer.java b/Model/src/main/java/org/gusdb/wdk/model/user/UserPasswordEmailer.java index 6a13737213..8806d2d425 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/user/UserPasswordEmailer.java +++ b/Model/src/main/java/org/gusdb/wdk/model/user/UserPasswordEmailer.java @@ -42,6 +42,8 @@ public void emailTemporaryPassword(User user, String password) throws WdkModelEx // Unwrap optionals here to maintain consistency with nullable arguments in Utilities.sendEmail method. String smtpUser = wdkModelConfig.getSmtpUserName().orElse(null); String smtpPass = wdkModelConfig.getSmtpPassword().orElse(null); + int smtpPort = wdkModelConfig.getSmtpPort(); + boolean smtpTlsEnabled = wdkModelConfig.isSmtpTlsEnabled(); // populate email content macros with user data String emailContent = wdkModelConfig.getEmailContent() @@ -52,6 +54,6 @@ public void emailTemporaryPassword(User user, String password) throws WdkModelEx .replaceAll("\\$\\$" + EMAIL_MACRO_PASSWORD + "\\$\\$", Matcher.quoteReplacement(password)); - Utilities.sendEmail(smtpServer, smtpUser, smtpPass, user.getEmail(), supportEmail, emailSubject, emailContent, null, null, new Attachment[]{}); + Utilities.sendEmail(smtpServer, smtpUser, smtpPass, user.getEmail(), supportEmail, emailSubject, emailContent, null, null, new Attachment[]{}, smtpPort, smtpTlsEnabled); } } diff --git a/Model/src/main/java/org/gusdb/wdk/model/user/UserReferenceFactory.java b/Model/src/main/java/org/gusdb/wdk/model/user/UserReferenceFactory.java index 9dc362ac09..4b81ec6fe3 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/user/UserReferenceFactory.java +++ b/Model/src/main/java/org/gusdb/wdk/model/user/UserReferenceFactory.java @@ -29,9 +29,10 @@ class UserReferenceFactory { public static final String USER_SCHEMA_MACRO = "$$USER_SCHEMA$$"; private static final String IS_GUEST_VALUE_MACRO = "$$IS_GUEST$$"; + private static final String FIRST_ACCESS_MACRO = "$$FIRST_ACCESS$$"; // types used by SQL returned by getInsertUserRefSql() below - private static final Integer[] INSERT_USER_REF_PARAM_TYPES = { Types.BIGINT, Types.TIMESTAMP }; + private static final Integer[] INSERT_USER_REF_PARAM_TYPES = { Types.BIGINT }; // SQL and types to select user ref by ID private static final String SELECT_USER_REF_BY_ID_SQL = @@ -72,12 +73,13 @@ public int addUserReference(User user) throws WdkModelException { try { long userId = user.getUserId(); boolean isGuest = user.isGuest(); - Timestamp insertedOn = new Timestamp(new Date().getTime()); + Date insertedOn = new Date(); String sql = getInsertUserRefSql() .replace(USER_SCHEMA_MACRO, _userSchema) - .replace(IS_GUEST_VALUE_MACRO, _userDb.getPlatform().convertBoolean(isGuest).toString()); + .replace(IS_GUEST_VALUE_MACRO, _userDb.getPlatform().convertBoolean(isGuest).toString()) + .replace(FIRST_ACCESS_MACRO, _userDb.getPlatform().toDbDateSqlValue(insertedOn)); return new SQLRunner(_userDb.getDataSource(), sql, "insert-user-ref") - .executeUpdate(new Object[]{ userId, insertedOn }, INSERT_USER_REF_PARAM_TYPES); + .executeUpdate(new Object[]{userId}, INSERT_USER_REF_PARAM_TYPES); } catch (SQLRunnerException e) { throw WdkModelException.translateFrom(e); @@ -86,7 +88,7 @@ public int addUserReference(User user) throws WdkModelException { private String getInsertUserRefSql() { return "MERGE INTO " + USER_SCHEMA_MACRO + TABLE_USERS + " tgt " + - "USING (SELECT ? AS user_id, ? AS first_access, " + IS_GUEST_VALUE_MACRO + " AS is_guest" + _userDb.getPlatform().getDummyTable() + ") src " + + "USING (SELECT ? AS user_id, " + FIRST_ACCESS_MACRO + " AS first_access, " + IS_GUEST_VALUE_MACRO + " AS is_guest" + _userDb.getPlatform().getDummyTable() + ") src " + "ON (tgt." + COL_USER_ID + " = src.user_id) " + "WHEN NOT MATCHED THEN " + "INSERT (" + COL_USER_ID + ", " + COL_IS_GUEST + ", " + COL_FIRST_ACCESS + ") " + From 7494aedfab9670653bfb9dbda2695c162ec9a2ef Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Wed, 11 Sep 2024 13:50:52 -0400 Subject: [PATCH 10/14] Add explicit stack trace dump to debut import issue --- .../src/main/java/org/gusdb/wdk/model/user/StrategyLoader.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Model/src/main/java/org/gusdb/wdk/model/user/StrategyLoader.java b/Model/src/main/java/org/gusdb/wdk/model/user/StrategyLoader.java index fc9ef37b32..9295ed107c 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/user/StrategyLoader.java +++ b/Model/src/main/java/org/gusdb/wdk/model/user/StrategyLoader.java @@ -236,6 +236,8 @@ private SearchResult doSearch(String sql, boolean propagateBuildErrors, Object[] } catch (Exception e) { LOG.error("Unable to execute search with SQL: " + NL + sql + NL + "and params [" + FormatUtil.join(paramValues, ",") + "].", e); + // FIXME: not sure why the line above is not printing the stack trace in the logs, but do so explicitly to help debug NPE + LOG.error(FormatUtil.getStackTrace(e)); return WdkModelException.unwrap(e); } } From 56e85d911c2967c7adc9e4f5a34310d1d1846d36 Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Mon, 6 Jan 2025 22:43:29 -0500 Subject: [PATCH 11/14] Deprecate override of deprecated method --- .../gusdb/wdk/controller/filter/HttpResponseHeaderLogger.java | 1 + 1 file changed, 1 insertion(+) diff --git a/Model/src/main/java/org/gusdb/wdk/controller/filter/HttpResponseHeaderLogger.java b/Model/src/main/java/org/gusdb/wdk/controller/filter/HttpResponseHeaderLogger.java index ea2dd06a03..4601079b76 100644 --- a/Model/src/main/java/org/gusdb/wdk/controller/filter/HttpResponseHeaderLogger.java +++ b/Model/src/main/java/org/gusdb/wdk/controller/filter/HttpResponseHeaderLogger.java @@ -139,6 +139,7 @@ public void setStatus(int sc) { super.setStatus(sc); } + @Deprecated @Override public void setStatus(int sc, String sm) { _responseStatus = sc; From dde0eff1c117cf74659d8395d0e6395cb4f944d1 Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Fri, 10 Jan 2025 10:45:15 -0500 Subject: [PATCH 12/14] Change 500 -> 400 if data param is missing --- .../main/java/org/gusdb/wdk/service/service/AnswerService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Service/src/main/java/org/gusdb/wdk/service/service/AnswerService.java b/Service/src/main/java/org/gusdb/wdk/service/service/AnswerService.java index 2d9757db8f..c9dee3aa1e 100644 --- a/Service/src/main/java/org/gusdb/wdk/service/service/AnswerService.java +++ b/Service/src/main/java/org/gusdb/wdk/service/service/AnswerService.java @@ -224,6 +224,8 @@ public Response createCustomReportAnswerFromForm( @PathParam(REPORT_NAME_PATH_PARAM) String reportName, @FormParam("data") String data) throws WdkModelException, DataValidationException, RequestMisformatException { + if (data == null) + throw new RequestMisformatException("Form data parameter 'data' is empty or malformed."); return createCustomReportAnswer(reportName, new JSONObject(data)); } From b88dc4db4f6a12047a34a6f0e53b3c75720d1083 Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Fri, 10 Jan 2025 15:01:44 -0500 Subject: [PATCH 13/14] Add better error logging to be more accurate and show underlying error cause --- .../wdk/model/user/analysis/StepAnalysisInstance.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Model/src/main/java/org/gusdb/wdk/model/user/analysis/StepAnalysisInstance.java b/Model/src/main/java/org/gusdb/wdk/model/user/analysis/StepAnalysisInstance.java index 22cfa37b04..fc3556a3c6 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/user/analysis/StepAnalysisInstance.java +++ b/Model/src/main/java/org/gusdb/wdk/model/user/analysis/StepAnalysisInstance.java @@ -2,6 +2,7 @@ import java.util.Map; import java.util.Optional; +import java.util.function.Function; import org.apache.log4j.Logger; import org.gusdb.fgputil.EncryptionUtil; @@ -128,8 +129,9 @@ static StepAnalysisInstance createFromStoredData(WdkModel wdkModel, JSONObject formObj = json.getJSONObject(JsonKey.formParams.name()); // load the owning step and validate - Step step = loadStep(instance._wdkModel, stepId, new WdkModelException("Unable " + - "to find step (ID=" + stepId + ") defined in step analysis instance (ID=" + analysisId + ")")); + Step step = loadStep(instance._wdkModel, stepId, e -> new WdkModelException("Unable " + + "to load step (ID=" + stepId + ") defined in step analysis instance (ID=" + analysisId + ")." + + " The step is either missing or not runnable (required for step analysis).", e)); instance._step = step; // validation bundle will be at the level of the analysis even though step is always checked at Runnable level @@ -281,12 +283,13 @@ public static StepAnalysisInstance createCopy(StepAnalysisInstance oldInstance, } private static Step loadStep(WdkModel wdkModel, long stepId, - T wdkUserException) throws T { + Function wdkUserException) throws T { try { return wdkModel.getStepFactory().getStepByValidId(stepId, ValidationLevel.RUNNABLE); } catch (WdkModelException e) { - throw wdkUserException; + LOG.error("Unable to load step for ID " + stepId + ", which was expected to be valid.", e); + throw wdkUserException.apply(e); } } From 599b9a01327a8dae3095b2442901f0844cd8bbce Mon Sep 17 00:00:00 2001 From: Ryan Doherty Date: Tue, 21 Jan 2025 23:01:55 -0500 Subject: [PATCH 14/14] Improve error message if display column is null --- .../wdk/model/query/param/FlatVocabularyFetcher.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Model/src/main/java/org/gusdb/wdk/model/query/param/FlatVocabularyFetcher.java b/Model/src/main/java/org/gusdb/wdk/model/query/param/FlatVocabularyFetcher.java index f9ff9db373..fdc3cc7148 100644 --- a/Model/src/main/java/org/gusdb/wdk/model/query/param/FlatVocabularyFetcher.java +++ b/Model/src/main/java/org/gusdb/wdk/model/query/param/FlatVocabularyFetcher.java @@ -140,7 +140,15 @@ private void populateVocabInstance(EnumParamVocabInstance vocabInstance) throws String term = objTerm.toString().trim(); String value = objInternal.toString().trim(); - String display = hasDisplay ? result.get(FlatVocabParam.COLUMN_DISPLAY).toString().trim() : term; + String display = term; // use term for display if no display column + if (hasDisplay) { + Object displayObj = result.get(FlatVocabParam.COLUMN_DISPLAY); + if (displayObj == null) { + throw new WdkModelException("Vocabulary query '" + _vocabQuery.getFullName() + "' has a '" + + FlatVocabParam.COLUMN_DISPLAY + "' column but its value is null in at least one row."); + } + display = displayObj.toString().trim(); + } String parentTerm = null; if (hasParent) { Object parent = result.get(FlatVocabParam.COLUMN_PARENT_TERM);