From 145a297c35c90759cf0651f262f3f1e2da07482a Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 28 Oct 2024 12:39:55 -0400 Subject: [PATCH 01/26] [BI-2328] Added new TraitValidator --- .../services/validators/TraitFileValidatorError.java | 5 +++++ .../services/validators/TraitValidatorError.java | 5 +++++ .../services/validators/TraitValidatorErrorInterface.java | 1 + 3 files changed, 11 insertions(+) diff --git a/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java b/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java index 4ba0f99d7..40807cf03 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java @@ -65,6 +65,11 @@ public ValidationError getMissingScaleDataTypeMsg() { return new ValidationError("Scale Class", "Missing scale class", HttpStatus.UNPROCESSABLE_ENTITY); } + @Override + public ValidationError getPeriodObsVarNameMsg() { + return new ValidationError("Name", "Period is invalid", HttpStatus.UNPROCESSABLE_ENTITY); + } + @Override public ValidationError getMissingObsVarNameMsg() { return new ValidationError("Name", "Missing name", HttpStatus.UNPROCESSABLE_ENTITY); diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java index b2db156f6..107419539 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java @@ -64,6 +64,11 @@ public ValidationError getMissingScaleUnitMsg() { public ValidationError getMissingScaleDataTypeMsg() { return new ValidationError("scale.dataType", "Missing scale class", HttpStatus.BAD_REQUEST); } + + @Override + public ValidationError getPeriodObsVarNameMsg() { + return new ValidationError("observationVariableName", "Period in name is invalid", HttpStatus.BAD_REQUEST); + } @Override public ValidationError getMissingObsVarNameMsg() { diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java index 49ef88da5..27400e2c5 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java @@ -31,6 +31,7 @@ public interface TraitValidatorErrorInterface { ValidationError getMissingScaleUnitMsg(); ValidationError getMissingScaleDataTypeMsg(); ValidationError getMissingObsVarNameMsg(); + ValidationError getPeriodObsVarNameMsg(); ValidationError getMissingTraitEntityMsg(); ValidationError getMissingTraitAttributeMsg(); ValidationError getMissingTraitDescriptionMsg(); From 4a3f253ae91e88e896879f37de35b9c45a07b49c Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 28 Oct 2024 12:49:47 -0400 Subject: [PATCH 02/26] [BI-2328] call new Trait Validator from Trait Valiator Service --- .../validators/TraitValidatorService.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java index e778898bd..e6f73ffcd 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java @@ -26,6 +26,8 @@ import javax.inject.Inject; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import static org.apache.commons.lang3.StringUtils.isBlank; @@ -206,7 +208,26 @@ public ValidationErrors checkTraitFieldsLength(List traits, TraitValidato } return errors; } + public ValidationErrors checkTraitFieldsFormat(List traits, TraitValidatorErrorInterface traitValidatorErrors) { + ValidationErrors errors = new ValidationErrors(); + + for (int i = 0; i < traits.size(); i++) { + + Trait trait = traits.get(i); + String name = trait.getObservationVariableName(); + + Pattern pattern = Pattern.compile("\\."); + Matcher matcher = pattern.matcher(name); + boolean containsInvalidCharacter = matcher.find(); + + if (name != null && containsInvalidCharacter){ + ValidationError error = traitValidatorErrors.getPeriodObsVarNameMsg(); + errors.addError(traitValidatorErrors.getRowNumber(i), error); + } + } + return errors; + } public List checkDuplicateTraitsExistingByName(UUID programId, List traits){ List duplicates = new ArrayList<>(); @@ -273,7 +294,8 @@ public Optional checkAllTraitValidations(List traits, T ValidationErrors dataConsistencyErrors = checkTraitDataConsistency(traits, traitValidatorError); ValidationErrors duplicateTraitsInFile = checkDuplicateTraitsInFile(traits, traitValidatorError); ValidationErrors fieldLengthError = checkTraitFieldsLength(traits, traitValidatorError); - validationErrors.mergeAll(requiredFieldErrors, dataConsistencyErrors, duplicateTraitsInFile, fieldLengthError); + ValidationErrors fieldFormatErrors = checkTraitFieldsFormat(traits, traitValidatorError); + validationErrors.mergeAll(requiredFieldErrors, dataConsistencyErrors, duplicateTraitsInFile, fieldLengthError, fieldFormatErrors); if (validationErrors.hasErrors()){ return Optional.of(validationErrors); From 9f936dfe085511e7fd3d66cffbde4e085120e9cd Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 28 Oct 2024 12:52:40 -0400 Subject: [PATCH 03/26] [BI-2328] added Unit test --- .../validators/TraitValidatorUnitTest.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java b/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java index 184e7308e..9eb1b727f 100644 --- a/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java +++ b/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java @@ -348,5 +348,26 @@ public void charLimitExceeded() { } } + @Test + @SneakyThrows + public void periodInName() { + + Trait trait = new Trait(); + trait.setObservationVariableName("Period.1"); + + ValidationErrors validationErrors = traitValidatorService.checkTraitFieldsFormat(List.of(trait), new TraitValidatorError()); + + assertEquals(1, validationErrors.getRowErrors().size(), "Wrong number of row errors returned"); + RowValidationErrors rowValidationErrors = validationErrors.getRowErrors().get(0); + assertEquals(1, rowValidationErrors.getErrors().size(), "Wrong number of errors for row"); + assertEquals(400, rowValidationErrors.getErrors().get(0).getHttpStatusCode(), "Wrong error code"); + assertEquals("observationVariableName", rowValidationErrors.getErrors().get(0).getField(), "Wrong error column"); + + //There should be no errors + Trait noPeriodTrait = new Trait(); + noPeriodTrait.setObservationVariableName("NoPeriod"); + validationErrors = traitValidatorService.checkTraitFieldsFormat(List.of(noPeriodTrait), new TraitValidatorError()); + assertEquals(0, validationErrors.getRowErrors().size(), "Wrong number of row errors returned"); + } } From 80f8ad760355525200ce212323273a1fff76c8cc Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 30 Oct 2024 18:21:48 +0000 Subject: [PATCH 04/26] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index f2a42027e..2114e4474 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.0.0+850 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/bae27e0403f883a499db0fc1d57740830e3ca594 +version=v1.0.0+854 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/8854b00dc8125e97d20a2614cc9d1e9702b33afe From ca599cda58563bb5aa1590cd0586f4bef2e2aa34 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 31 Oct 2024 17:51:40 +0000 Subject: [PATCH 05/26] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 2114e4474..3337cdf9f 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.0.0+854 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/8854b00dc8125e97d20a2614cc9d1e9702b33afe +version=v1.0.0+858 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c9bc45f7e4c3b89ad16f41040b5fb4c2a2f63093 From 3c3b2d20216eb3b060ca888361414205da3dafeb Mon Sep 17 00:00:00 2001 From: HMS17 Date: Thu, 31 Oct 2024 14:12:09 -0400 Subject: [PATCH 06/26] [BI-2355] - Non-informative error message: regression --- .../brapps/importer/services/FileImportService.java | 8 +++++--- .../services/validators/TraitFileValidatorError.java | 4 ++-- .../services/validators/TraitValidatorError.java | 5 ++--- .../services/validators/TraitValidatorErrorInterface.java | 2 +- .../services/validators/TraitValidatorService.java | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index c0df7711c..8acb40144 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -171,11 +171,12 @@ private Table parseUploadedFile(CompletedFileUpload file) throws UnsupportedType throw new UnsupportedTypeException("Unsupported mime type"); } - // replace "." with "" in column names to deal with json flattening issue in tablesaw + // replace certain special characters with "" in column names to deal with json flattening issue in tablesaw + // this includes ".", "[", "[" List columnNames = df.columnNames(); List namesToReplace = new ArrayList<>(); for (String name : columnNames) { - if (name.contains(".")) { + if (name.contains(".") || name.contains("[") || name.contains("]")) { namesToReplace.add(name); } } @@ -183,7 +184,8 @@ private Table parseUploadedFile(CompletedFileUpload file) throws UnsupportedType List> columns = df.columns(namesToReplace.stream().toArray(String[]::new)); for (int i=0; i column = columns.get(i); - column.setName(namesToReplace.get(i).replace(".","")); + //if more characters, could use replaceall and regex, but this works presently + column.setName(namesToReplace.get(i).replace(".","").replace("[","").replace("]","")); } return df; diff --git a/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java b/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java index 40807cf03..d19e6a295 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java @@ -66,8 +66,8 @@ public ValidationError getMissingScaleDataTypeMsg() { } @Override - public ValidationError getPeriodObsVarNameMsg() { - return new ValidationError("Name", "Period is invalid", HttpStatus.UNPROCESSABLE_ENTITY); + public ValidationError getInvalidCharObsVarNameMsg() { + return new ValidationError("Name", "Periods and brackets are invalid", HttpStatus.UNPROCESSABLE_ENTITY); } @Override diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java index 107419539..731f61f89 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java @@ -66,11 +66,10 @@ public ValidationError getMissingScaleDataTypeMsg() { } @Override - public ValidationError getPeriodObsVarNameMsg() { - return new ValidationError("observationVariableName", "Period in name is invalid", HttpStatus.BAD_REQUEST); + public ValidationError getInvalidCharObsVarNameMsg() { + return new ValidationError("observationVariableName", "Periods and brackets in name is invalid", HttpStatus.BAD_REQUEST); } - @Override public ValidationError getMissingObsVarNameMsg() { return new ValidationError("observationVariableName", "Missing Name", HttpStatus.BAD_REQUEST); } diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java index 27400e2c5..73c92a425 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java @@ -31,7 +31,7 @@ public interface TraitValidatorErrorInterface { ValidationError getMissingScaleUnitMsg(); ValidationError getMissingScaleDataTypeMsg(); ValidationError getMissingObsVarNameMsg(); - ValidationError getPeriodObsVarNameMsg(); + ValidationError getInvalidCharObsVarNameMsg(); ValidationError getMissingTraitEntityMsg(); ValidationError getMissingTraitAttributeMsg(); ValidationError getMissingTraitDescriptionMsg(); diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java index e6f73ffcd..aab931921 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java @@ -217,12 +217,12 @@ public ValidationErrors checkTraitFieldsFormat(List traits, TraitValidato Trait trait = traits.get(i); String name = trait.getObservationVariableName(); - Pattern pattern = Pattern.compile("\\."); + Pattern pattern = Pattern.compile("[\\.\\]\\[]"); Matcher matcher = pattern.matcher(name); boolean containsInvalidCharacter = matcher.find(); if (name != null && containsInvalidCharacter){ - ValidationError error = traitValidatorErrors.getPeriodObsVarNameMsg(); + ValidationError error = traitValidatorErrors.getInvalidCharObsVarNameMsg(); errors.addError(traitValidatorErrors.getRowNumber(i), error); } } From 72e103ff4452f7a62de12b511631dc5e19c72a00 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Thu, 31 Oct 2024 15:41:02 -0400 Subject: [PATCH 07/26] [BI-2355] - Restore @override --- .../breedinginsight/services/validators/TraitValidatorError.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java index 731f61f89..9a94e75b4 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java @@ -70,6 +70,7 @@ public ValidationError getInvalidCharObsVarNameMsg() { return new ValidationError("observationVariableName", "Periods and brackets in name is invalid", HttpStatus.BAD_REQUEST); } + @Override public ValidationError getMissingObsVarNameMsg() { return new ValidationError("observationVariableName", "Missing Name", HttpStatus.BAD_REQUEST); } From 7826135cbb1b70e67b15ceb6a5eae82ade03523e Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Fri, 1 Nov 2024 15:38:31 -0400 Subject: [PATCH 08/26] [BI-2355] Comment fix Co-authored-by: mlm483 <128052931+mlm483@users.noreply.github.com> --- .../brapps/importer/services/FileImportService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 8acb40144..2696df137 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -172,7 +172,7 @@ private Table parseUploadedFile(CompletedFileUpload file) throws UnsupportedType } // replace certain special characters with "" in column names to deal with json flattening issue in tablesaw - // this includes ".", "[", "[" + // this includes ".", "[", "]" List columnNames = df.columnNames(); List namesToReplace = new ArrayList<>(); for (String name : columnNames) { From 3ac46357a7cf7e902667abfa69bdf37708886c04 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Fri, 1 Nov 2024 16:17:18 -0400 Subject: [PATCH 09/26] [BI-2355] - Code review backend logic more elegant --- .../importer/services/FileImportService.java | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 2696df137..1478cf985 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -173,20 +173,9 @@ private Table parseUploadedFile(CompletedFileUpload file) throws UnsupportedType // replace certain special characters with "" in column names to deal with json flattening issue in tablesaw // this includes ".", "[", "]" - List columnNames = df.columnNames(); - List namesToReplace = new ArrayList<>(); - for (String name : columnNames) { - if (name.contains(".") || name.contains("[") || name.contains("]")) { - namesToReplace.add(name); - } - } - - List> columns = df.columns(namesToReplace.stream().toArray(String[]::new)); - for (int i=0; i column = columns.get(i); - //if more characters, could use replaceall and regex, but this works presently - column.setName(namesToReplace.get(i).replace(".","").replace("[","").replace("]","")); - } + df.columns().forEach( + (c) -> c.setName(c.name().replace(".","").replace("[","").replace("]","")) + ); return df; } From 66b527125413997fe351b79c2a41e0af6231d691 Mon Sep 17 00:00:00 2001 From: HMS17 <84345306+HMS17@users.noreply.github.com> Date: Fri, 1 Nov 2024 16:32:48 -0400 Subject: [PATCH 10/26] [BI-2355] Grammar fix --- .../services/validators/TraitValidatorError.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java index 9a94e75b4..bb0b5165f 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java @@ -67,7 +67,7 @@ public ValidationError getMissingScaleDataTypeMsg() { @Override public ValidationError getInvalidCharObsVarNameMsg() { - return new ValidationError("observationVariableName", "Periods and brackets in name is invalid", HttpStatus.BAD_REQUEST); + return new ValidationError("observationVariableName", "Periods and brackets in name are invalid", HttpStatus.BAD_REQUEST); } @Override From 27fc57cf4c177c9a36aeb0b800048cc4deaf661d Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 1 Nov 2024 21:33:00 +0000 Subject: [PATCH 11/26] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 3337cdf9f..9a062df45 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.0.0+858 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c9bc45f7e4c3b89ad16f41040b5fb4c2a2f63093 +version=v1.0.0+860 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0fed12cdc798c4c5a65ff8d51a8dd1961a76aaaa From 07e5eef2fc7ac83546586fc0755e1683aa18d54f Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 4 Nov 2024 16:23:14 +0000 Subject: [PATCH 12/26] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 9a062df45..1cf02537b 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.0.0+860 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0fed12cdc798c4c5a65ff8d51a8dd1961a76aaaa +version=v1.0.0+862 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6862768dea345338252d91a89b0d2cc7db2499e9 From 7bec8060d24c9ee8ac88f5ec1e43f6fa501baa86 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 4 Nov 2024 17:29:59 +0000 Subject: [PATCH 13/26] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1cf02537b..1d39558d4 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.0.0+862 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6862768dea345338252d91a89b0d2cc7db2499e9 +version=v1.0.0+866 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c09857c439c8c0ff9dc952b0119223fbd1cd80a8 From c0e0f9a48b3b55cc419d2a1115b51021da715d0f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 5 Nov 2024 10:58:29 -0500 Subject: [PATCH 14/26] Renamed Honey to Honey Bee --- .../V1.31.0__update_species_honey.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/main/resources/db/migration/V1.31.0__update_species_honey.sql diff --git a/src/main/resources/db/migration/V1.31.0__update_species_honey.sql b/src/main/resources/db/migration/V1.31.0__update_species_honey.sql new file mode 100644 index 000000000..3f126b559 --- /dev/null +++ b/src/main/resources/db/migration/V1.31.0__update_species_honey.sql @@ -0,0 +1,18 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +UPDATE species SET common_name = 'Honey Bee' WHERE common_name = 'Honey'; From 27e94d10e8c5340cea069b932037eed1d25d9df1 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 5 Nov 2024 13:34:26 -0500 Subject: [PATCH 15/26] Rename Honey to Honey Bee in brapi server --- src/main/resources/brapi/sql/R__species.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index 9353eb5e1..00c1b7042 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -26,7 +26,8 @@ INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'O INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey') ON CONFLICT DO NOTHING; +-- for the Honey Bee case, want to overwrite name, not preserve existing +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name; INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; From 84b1d4ef0e9e03b31f23c205c3fbb42fd9c75886 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 5 Nov 2024 21:52:09 +0000 Subject: [PATCH 16/26] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1d39558d4..5638e86c5 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.0.0+866 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c09857c439c8c0ff9dc952b0119223fbd1cd80a8 +version=v1.0.0+870 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/46d0af60b97d2ebafe47feff7e8b64606c975230 From 72f3c7a41983329027e7bea8502b6b3709f5d237 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 6 Nov 2024 16:32:33 -0500 Subject: [PATCH 17/26] [BI-2328] [BI-2355] Addressed Failed QA. An Experiment import file cannot have a period or square brackets in the observation headdings --- .../brapps/importer/services/FileImportService.java | 6 ------ .../processors/experiment/DynamicColumnParser.java | 11 +++++++++++ .../middleware/process/ImportTableProcess.java | 10 ++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 1478cf985..3c3c6a093 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -171,12 +171,6 @@ private Table parseUploadedFile(CompletedFileUpload file) throws UnsupportedType throw new UnsupportedTypeException("Unsupported mime type"); } - // replace certain special characters with "" in column names to deal with json flattening issue in tablesaw - // this includes ".", "[", "]" - df.columns().forEach( - (c) -> c.setName(c.name().replace(".","").replace("[","").replace("]","")) - ); - return df; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java index 6b250facc..2f745f63c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java @@ -16,6 +16,8 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; import lombok.Getter; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; @@ -35,6 +37,15 @@ public class DynamicColumnParser { * @return A DynamicColumnParseResult object containing the parsed phenotype and timestamp columns. */ public static DynamicColumnParseResult parse(Table data, String[] dynamicColumnNames) { + + // don't allow periods (.) or square brackets in Dynamic Column Names + for (String dynamicColumnName: dynamicColumnNames) { + if(dynamicColumnName.contains(".") || dynamicColumnName.contains("[") || dynamicColumnName.contains("]")){ + String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", dynamicColumnName); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, errorMsg); + } + } + List> dynamicCols = data.columns(dynamicColumnNames); List> phenotypeCols = new ArrayList<>(); List> timestampCols = new ArrayList<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 6863cd561..7ee29e51f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -22,6 +22,8 @@ import com.google.gson.GsonBuilder; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; @@ -112,6 +114,14 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext ImportUpload upload = context.getImportContext().getUpload(); Table data = context.getImportContext().getData(); String[] dynamicColNames = upload.getDynamicColumnNames(); + + // don't allow periods (.) or square brackets in Dynamic Column Names + for (String dynamicColumnName: dynamicColNames) { + if(dynamicColumnName.contains(".") || dynamicColumnName.contains("[") || dynamicColumnName.contains("]")){ + String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", dynamicColumnName); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, errorMsg); + } + } List> dynamicCols = data.columns(dynamicColNames); // Collect the columns for observation variable data From 2c5ec7fe2cb04a3b2d99ba7541a5f2813c1997df Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 7 Nov 2024 17:02:33 +0000 Subject: [PATCH 18/26] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5638e86c5..dca2bbcea 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.0.0+870 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/46d0af60b97d2ebafe47feff7e8b64606c975230 +version=v1.0.0+872 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/8b73c45263b416ed0944240f6da23c2810cdd11c From 5ac1ba9bdc93ba0d0607a7211b2f3e6e58775326 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 14 Nov 2024 15:52:52 -0500 Subject: [PATCH 19/26] [BI-2156] - updated apache tika also removed hard-coded transitive dependencies --- pom.xml | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) diff --git a/pom.xml b/pom.xml index 1bfde3251..75dd35c02 100644 --- a/pom.xml +++ b/pom.xml @@ -91,10 +91,7 @@ 4.3.1 2.1-SNAPSHOT 2.11.0 - 2.2.1 - - 4.1.2 - 4.1.2 + 2.9.2 1.6.2 4.3.1 1.0.0-SNAPSHOT @@ -378,31 +375,6 @@ brapi-java-client ${brapi-java-client.version} - - org.apache.commons - commons-csv - ${apache-commons-csv.version} - - - org.apache.poi - poi - ${apache-poi.version} - - - org.apache.poi - poi-ooxml - ${apache-poi-ooxml.version} - - - org.apache.commons - commons-lang3 - ${apache-commons-lang.version} - - - commons-io - commons-io - ${commons-io.version} - org.apache.tika tika-app From 92b08d063d5674409eabf8f0ab1f066dbcf41c48 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 14 Nov 2024 16:26:34 -0500 Subject: [PATCH 20/26] [BI-2156] - hard-coded apache commons compress version --- pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pom.xml b/pom.xml index 75dd35c02..d0efae5cd 100644 --- a/pom.xml +++ b/pom.xml @@ -92,6 +92,8 @@ 2.1-SNAPSHOT 2.11.0 2.9.2 + + 1.26.1 1.6.2 4.3.1 1.0.0-SNAPSHOT @@ -380,6 +382,11 @@ tika-app ${tika-app.version} + + org.apache.commons + commons-compress + ${commons-compress.version} + com.sun.mail javax.mail From da84b7c301a030bab34b447b30c8a75a1195d1ba Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Fri, 15 Nov 2024 15:51:31 -0500 Subject: [PATCH 21/26] [BI-2203] - renamed migration version 1.31.0 was already used --- ...t_Dev_Admin_Email.java => V1_32_0__Set_Dev_Admin_Email.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/main/java/org/breedinginsight/db/migration/{V1_31_0__Set_Dev_Admin_Email.java => V1_32_0__Set_Dev_Admin_Email.java} (98%) diff --git a/src/main/java/org/breedinginsight/db/migration/V1_31_0__Set_Dev_Admin_Email.java b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java similarity index 98% rename from src/main/java/org/breedinginsight/db/migration/V1_31_0__Set_Dev_Admin_Email.java rename to src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java index 36b47b302..8ffcc02e7 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_31_0__Set_Dev_Admin_Email.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java @@ -29,7 +29,7 @@ import java.util.*; @Slf4j -public class V1_31_0__Set_Dev_Admin_Email extends BaseJavaMigration { +public class V1_32_0__Set_Dev_Admin_Email extends BaseJavaMigration { @Inject private DSLContext dsl; From 893ebe04662e07d4c3b5090869b999e198a8f70c Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 18 Nov 2024 20:00:17 +0000 Subject: [PATCH 22/26] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 32ceac4bf..b872e8d73 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+868 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2f32914be2408d1b7511ba83e79b62553bfe2287 +version=v1.1.0+887 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/b7d65b01c976e1c729ef9bb450143e63dad4c014 From beae4476b34b6398d3ec5dc203dcef31abbab9c3 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 18 Nov 2024 20:36:14 +0000 Subject: [PATCH 23/26] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index b872e8d73..323e40196 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+887 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/b7d65b01c976e1c729ef9bb450143e63dad4c014 +version=v1.1.0+889 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6afa35526f279ed0938cb12ce6bf53d3e67f3d92 From 3716bfc6892192c8802fa5c81952ae079d312c2e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 18 Nov 2024 21:06:54 +0000 Subject: [PATCH 24/26] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 323e40196..cba537926 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+889 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6afa35526f279ed0938cb12ce6bf53d3e67f3d92 +version=v1.1.0+891 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/75e978c62c555b745b4779800c350d9780bbcb75 From 22e00037c7eab715201de32b19fdfcdf21df96aa Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 19 Nov 2024 21:15:01 -0500 Subject: [PATCH 25/26] [BI-2156] - removed unused variable from pom.xml --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index d0efae5cd..4280fd0b1 100644 --- a/pom.xml +++ b/pom.xml @@ -90,7 +90,6 @@ 4.9.3 4.3.1 2.1-SNAPSHOT - 2.11.0 2.9.2 1.26.1 From e778c65690012e6871e14d9cd626f3aec821c267 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 20 Nov 2024 21:44:18 +0000 Subject: [PATCH 26/26] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index cba537926..e73ad356d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+891 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/75e978c62c555b745b4779800c350d9780bbcb75 +version=v1.1.0+893 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/16cdff9a3f15138b35958dd4aee554ed1a7dc26c