diff --git a/pom.xml b/pom.xml index 1bfde3251..4280fd0b1 100644 --- a/pom.xml +++ b/pom.xml @@ -90,11 +90,9 @@ 4.9.3 4.3.1 2.1-SNAPSHOT - 2.11.0 - 2.2.1 - - 4.1.2 - 4.1.2 + 2.9.2 + + 1.26.1 1.6.2 4.3.1 1.0.0-SNAPSHOT @@ -378,36 +376,16 @@ brapi-java-client ${brapi-java-client.version} - - org.apache.commons - commons-csv - ${apache-commons-csv.version} - - - org.apache.poi - poi - ${apache-poi.version} - - - org.apache.poi - poi-ooxml - ${apache-poi-ooxml.version} - - - org.apache.commons - commons-lang3 - ${apache-commons-lang.version} - - - commons-io - commons-io - ${commons-io.version} - org.apache.tika tika-app ${tika-app.version} + + org.apache.commons + commons-compress + ${commons-compress.version} + com.sun.mail javax.mail diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index c0df7711c..3c3c6a093 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -171,21 +171,6 @@ private Table parseUploadedFile(CompletedFileUpload file) throws UnsupportedType throw new UnsupportedTypeException("Unsupported mime type"); } - // replace "." with "" in column names to deal with json flattening issue in tablesaw - List columnNames = df.columnNames(); - List namesToReplace = new ArrayList<>(); - for (String name : columnNames) { - if (name.contains(".")) { - namesToReplace.add(name); - } - } - - List> columns = df.columns(namesToReplace.stream().toArray(String[]::new)); - for (int i=0; i column = columns.get(i); - column.setName(namesToReplace.get(i).replace(".","")); - } - return df; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java index 6b250facc..2f745f63c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java @@ -16,6 +16,8 @@ */ package org.breedinginsight.brapps.importer.services.processors.experiment; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; import lombok.Getter; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; @@ -35,6 +37,15 @@ public class DynamicColumnParser { * @return A DynamicColumnParseResult object containing the parsed phenotype and timestamp columns. */ public static DynamicColumnParseResult parse(Table data, String[] dynamicColumnNames) { + + // don't allow periods (.) or square brackets in Dynamic Column Names + for (String dynamicColumnName: dynamicColumnNames) { + if(dynamicColumnName.contains(".") || dynamicColumnName.contains("[") || dynamicColumnName.contains("]")){ + String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", dynamicColumnName); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, errorMsg); + } + } + List> dynamicCols = data.columns(dynamicColumnNames); List> phenotypeCols = new ArrayList<>(); List> timestampCols = new ArrayList<>(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 6863cd561..7ee29e51f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -22,6 +22,8 @@ import com.google.gson.GsonBuilder; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; @@ -112,6 +114,14 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext ImportUpload upload = context.getImportContext().getUpload(); Table data = context.getImportContext().getData(); String[] dynamicColNames = upload.getDynamicColumnNames(); + + // don't allow periods (.) or square brackets in Dynamic Column Names + for (String dynamicColumnName: dynamicColNames) { + if(dynamicColumnName.contains(".") || dynamicColumnName.contains("[") || dynamicColumnName.contains("]")){ + String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", dynamicColumnName); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, errorMsg); + } + } List> dynamicCols = data.columns(dynamicColNames); // Collect the columns for observation variable data diff --git a/src/main/java/org/breedinginsight/db/migration/V1_31_0__Set_Dev_Admin_Email.java b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java similarity index 98% rename from src/main/java/org/breedinginsight/db/migration/V1_31_0__Set_Dev_Admin_Email.java rename to src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java index 36b47b302..8ffcc02e7 100644 --- a/src/main/java/org/breedinginsight/db/migration/V1_31_0__Set_Dev_Admin_Email.java +++ b/src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java @@ -29,7 +29,7 @@ import java.util.*; @Slf4j -public class V1_31_0__Set_Dev_Admin_Email extends BaseJavaMigration { +public class V1_32_0__Set_Dev_Admin_Email extends BaseJavaMigration { @Inject private DSLContext dsl; diff --git a/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java b/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java index 40807cf03..d19e6a295 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java @@ -66,8 +66,8 @@ public ValidationError getMissingScaleDataTypeMsg() { } @Override - public ValidationError getPeriodObsVarNameMsg() { - return new ValidationError("Name", "Period is invalid", HttpStatus.UNPROCESSABLE_ENTITY); + public ValidationError getInvalidCharObsVarNameMsg() { + return new ValidationError("Name", "Periods and brackets are invalid", HttpStatus.UNPROCESSABLE_ENTITY); } @Override diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java index c9765ed82..bb0b5165f 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java @@ -64,12 +64,13 @@ public ValidationError getMissingScaleUnitMsg() { public ValidationError getMissingScaleDataTypeMsg() { return new ValidationError("scale.dataType", "Missing scale class", HttpStatus.BAD_REQUEST); } - + @Override - public ValidationError getPeriodObsVarNameMsg() { - return new ValidationError("observationVariableName", "Period in name is invalid", HttpStatus.BAD_REQUEST); + public ValidationError getInvalidCharObsVarNameMsg() { + return new ValidationError("observationVariableName", "Periods and brackets in name are invalid", HttpStatus.BAD_REQUEST); } + @Override public ValidationError getMissingObsVarNameMsg() { return new ValidationError("observationVariableName", "Missing Name", HttpStatus.BAD_REQUEST); } diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java index 27400e2c5..73c92a425 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java @@ -31,7 +31,7 @@ public interface TraitValidatorErrorInterface { ValidationError getMissingScaleUnitMsg(); ValidationError getMissingScaleDataTypeMsg(); ValidationError getMissingObsVarNameMsg(); - ValidationError getPeriodObsVarNameMsg(); + ValidationError getInvalidCharObsVarNameMsg(); ValidationError getMissingTraitEntityMsg(); ValidationError getMissingTraitAttributeMsg(); ValidationError getMissingTraitDescriptionMsg(); diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java index d8bd6bf7e..34d9eaa8a 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java @@ -16,7 +16,6 @@ */ package org.breedinginsight.services.validators; -import io.micronaut.http.HttpStatus; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.dao.db.enums.DataType; @@ -209,7 +208,6 @@ public ValidationErrors checkTraitFieldsLength(List traits, TraitValidato } return errors; } - public ValidationErrors checkTraitFieldsFormat(List traits, TraitValidatorErrorInterface traitValidatorErrors) { ValidationErrors errors = new ValidationErrors(); @@ -219,12 +217,12 @@ public ValidationErrors checkTraitFieldsFormat(List traits, TraitValidato Trait trait = traits.get(i); String name = trait.getObservationVariableName(); - Pattern pattern = Pattern.compile("\\."); + Pattern pattern = Pattern.compile("[\\.\\]\\[]"); Matcher matcher = pattern.matcher(name); boolean containsInvalidCharacter = matcher.find(); if (name != null && containsInvalidCharacter){ - ValidationError error = traitValidatorErrors.getPeriodObsVarNameMsg(); + ValidationError error = traitValidatorErrors.getInvalidCharObsVarNameMsg(); errors.addError(traitValidatorErrors.getRowNumber(i), error); } } diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index 9353eb5e1..00c1b7042 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -26,7 +26,8 @@ INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'O INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey') ON CONFLICT DO NOTHING; +-- for the Honey Bee case, want to overwrite name, not preserve existing +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name; INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; diff --git a/src/main/resources/db/migration/V1.31.0__update_species_honey.sql b/src/main/resources/db/migration/V1.31.0__update_species_honey.sql new file mode 100644 index 000000000..3f126b559 --- /dev/null +++ b/src/main/resources/db/migration/V1.31.0__update_species_honey.sql @@ -0,0 +1,18 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +UPDATE species SET common_name = 'Honey Bee' WHERE common_name = 'Honey'; diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 32ceac4bf..e73ad356d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+868 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2f32914be2408d1b7511ba83e79b62553bfe2287 +version=v1.1.0+893 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/16cdff9a3f15138b35958dd4aee554ed1a7dc26c diff --git a/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java b/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java index ea60ac285..9eb1b727f 100644 --- a/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java +++ b/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java @@ -370,6 +370,4 @@ public void periodInName() { assertEquals(0, validationErrors.getRowErrors().size(), "Wrong number of row errors returned"); } - - }