BI-2376 - Create Delete Endpoint for Deltabreed Experiment Controller

pom.xml

@@ -90,11 +90,9 @@
         <okhttp.version>4.9.3</okhttp.version>
         <mockito.version>4.3.1</mockito.version>
         <brapi-java-client.version>2.1-SNAPSHOT</brapi-java-client.version>
-        <commons-io.version>2.11.0</commons-io.version>
-        <tika-app.version>2.2.1</tika-app.version>
-        <!-- Version of apache-poi depends on version of tika. Tika uses these. -->
-        <apache-poi.version>4.1.2</apache-poi.version>
-        <apache-poi-ooxml.version>4.1.2</apache-poi-ooxml.version>
+        <tika-app.version>2.9.2</tika-app.version>
+        <!-- Hard-coded version of transitive dependency commons compress seems to be needed. -->
+        <commons-compress.version>1.26.1</commons-compress.version>
         <javaxmail.version>1.6.2</javaxmail.version>
         <st4.version>4.3.1</st4.version>
         <tablesaw.version>1.0.0-SNAPSHOT</tablesaw.version>
@@ -378,36 +376,16 @@
             <artifactId>brapi-java-client</artifactId>
             <version>${brapi-java-client.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-csv</artifactId>
-            <version>${apache-commons-csv.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.poi</groupId>
-            <artifactId>poi</artifactId>
-            <version>${apache-poi.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.poi</groupId>
-            <artifactId>poi-ooxml</artifactId>
-            <version>${apache-poi-ooxml.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-lang3</artifactId>
-            <version>${apache-commons-lang.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>commons-io</groupId>
-            <artifactId>commons-io</artifactId>
-            <version>${commons-io.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.apache.tika</groupId>
             <artifactId>tika-app</artifactId>
             <version>${tika-app.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>${commons-compress.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.sun.mail</groupId>
             <artifactId>javax.mail</artifactId>

src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java

@@ -171,21 +171,6 @@ private Table parseUploadedFile(CompletedFileUpload file) throws UnsupportedType
             throw new UnsupportedTypeException("Unsupported mime type");
         }
 
-        // replace "." with "" in column names to deal with json flattening issue in tablesaw
-        List<String> columnNames = df.columnNames();
-        List<String> namesToReplace = new ArrayList<>();
-        for (String name : columnNames) {
-            if (name.contains(".")) {
-                namesToReplace.add(name);
-            }
-        }
-
-        List<Column<?>> columns = df.columns(namesToReplace.stream().toArray(String[]::new));
-        for (int i=0; i<columns.size(); i++) {
-            Column<?> column = columns.get(i);
-            column.setName(namesToReplace.get(i).replace(".",""));
-        }
-
         return df;
     }
 

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/DynamicColumnParser.java

@@ -16,6 +16,8 @@
  */
 package org.breedinginsight.brapps.importer.services.processors.experiment;
 
+import io.micronaut.http.HttpStatus;
+import io.micronaut.http.exceptions.HttpStatusException;
 import lombok.Getter;
 import tech.tablesaw.api.Table;
 import tech.tablesaw.columns.Column;
@@ -35,6 +37,15 @@ public class DynamicColumnParser {
      * @return A DynamicColumnParseResult object containing the parsed phenotype and timestamp columns.
      */
     public static DynamicColumnParseResult parse(Table data, String[] dynamicColumnNames) {
+
+        // don't allow periods (.) or square brackets in Dynamic Column Names
+        for (String dynamicColumnName: dynamicColumnNames) {
+            if(dynamicColumnName.contains(".") || dynamicColumnName.contains("[") || dynamicColumnName.contains("]")){
+                String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", dynamicColumnName);
+                throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, errorMsg);
+            }
+        }
+
         List<Column<?>> dynamicCols = data.columns(dynamicColumnNames);
         List<Column<?>> phenotypeCols = new ArrayList<>();
         List<Column<?>> timestampCols = new ArrayList<>();

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java

@@ -22,6 +22,8 @@
 import com.google.gson.GsonBuilder;
 import io.micronaut.context.annotation.Property;
 import io.micronaut.context.annotation.Prototype;
+import io.micronaut.http.HttpStatus;
+import io.micronaut.http.exceptions.HttpStatusException;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.map.CaseInsensitiveMap;
 import org.apache.commons.lang3.StringUtils;
@@ -112,6 +114,14 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext
         ImportUpload upload = context.getImportContext().getUpload();
         Table data = context.getImportContext().getData();
         String[] dynamicColNames = upload.getDynamicColumnNames();
+
+        // don't allow periods (.) or square brackets in Dynamic Column Names
+        for (String dynamicColumnName: dynamicColNames) {
+            if(dynamicColumnName.contains(".") || dynamicColumnName.contains("[") || dynamicColumnName.contains("]")){
+                String errorMsg = String.format("Observation columns may not contain periods or square brackets (see column '%s')", dynamicColumnName);
+                throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, errorMsg);
+            }
+        }
         List<Column<?>> dynamicCols = data.columns(dynamicColNames);
 
         // Collect the columns for observation variable data

src/main/java/org/breedinginsight/db/migration/V1_31_0__Set_Dev_Admin_Email.java → src/main/java/org/breedinginsight/db/migration/V1_32_0__Set_Dev_Admin_Email.java

@@ -29,7 +29,7 @@
 import java.util.*;
 
 @Slf4j
-public class V1_31_0__Set_Dev_Admin_Email extends BaseJavaMigration {
+public class V1_32_0__Set_Dev_Admin_Email extends BaseJavaMigration {
 
     @Inject
     private DSLContext dsl;

src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java

@@ -66,8 +66,8 @@ public ValidationError getMissingScaleDataTypeMsg() {
     }
 
     @Override
-    public ValidationError getPeriodObsVarNameMsg() {
-        return new ValidationError("Name", "Period is invalid", HttpStatus.UNPROCESSABLE_ENTITY);
+    public ValidationError getInvalidCharObsVarNameMsg() {
+        return new ValidationError("Name", "Periods and brackets are invalid", HttpStatus.UNPROCESSABLE_ENTITY);
     }
 
     @Override

src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java

@@ -64,12 +64,13 @@ public ValidationError getMissingScaleUnitMsg() {
     public ValidationError getMissingScaleDataTypeMsg() {
         return new ValidationError("scale.dataType", "Missing scale class", HttpStatus.BAD_REQUEST);
     }
-
+    
     @Override
-    public ValidationError getPeriodObsVarNameMsg() {
-        return new ValidationError("observationVariableName", "Period in name is invalid", HttpStatus.BAD_REQUEST);
+    public ValidationError getInvalidCharObsVarNameMsg() {
+        return new ValidationError("observationVariableName", "Periods and brackets in name are invalid", HttpStatus.BAD_REQUEST);
     }
 
+    @Override
     public ValidationError getMissingObsVarNameMsg() {
         return new ValidationError("observationVariableName", "Missing Name", HttpStatus.BAD_REQUEST);
     }

src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java

@@ -31,7 +31,7 @@ public interface TraitValidatorErrorInterface {
     ValidationError getMissingScaleUnitMsg();
     ValidationError getMissingScaleDataTypeMsg();
     ValidationError getMissingObsVarNameMsg();
-    ValidationError getPeriodObsVarNameMsg();
+    ValidationError getInvalidCharObsVarNameMsg();
     ValidationError getMissingTraitEntityMsg();
     ValidationError getMissingTraitAttributeMsg();
     ValidationError getMissingTraitDescriptionMsg();

src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java

@@ -16,7 +16,6 @@
  */
 package org.breedinginsight.services.validators;
 
-import io.micronaut.http.HttpStatus;
 import org.breedinginsight.api.model.v1.response.ValidationError;
 import org.breedinginsight.api.model.v1.response.ValidationErrors;
 import org.breedinginsight.dao.db.enums.DataType;
@@ -209,7 +208,6 @@ public ValidationErrors checkTraitFieldsLength(List<Trait> traits, TraitValidato
         }
         return errors;
     }
-
     public ValidationErrors checkTraitFieldsFormat(List<Trait> traits, TraitValidatorErrorInterface traitValidatorErrors) {
 
         ValidationErrors errors = new ValidationErrors();
@@ -219,12 +217,12 @@ public ValidationErrors checkTraitFieldsFormat(List<Trait> traits, TraitValidato
             Trait trait = traits.get(i);
             String name = trait.getObservationVariableName();
 
-            Pattern pattern = Pattern.compile("\\.");
+            Pattern pattern = Pattern.compile("[\\.\\]\\[]");
             Matcher matcher = pattern.matcher(name);
             boolean containsInvalidCharacter = matcher.find();
 
             if (name != null && containsInvalidCharacter){
-                ValidationError error = traitValidatorErrors.getPeriodObsVarNameMsg();
+                ValidationError error = traitValidatorErrors.getInvalidCharObsVarNameMsg();
                 errors.addError(traitValidatorErrors.getRowNumber(i), error);
             }
         }

src/main/resources/brapi/sql/R__species.sql

@@ -26,7 +26,8 @@ INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'O
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING;
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING;
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING;
-INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey') ON CONFLICT DO NOTHING;
+-- for the Honey Bee case, want to overwrite name, not preserve existing
+INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name;
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING;
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING;
 INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING;

src/main/resources/db/migration/V1.31.0__update_species_honey.sql

@@ -0,0 +1,18 @@
+/*
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+UPDATE species SET common_name = 'Honey Bee' WHERE common_name = 'Honey';

src/main/resources/version.properties

@@ -15,5 +15,5 @@
 #
 
 
-version=v1.1.0+868
-versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2f32914be2408d1b7511ba83e79b62553bfe2287
+version=v1.1.0+893
+versionInfo=https://github.com/Breeding-Insight/bi-api/commit/16cdff9a3f15138b35958dd4aee554ed1a7dc26c

src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java

@@ -370,6 +370,4 @@ public void periodInName() {
         assertEquals(0, validationErrors.getRowErrors().size(), "Wrong number of row errors returned");
     }
 
-
-
 }