Merge pull request #442 from Breeding-Insight/BI-2156-qa

BI-2156-qa - Append Workflow Edge Cases

Author: mlm483

src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java

@@ -449,7 +449,7 @@ private void processFile(String workflowId, List<BrAPIImport> finalBrAPIImportLi
                 progress.setMessage(e.getMessage());
                 progress.setUpdatedBy(actingUser.getId());
                 importDAO.update(upload);
-            }catch (ValidatorException e) {
+            } catch (ValidatorException e) {
                 log.info("Validation errors: \n" + e);
                 ImportProgress progress = upload.getProgress();
                 progress.setStatuscode((short) HttpStatus.UNPROCESSABLE_ENTITY.getCode());

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java

@@ -44,6 +44,7 @@
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext;
 import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData;
 import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData;
+import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException;
 import org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants;
 import org.breedinginsight.model.Program;
 import org.breedinginsight.model.Scale;
@@ -70,8 +71,7 @@ public class ExperimentUtilities {
     public static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists";
     public static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs";
     public static final String UNMATCHED_COLUMN = "Ontology term(s) not found: ";
-
-
+    public static final String INVALID_OBS_UNIT_ID_ERROR = "Invalid ObsUnitID";
 
 
     Gson gson;
@@ -294,44 +294,104 @@ public static void addYearToStudyAdditionalInfo(Program program, BrAPIStudy stud
     }
 
     /**
-     * This method is responsible for collating unique ObsUnit IDs from the provided context data.
+     * Collates unique Observation Unit IDs from the import context.
+     *
+     * This method iterates through all import rows in the given context and
+     * extracts unique Observation Unit IDs (ObsUnit IDs) that are not null or blank.
+     *
+     * @param context The AppendOverwriteMiddlewareContext containing the import data.
+     * @return A Set of String containing all unique, non-null, non-blank Observation Unit IDs.
      *
-     * @param context the AppendOverwriteMiddlewareContext containing the import rows to process
-     * @return a Set of unique ObsUnit IDs collated from the import rows
-     * @throws IllegalStateException if any ObsUnit ID is repeated in the import rows
-     * @throws HttpStatusException if there is a mix of ObsUnit IDs for some but not all rows
+     * @implNote The method performs the following steps:
+     * 1. Initializes an empty HashSet to store unique ObsUnit IDs.
+     * 2. Iterates through each import row in the context.
+     * 3. For each row, checks if the ObsUnit ID is not null and not blank.
+     * 4. If valid, adds the ObsUnit ID to the set.
+     * 5. Returns the set of unique ObsUnit IDs.
      */
-    public static Set<String> collateReferenceOUIds(AppendOverwriteMiddlewareContext context) throws HttpStatusException, IllegalStateException {
+    public static Set<String> collateUniqueOUIds(AppendOverwriteMiddlewareContext context) {
         // Initialize variables to track the presence of ObsUnit IDs
         Set<String> referenceOUIds = new HashSet<>();
-        boolean hasNoReferenceUnitIds = true;
-        boolean hasAllReferenceUnitIds = true;
+
+        // Iterate through the import rows to process ObsUnit IDs
+        for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) {
+            ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum);
+            if (importRow.getObsUnitID() != null && !importRow.getObsUnitID().isBlank()) {
+                referenceOUIds.add(importRow.getObsUnitID());
+            }
+        }
+        return referenceOUIds;
+    }
+
+    /**
+     * Validates Observation Unit ID values in the import context.
+     *
+     * This method checks each import row for the validity of its Observation Unit ID (ObsUnitID).
+     * It performs the following validations:
+     * 1. Checks if the ObsUnitID is null or blank.
+     * 2. Checks if the ObsUnitID is a duplicate within the import data.
+     *
+     * @param context The AppendOverwriteMiddlewareContext containing import data and validation error storage.
+     * @throws HttpStatusException If there's an HTTP-related error during the validation process.
+     * @throws IllegalStateException If the system encounters an unexpected state during validation.
+     *
+     * @implNote The method performs the following steps:
+     * 1. Retrieves the ValidationErrors object from the context.
+     * 2. Initializes a HashSet to track unique ObsUnitIDs.
+     * 3. Iterates through each import row in the context.
+     * 4. For each row:
+     *    - If ObsUnitID is null or blank, adds a "missing ObsUnitID" error.
+     *    - If ObsUnitID is already in the set (duplicate), adds a "duplicate ObsUnitID" error.
+     *    - Otherwise, adds the ObsUnitID to the set of unique IDs.
+     * 5. Errors are added using the addRowError method, specifying the OBS_UNIT_ID column and appropriate error messages.
+     */
+    public static void validateReferenceOUIdValues(AppendOverwriteMiddlewareContext context) throws HttpStatusException, IllegalStateException {
+        ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors();
+        Set<String> referenceOUIds = new HashSet<>();
 
         // Iterate through the import rows to process ObsUnit IDs
         for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) {
             ExperimentObservation importRow = (ExperimentObservation) context.getImportContext().getImportRows().get(rowNum);
 
-            // Check if ObsUnitID is blank
             if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) {
-                // Set flag to indicate missing ObsUnit ID for current row
-                hasAllReferenceUnitIds = false;
+                // Check if ObsUnitID is blank
+                addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID.getValue(), validationErrors, rowNum);
             } else if (referenceOUIds.contains(importRow.getObsUnitID())) {
-                // Throw exception if ObsUnitID is repeated
-                throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID());
+                // Check if ObsUnitID is repeated
+                addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExpImportProcessConstants.ErrMessage.DUPLICATE_OBS_UNIT_ID.getValue(), validationErrors, rowNum);
             } else {
                 // Add ObsUnitID to referenceOUIds
                 referenceOUIds.add(importRow.getObsUnitID());
-                // Set flag to indicate presence of ObsUnit ID
-                hasNoReferenceUnitIds = false;
             }
         }
+    }
 
-        if (!hasNoReferenceUnitIds && !hasAllReferenceUnitIds) {
-            // Throw exception if there is a mix of ObsUnit IDs for some but not all rows
-            throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, ExpImportProcessConstants.ErrMessage.MISSING_OBS_UNIT_ID_ERROR.getValue());
-        }
+    /**
+     * Adds validation errors for observation units that were not found in the database.
+     *
+     * This method processes an EntityNotFoundException and adds corresponding validation errors
+     * to the context for each import row where the Observation Unit ID was not found.
+     *
+     * @param e The EntityNotFoundException containing information about missing Observation Unit IDs.
+     * @param context The AppendOverwriteMiddlewareContext containing import data and validation error storage.
+     *
+     * @implNote The method performs the following steps:
+     * 1. Retrieves the ValidationErrors object from the context.
+     * 2. Iterates through each import row in the context.
+     * 3. For each row, checks if its Observation Unit ID is in the set of missing entity IDs from the exception.
+     * 4. If a match is found, adds a validation error for that row, indicating an invalid Observation Unit ID.
+     * 5. The error is added using the addRowError method, specifying the OBS_UNIT_ID column and using a predefined error message.
+     */
+    public static void addValidationErrorsForObsUnitsNotFound(EntityNotFoundException e, AppendOverwriteMiddlewareContext context) {
+        ValidationErrors validationErrors = context.getAppendOverwriteWorkflowContext().getValidationErrors();
+        List<ValidationError> errors = new ArrayList<>();
 
-        return referenceOUIds;
+        for (int rowNum = 0; rowNum < context.getImportContext().getImportRows().size(); rowNum++) {
+            String rowObsUnitId = ((ExperimentObservation)context.getImportContext().getImportRows().get(rowNum)).getObsUnitID();
+            if (e.getMissingEntityIds().contains(rowObsUnitId)) {
+                addRowError(ExperimentObservation.Columns.OBS_UNIT_ID, ExperimentUtilities.INVALID_OBS_UNIT_ID_ERROR, validationErrors, rowNum);
+            }
+        }
     }
 
     /**

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/BrAPIAction.java

@@ -20,9 +20,11 @@
 import org.brapi.client.v2.model.exceptions.ApiException;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity;
+import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException;
 import org.breedinginsight.services.exceptions.DoesNotExistException;
 import org.breedinginsight.services.exceptions.MissingRequiredInfoException;
 import org.breedinginsight.services.exceptions.UnprocessableEntityException;
+import org.breedinginsight.services.exceptions.ValidatorException;
 
 import java.util.Optional;
 
@@ -42,7 +44,7 @@ public interface BrAPIAction<T> {
      * @return An Optional containing the relevant BrAPI state after executing the action.
      * @throws ApiException if an error occurs during the execution of the action.
      */
-    Optional<BrAPIState<T>> execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException;
+    Optional<BrAPIState<T>> execute() throws ApiException, MissingRequiredInfoException, UnprocessableEntityException, DoesNotExistException, EntityNotFoundException;
 
     /**
      * Get the BrAPI entity being acted on based on the provided ExpUnitMiddlewareContext.

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/action/WorkflowReadInitialization.java

@@ -23,6 +23,8 @@
 import org.brapi.client.v2.model.exceptions.ApiException;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.BrAPIState;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity.ExperimentImportEntity;
+import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException;
+import org.breedinginsight.services.exceptions.ValidatorException;
 import org.breedinginsight.utilities.Utilities;
 
 import java.util.List;
@@ -44,7 +46,7 @@ protected WorkflowReadInitialization(ExperimentImportEntity<T> entity) {
      * @return an Optional containing the BrAPIState representing the completed read workflow
      * @throws ApiException if an error occurs during execution
      */
-    public Optional<BrAPIState<T>> execute() throws ApiException {
+    public Optional<BrAPIState<T>> execute() throws ApiException, EntityNotFoundException {
         try {
             List<T> fetchedMembers = entity.brapiRead();
             entity.initializeWorkflow(fetchedMembers);

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/ProcessedDataFactory.java

@@ -112,6 +112,10 @@ public static EmptyData emptyData(String brapiReferenceSource,
         return new EmptyData(brapiReferenceSource, isCommit, germplasmName, study, phenoColumnName, trialId, studyId, unitId, studyYear, observationUnit, user, program, studyService, observationService);
     }
 
+    public static UndefinedDataset undefinedDataset() {
+        return new UndefinedDataset();
+    }
+
     @Bean
     @Prototype
     public InitialData initialDataBean(String brapiReferenceSource,
@@ -173,5 +177,11 @@ public EmptyData emptyDataBean(String brapiReferenceSource,
                                    Program program) {
         return emptyData(brapiReferenceSource, isCommit, germplasmName, study, phenoColumnName, trialId, studyId, unitId, studyYear, observationUnit, user, program, studyService, observationService);
     }
+
+    @Bean
+    @Prototype
+    public UndefinedDataset undefinedDatasetBean() {
+        return undefinedDataset();
+    }
 }
 

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/UndefinedDataset.java

@@ -0,0 +1,43 @@
+/*
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data;
+
+import org.brapi.v2.model.pheno.BrAPIObservation;
+import org.breedinginsight.api.model.v1.response.ValidationError;
+import org.breedinginsight.brapps.importer.model.response.PendingImportObject;
+import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.AppendStatistic;
+
+import java.util.List;
+import java.util.Optional;
+
+public class UndefinedDataset extends VisitedObservationData {
+    @Override
+    public Optional<List<ValidationError>> getValidationErrors() {
+        return Optional.empty();
+    }
+
+    @Override
+    public PendingImportObject<BrAPIObservation> constructPendingObservation() {
+        return null;
+    }
+
+    @Override
+    public void updateTally(AppendStatistic statistic) {
+
+    }
+}

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/ExperimentImportEntity.java

@@ -19,9 +19,11 @@
 
 import org.brapi.client.v2.model.exceptions.ApiException;
 import org.breedinginsight.brapps.importer.model.response.ImportObjectState;
+import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException;
 import org.breedinginsight.services.exceptions.DoesNotExistException;
 import org.breedinginsight.services.exceptions.MissingRequiredInfoException;
 import org.breedinginsight.services.exceptions.UnprocessableEntityException;
+import org.breedinginsight.services.exceptions.ValidatorException;
 
 import java.util.List;
 
@@ -43,7 +45,7 @@ public interface ExperimentImportEntity<T> {
      * @return List of fetched entities
      * @throws ApiException if there is an issue with the API call
      */
-    public List<T> brapiRead() throws ApiException;
+    public List<T> brapiRead() throws ApiException, EntityNotFoundException;
 
     /**
      * Commit objects changed by the workflow to the BrAPI service.

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java

@@ -26,6 +26,7 @@
 import org.breedinginsight.brapps.importer.model.response.ImportObjectState;
 import org.breedinginsight.brapps.importer.model.response.PendingImportObject;
 import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities;
+import org.breedinginsight.brapps.importer.services.processors.experiment.model.EntityNotFoundException;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext;
 import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext;
@@ -94,12 +95,12 @@ public List<BrAPIObservationUnit> brapiPost(List<BrAPIObservationUnit> members)
      * @throws ApiException if there is an issue with the API call
      */
     @Override
-    public List<BrAPIObservationUnit> brapiRead() throws ApiException {
+    public List<BrAPIObservationUnit> brapiRead() throws ApiException, EntityNotFoundException {
         // Collect deltabreed-generated obs unit ids listed in the import
         Set<String> obsUnitIds = cache.getReferenceOUIds();
 
         // For each id fetch the observation unit from the brapi data store
-        return observationUnitService.getObservationUnitsByDbId(new HashSet<>(obsUnitIds), importContext.getProgram());
+        return observationUnitService.getObservationUnitsById(new HashSet<>(obsUnitIds), importContext.getProgram());
     }
 
     /**