[BI-2110] New Append Exp Validations

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentUtilities.java

@@ -92,7 +92,7 @@ public ExperimentUtilities() {
      */
     public boolean isInvalidMemberListForClass(List<?> list, Class<?> clazz) {
         // Check if the input list is null, empty, or contains any member that is not an instance of the specified class
-        return list == null || list.isEmpty() || !list.stream().allMatch(clazz::isInstance);
+        return (list == null) || !list.stream().allMatch(clazz::isInstance);
     }
 
     /**
@@ -156,32 +156,35 @@ public static List<ExperimentObservation> importRowsToExperimentObservations(Lis
     }
 
     /**
-     * This method generates a unique key for an observation unit based on the environment and experimental unit ID.
+     * This method generates a unique key for an observation unit based on the environment and experimental unit ID and germplasm GID.
      *
-     * @param importRow the ExperimentObservation object containing the environment and experimental unit ID
+     * @param importRow the ExperimentObservation object containing the environment and experimental unit ID and germplasm GID
      * @return a String representing the unique key for the observation unit
      */
     public static String createObservationUnitKey(ExperimentObservation importRow) {
-        // Extract the environment and experimental unit ID from the ExperimentObservation object
+        // Extract the environment and experimental unit ID and germplasm GID from the ExperimentObservation object
         // and pass them to the createObservationUnitKey method
-        return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId());
+        return createObservationUnitKey(importRow.getEnv(), importRow.getExpUnitId(), importRow.getGid());
     }
 
     /**
      * Create Observation Unit Key
      *
-     * This method takes in the name of a study and the name of an observation unit and concatenates them to create a unique key.
+     * This method takes in the name of a study and the name of an observation unit and the germplasm name and concatenates them to create a unique key.
+     * Sub-observation unit name needed when repeated measures due to how they are created and named
      *
-     * If one or both of the inputs is null, returns an empty string since not a valid combination
+     * If any of the inputs are null, returns an empty string since not a valid combination
      *
      * @param studyName The name of the study
      * @param obsUnitName The name of the observation unit
-     * @return A string representing the unique key formed by concatenating the study name and observation unit name
+     * @param germplasmGID The germplasm gid
+     * @return A string representing the unique key formed by concatenating the study name and observation unit name and germplasm gid
      */
-    public static String createObservationUnitKey(String studyName, String obsUnitName) {
-        // Concatenate the study name and observation unit name to create the unique key
-        if (studyName != null && obsUnitName != null) {
-            return studyName + obsUnitName;
+    public static String createObservationUnitKey(String studyName, String obsUnitName, String germplasmGID) {
+        // Concatenate the study name and observation unit name and germplasm gid to create the unique key
+        if (studyName != null && obsUnitName != null && germplasmGID != null) {
+            String keyDelim = "@*";
+            return studyName + keyDelim + obsUnitName + keyDelim + germplasmGID;
         } else {
             return "";
         }

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/AppendOverwritePhenotypesWorkflow.java

@@ -19,6 +19,9 @@
 
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.breedinginsight.api.model.v1.response.ValidationErrors;
 import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext;
 import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse;
 import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics;
@@ -28,6 +31,7 @@
 import org.breedinginsight.brapps.importer.services.ImportStatusService;
 import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.AppendOverwriteIDValidation;
+import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.AppendOverwriteVariableValidation;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.commit.BrAPICommit;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.initialize.WorkflowInitialization;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.middleware.process.ImportTableProcess;
@@ -36,6 +40,7 @@
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.MiddlewareException;
 import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext;
+import org.breedinginsight.services.exceptions.ValidatorException;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
@@ -46,25 +51,56 @@
 @Singleton
 public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow {
     private final ExperimentWorkflowNavigator.Workflow workflow;
+    private final AppendOverwriteMiddleware validationMiddleware;
     private final AppendOverwriteMiddleware importPreviewMiddleware;
     private final AppendOverwriteMiddleware brapiCommitMiddleware;
     private final ImportStatusService statusService;
 
     @Inject
     public AppendOverwritePhenotypesWorkflow(AppendOverwriteIDValidation expUnitIDValidation,
+                                             AppendOverwriteVariableValidation obsVariableValidation,
                                              WorkflowInitialization workflowInitialization,
                                              ImportTableProcess importTableProcess,
                                              BrAPICommit brAPICommit,
                                              ImportStatusService statusService){
         this.statusService = statusService;
         this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE;
+        this.validationMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link(
+            expUnitIDValidation,
+            obsVariableValidation);
         this.importPreviewMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link(
-                expUnitIDValidation,
                 workflowInitialization,
                 importTableProcess);
         this.brapiCommitMiddleware = (AppendOverwriteMiddleware) AppendOverwriteMiddleware.link(brAPICommit);
     }
 
+    /**
+     * Determines if any validation or process errors are present in the context and handles result accordingly
+     *
+     * @param context The import service context containing upload, data, program, user, commit flag, and workflow information.
+     * @param result the ImportWorkflowResult to be modified in response to any errors found in context
+     * @return Pair containing a Boolean indicating whether any errors were found, and the potentially modified ImportWorkflowResult
+     */
+    public Pair<Boolean, Optional<ImportWorkflowResult>> checkForExistingErrors(AppendOverwriteMiddlewareContext context, Optional<ImportWorkflowResult> result){
+        // Stop and return any validation errors
+        Optional<ValidationErrors> validationErrorOptional = Optional
+                .ofNullable(context.getAppendOverwriteWorkflowContext().getValidationErrors());
+        if (validationErrorOptional.isPresent() && validationErrorOptional.get().hasErrors()){
+            result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.of(new ValidatorException(validationErrorOptional.get()))));
+            return new ImmutablePair<>(true, result);
+        }
+
+        // Stop and return any errors that occurred while processing
+        Optional<MiddlewareException> previewException = Optional
+                .ofNullable(context.getAppendOverwriteWorkflowContext().getProcessError());
+        if (previewException.isPresent()) {
+            log.debug(String.format("%s in %s", previewException.get().getException().getClass().getName(), previewException.get().getLocalTransactionName()));
+            result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(previewException.get().getException())));
+            return new ImmutablePair<>(true, result);
+        }
+        return new ImmutablePair<>(false, result);
+    }
+
     /**
      * Processes the import workflow based on the provided import service context.
      * If the provided context is not valid or if the workflow is not equal to the context workflow, returns an empty Optional.
@@ -101,6 +137,8 @@ public Optional<ImportWorkflowResult> process(ImportServiceContext context) {
             return result;
         }
 
+        Pair<Boolean, Optional<ImportWorkflowResult>> errors;
+
         // Build the workflow context for processing the import
         ImportContext importContext = ImportContext.builder()
                 .upload(context.getUpload())
@@ -115,16 +153,19 @@ public Optional<ImportWorkflowResult> process(ImportServiceContext context) {
                 .appendOverwriteWorkflowContext(new AppendOverwriteWorkflowContext())
                 .build();
 
+        // Validate the import
+        AppendOverwriteMiddlewareContext validatedImportContext = this.validationMiddleware.process(workflowContext);
+
+         //Stop and return any validation or process errors, needs to be done before processing import preview to avoid null pointer exceptions
+        errors = checkForExistingErrors(validatedImportContext, result);
+        if (errors.getLeft()) return errors.getRight();
+
         // Process the import preview
-        AppendOverwriteMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(workflowContext);
+        AppendOverwriteMiddlewareContext processedPreviewContext = this.importPreviewMiddleware.process(validatedImportContext);
 
-        // Stop and return any errors that occurred while processing
-        Optional<MiddlewareException> previewException = Optional.ofNullable(processedPreviewContext.getAppendOverwriteWorkflowContext().getProcessError());
-        if (previewException.isPresent()) {
-            log.debug(String.format("%s in %s", previewException.get().getException().getClass().getName(), previewException.get().getLocalTransactionName()));
-            result.ifPresent(importWorkflowResult -> importWorkflowResult.setCaughtException(Optional.ofNullable(previewException.get().getException())));
-            return result;
-        }
+        // Stop and return any validation or process errors
+        errors = checkForExistingErrors(validatedImportContext, result);
+        if (errors.getLeft()) return errors.getRight();
 
         // Build and return the preview response
         ImportPreviewResponse response = new ImportPreviewResponse();

src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingDataset.java

@@ -17,29 +17,28 @@
 
 package org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.entity;
 
+import io.micronaut.context.annotation.Property;
 import io.micronaut.context.annotation.Prototype;
 import com.google.gson.JsonArray;
 import org.brapi.client.v2.model.exceptions.ApiException;
+import org.brapi.v2.model.BrAPIExternalReference;
 import org.brapi.v2.model.core.BrAPIListSummary;
 import org.brapi.v2.model.core.request.BrAPIListNewRequest;
 import org.brapi.v2.model.core.response.BrAPIListDetails;
 import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields;
 import org.breedinginsight.brapi.v2.dao.BrAPIListDAO;
 import org.breedinginsight.brapps.importer.model.response.ImportObjectState;
 import org.breedinginsight.brapps.importer.model.response.PendingImportObject;
+import org.breedinginsight.brapps.importer.services.ExternalReferenceSource;
 import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteMiddlewareContext;
 import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.model.AppendOverwriteWorkflowContext;
 import org.breedinginsight.brapps.importer.services.processors.experiment.model.ImportContext;
 import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService;
-import org.breedinginsight.services.exceptions.DoesNotExistException;
-import org.breedinginsight.services.exceptions.MissingRequiredInfoException;
-import org.breedinginsight.services.exceptions.UnprocessableEntityException;
 import org.breedinginsight.utilities.DatasetUtil;
+import org.breedinginsight.utilities.Utilities;
 
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 @Prototype
@@ -49,16 +48,19 @@ public class PendingDataset implements ExperimentImportEntity<BrAPIListDetails>
     BrAPIListDAO brAPIListDAO;
     DatasetService datasetService;
     ExperimentUtilities experimentUtilities;
+    String referenceSourceBase;
 
     public PendingDataset(AppendOverwriteMiddlewareContext context,
                           BrAPIListDAO brAPIListDAO,
                           DatasetService datasetService,
-                          ExperimentUtilities experimentUtilities) {
+                          ExperimentUtilities experimentUtilities,
+                          String referenceSourceBase) {
         this.cache = context.getAppendOverwriteWorkflowContext();
         this.importContext = context.getImportContext();
         this.brAPIListDAO = brAPIListDAO;
         this.datasetService = datasetService;
         this.experimentUtilities = experimentUtilities;
+        this.referenceSourceBase = referenceSourceBase;
     }
     /**
      * Create new objects generated by the workflow in the BrAPI service.
@@ -103,14 +105,25 @@ public List<BrAPIListDetails> brapiPost(List<BrAPIListDetails> members) throws A
      */
     @Override
     public List<BrAPIListDetails> brapiRead() throws ApiException {
-        // Get the id of the dataset belonging to the required exp units
-        JsonArray datasetsJson = cache.getTrialByNameNoScope().values().iterator().next().getBrAPIObject()
-                .getAdditionalInfo()
-                .getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS);
-        String datasetId = DatasetUtil.getTopLevelDatasetFromJson(datasetsJson).getId().toString();
-
-        // Get the dataset belonging to required exp units
-        return List.of(datasetService.fetchDatasetById(datasetId, importContext.getProgram()).orElseThrow(ApiException::new));
+        // Get the dataset id from the xref of any observation unit in the import
+        List<BrAPIExternalReference> refs = cache
+                .getPendingObsUnitByOUId()
+                .values()
+                .stream()
+                .findFirst()
+                .get()
+                .getBrAPIObject()
+                .getExternalReferences();
+        String datasetId = Utilities
+                .getExternalReference(refs, referenceSourceBase, ExternalReferenceSource.DATASET)
+                .map(BrAPIExternalReference::getReferenceId)
+                .get();
+
+        // Get the dataset
+        return datasetService
+                .fetchDatasetById(datasetId, importContext.getProgram())
+                .map(List::of)
+                .orElseGet(List::of);
     }
 
     /**
@@ -230,18 +243,22 @@ public <U> void initializeWorkflow(List<U> members) {
                 .collect(Collectors.toMap(pio -> pio.getBrAPIObject().getListName(),pio -> pio));
 
         // Construct a hashmap to look up the pending dataset by the observation unit ID of a unit stored in the BrAPI service
-        Map<String, PendingImportObject<BrAPIListDetails>> pendingObsDatasetByOUId = cache.getPendingObsUnitByOUId().entrySet().stream()
-                .collect(Collectors.toMap(
-                        Map.Entry::getKey,
-                        e -> {
-                            if (cache.getPendingTrialByOUId().isEmpty() ||
-                                    pendingDatasetByName.isEmpty() ||
-                                    cache.getPendingTrialByOUId().values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) {
-                                throw new IllegalStateException("There is not an observation data set for this unit: " + e.getKey());
+        Map<String, PendingImportObject<BrAPIListDetails>> pendingObsDatasetByOUId;
+        if (pendingDatasetByName.isEmpty()) {
+            pendingObsDatasetByOUId = Collections.emptyMap();
+        } else {
+            pendingObsDatasetByOUId = cache.getPendingObsUnitByOUId().entrySet().stream()
+                    .collect(Collectors.toMap(
+                            Map.Entry::getKey,
+                            e -> {
+                                if (cache.getPendingTrialByOUId().isEmpty() ||
+                                        cache.getPendingTrialByOUId().values().iterator().next().getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS).isEmpty()) {
+                                    throw new IllegalStateException("There is not an observation data set for this unit: " + e.getKey());
+                                }
+                                return pendingDatasetByName.values().iterator().next();
                             }
-                            return pendingDatasetByName.values().iterator().next();
-                        }
-                ));
+                    ));
+        }
 
         // Add the maps to the context for use in processing import
         cache.setObsVarDatasetByName(pendingDatasetByName);