From 241a57ffde810e20d61076fea8facadddd666aab Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:34:32 -0400 Subject: [PATCH 1/3] [BI-2134] - made changes for troubleshooting --- .../model/imports/DomainImportService.java | 6 +- .../ExperimentWorkflowNavigator.java | 25 +- .../AppendOverwritePhenotypesWorkflow.java | 54 --- .../workflow/CreateNewExperimentWorkflow.java | 307 ------------------ 4 files changed, 5 insertions(+), 387 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 6cfffe73c..4ffea6ba6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -26,7 +26,7 @@ import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; +import org.breedinginsight.brapps.importer.services.processors.experiment.WorkflowEnum; import javax.inject.Inject; import javax.inject.Provider; @@ -43,7 +43,7 @@ public abstract class DomainImportService implements BrAPIImportService { private final Provider processorManagerProvider; private final Workflow workflowNavigator; - + @Inject public DomainImportService(Provider experimentProcessorProvider, Provider processorManagerProvider, Workflow workflowNavigator) @@ -72,7 +72,7 @@ public ImportPreviewResponse process(ImportServiceContext context) // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); - if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflowId())) { + if (WorkflowEnum.NEW_OBSERVATION.getId().equals(context.getWorkflowId())) { Optional result = workflowNavigator.process(context); // Throw any exceptions caught during workflow processing diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index 6acd4aad5..811e24104 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -6,6 +6,7 @@ import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import javax.inject.Inject; import javax.inject.Singleton; import java.util.List; import java.util.Optional; @@ -16,6 +17,7 @@ public class ExperimentWorkflowNavigator implements ExperimentWorkflow { private final List workflows; + @Inject public ExperimentWorkflowNavigator(List workflows) { this.workflows = workflows; } @@ -61,27 +63,4 @@ public List getWorkflows() { return workflowSummaryList; // Return the list of workflow metadata } - - public enum Workflow { - NEW_OBSERVATION("new-experiment","Create new experiment"), - APPEND_OVERWRITE("append-dataset", "Append experimental dataset"); - - private String id; - private String name; - - Workflow(String id, String name) { - - this.id = id; - this.name = name; - } - - public String getId() { - return id; - } - public String getName() { return name; } - - public boolean isEqual(String value) { - return Optional.ofNullable(id.equals(value)).orElse(false); - } - } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index ea5c388cb..e69de29bb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -1,54 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; - -import lombok.Getter; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; - -import javax.inject.Singleton; -import java.util.Optional; - -@Getter -@Singleton -public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { - private final ExperimentWorkflowNavigator.Workflow workflow; - - public AppendOverwritePhenotypesWorkflow(){ - this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; - } - - public Optional process(ImportServiceContext context) { - // Workflow processing the context - ImportWorkflow workflow = ImportWorkflow.builder() - .id(getWorkflow().getId()) - .name(getWorkflow().getName()) - .build(); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending or overwriting observation data - if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 2; - } - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index d0a4ca975..e69de29bb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -1,307 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; - -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.HttpStatus; -import io.micronaut.http.exceptions.HttpStatusException; -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.apache.commons.lang3.StringUtils; -import org.brapi.v2.model.pheno.BrAPIObservation; -import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.PendingImport; -import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.ImportStatusService; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.CommitPendingImportObjectsStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateExistingPendingImportObjectsStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateNewPendingImportObjectsStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ValidatePendingImportObjectsStep; -import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; -import org.breedinginsight.services.exceptions.ValidatorException; - -import javax.inject.Inject; -import javax.inject.Named; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; - -import lombok.Getter; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; - -import javax.inject.Singleton; -import java.util.Optional; - -@Slf4j -@Getter -@Singleton -public class CreateNewExperimentWorkflow implements ExperimentWorkflow { - private final ExperimentWorkflowNavigator.Workflow workflow; - private final PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep; - private final PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep; - private final CommitPendingImportObjectsStep commitPendingImportObjectsStep; - private final ValidatePendingImportObjectsStep validatePendingImportObjectsStep; - private final ImportStatusService statusService; - private final ExperimentPhenotypeService experimentPhenotypeService; - - @Inject - public CreateNewExperimentWorkflow(PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep, - PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep, - CommitPendingImportObjectsStep commitPendingImportObjectsStep, - ValidatePendingImportObjectsStep validatePendingImportObjectsStep, - ImportStatusService statusService, - ExperimentPhenotypeService experimentPhenotypeService) { - this.populateExistingPendingImportObjectsStep = populateExistingPendingImportObjectsStep; - this.populateNewPendingImportObjectsStep = populateNewPendingImportObjectsStep; - this.commitPendingImportObjectsStep = commitPendingImportObjectsStep; - this.validatePendingImportObjectsStep = validatePendingImportObjectsStep; - this.statusService = statusService; - this.experimentPhenotypeService = experimentPhenotypeService; - this.workflow = ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION; - } - - private ImportPreviewResponse runWorkflow(ImportContext context) throws Exception { - - ImportUpload upload = context.getUpload(); - boolean commit = context.isCommit(); - List importRows = context.getImportRows(); - ProcessedData processedData = new ProcessedData(); - - // Make sure the file does not contain obs unit ids before proceeding - if (containsObsUnitIDs(context)) { - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "ObsUnitIDs are detected"); - } - - statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); - - ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); - ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); - populateNewPendingImportObjectsStep.process(processContext, phenotypeData); - ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context, processContext.getPendingData(), phenotypeData, processedData); - - // short circuit if there were validation errors - if (validationErrors.hasErrors()) { - throw new ValidatorException(validationErrors); - } - - // TODO: move to experiment import service - ImportPreviewResponse response = buildImportPreviewResponse(importRows, processContext.getPendingData(), processedData, upload); - - statusService.updateMappedData(upload, response, "Finished mapping data to brapi objects"); - - // preview data - if (!commit) { - statusService.updateOk(upload); - return response; - } - - // commit data - long totalObjects = getNewObjectCount(response); - statusService.startUpload(upload, totalObjects, "Starting upload to brapi service"); - statusService.updateMessage(upload, "Creating new experiment objects in brapi service"); - - commitPendingImportObjectsStep.process(processContext, processedData); - - statusService.finishUpload(upload, totalObjects, "Completed upload to brapi service"); - return response; - } - - /** - * Process the import service context and returns an Optional ImportWorkflowResult. - * - * @param context The import service context to be processed. If null, then it skips processing but returns the result with no-preview. - * @return An Optional ImportWorkflowResult which contains the workflow and import preview response (if available). - * If the context is null, it returns the result with no-preview. - */ - public Optional process(ImportServiceContext context) { - // Workflow processing the context - ImportWorkflow workflow = ImportWorkflow.builder() - .id(getWorkflow().getId()) - .name(getWorkflow().getName()) - .build(); - - // No-preview result - ImportWorkflowResult workflowResult = ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.empty()) - .caughtException(Optional.empty()) - .build(); - - // Skip this workflow unless creating a new experiment - if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return Optional.of(workflowResult); - } - - // TODO: unify usage of single import context type throughout - ImportContext importContext = ImportContext.from(context); - - // Start processing the import... - ImportPreviewResponse response; - try { - response = runWorkflow(importContext); - workflowResult.setImportPreviewResponse(Optional.of(response)); - } catch(Exception e) { - workflowResult.setCaughtException(Optional.of(e)); - } - - return Optional.of(workflowResult); - } - - @Override - public int getOrder() { - return 1; - } - - // TODO: move to shared area - private ImportPreviewResponse buildImportPreviewResponse(List importRows, PendingData pendingData, ProcessedData processedData, - ImportUpload upload) { - - Map mappedBrAPIImport = processedData.getMappedBrAPIImport(); - Map statistics = generateStatisticsMap(pendingData, importRows); - - ImportPreviewResponse response = new ImportPreviewResponse(); - response.setStatistics(statistics); - List mappedBrAPIImportList = new ArrayList<>(mappedBrAPIImport.values()); - response.setRows(mappedBrAPIImportList); - response.setDynamicColumnNames(upload.getDynamicColumnNamesList()); - return response; - } - - // TODO: move to shared area - private long getNewObjectCount(ImportPreviewResponse response) { - // get total number of new brapi objects to create - long totalObjects = 0; - for (ImportPreviewStatistics stats : response.getStatistics().values()) { - totalObjects += stats.getNewObjectCount(); - } - return totalObjects; - } - - private boolean containsObsUnitIDs(ImportContext importContext) { - List importRows = importContext.getImportRows(); - return importRows.stream() - .anyMatch(row -> { - ExperimentObservation expRow = (ExperimentObservation) row; - return StringUtils.isNotBlank(expRow.getObsUnitID()); - }); - } - - // TODO: move to shared area: experiment import service - private Map generateStatisticsMap(PendingData pendingData, List importRows) { - // Data for stats. - HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names - HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's - HashSet gidCounter = new HashSet<>(); // set of unique GID's - - Map> observationByHash = pendingData.getObservationByHash(); - - for (BrAPIImport row : importRows) { - ExperimentObservation importRow = (ExperimentObservation) row; - // Collect date for stats. - addIfNotNull(environmentNameCounter, importRow.getEnv()); - addIfNotNull(obsUnitsIDCounter, ExperimentUtilities.createObservationUnitKey(importRow)); - addIfNotNull(gidCounter, importRow.getGid()); - } - - int numNewObservations = Math.toIntExact( - observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - int numExistingObservations = Math.toIntExact( - observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - int numMutatedObservations = Math.toIntExact( - observationByHash.values() - .stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && - !StringUtils.isBlank(preview.getBrAPIObject() - .getValue())) - .count() - ); - - ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() - .newObjectCount(environmentNameCounter.size()) - .build(); - ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() - .newObjectCount(obsUnitsIDCounter.size()) - .build(); - ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() - .newObjectCount(gidCounter.size()) - .build(); - ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() - .newObjectCount(numNewObservations) - .build(); - ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() - .newObjectCount(numExistingObservations) - .build(); - ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() - .newObjectCount(numMutatedObservations) - .build(); - - return Map.of( - "Environments", environmentStats, - "Observation_Units", obdUnitStats, - "GIDs", gidStats, - "Observations", observationStats, - "Existing_Observations", existingObservationStats, - "Mutated_Observations", mutatedObservationStats - ); - } - - // TODO: move to common area - private void addIfNotNull(HashSet set, String setValue) { - if (setValue != null) { - set.add(setValue); - } - } -} - From 9d40e55c40eb7ffcd6aa06b7eb4f32dc4df04fd9 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 3 Jul 2024 15:37:28 -0400 Subject: [PATCH 2/3] [BI-2134] - added file --- .../processors/experiment/WorkflowEnum.java | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/WorkflowEnum.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/WorkflowEnum.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/WorkflowEnum.java new file mode 100644 index 000000000..22f5600e6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/WorkflowEnum.java @@ -0,0 +1,26 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import java.util.Optional; + +public enum WorkflowEnum { + NEW_OBSERVATION("new-experiment","Create new experiment"), + APPEND_OVERWRITE("append-dataset", "Append experimental dataset"); + + private String id; + private String name; + + WorkflowEnum(String id, String name) { + + this.id = id; + this.name = name; + } + + public String getId() { + return id; + } + public String getName() { return name; } + + public boolean isEqual(String value) { + return Optional.ofNullable(id.equals(value)).orElse(false); + } +} \ No newline at end of file From b4fb5125e0cb704dd94db502e1c522f039d02e5d Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 3 Jul 2024 17:03:03 -0400 Subject: [PATCH 3/3] [BI-2134] - added back files --- .../AppendOverwritePhenotypesWorkflow.java | 54 ++++ .../workflow/CreateNewExperimentWorkflow.java | 304 ++++++++++++++++++ 2 files changed, 358 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index e69de29bb..8dcafdcc6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -0,0 +1,54 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; + +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.WorkflowEnum; + +import javax.inject.Singleton; +import java.util.Optional; + +@Getter +@Singleton +public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { + private final WorkflowEnum workflow; + + public AppendOverwritePhenotypesWorkflow(){ + this.workflow = WorkflowEnum.APPEND_OVERWRITE; + } + + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending or overwriting observation data + if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; + } + + @Override + public int getOrder() { + return 2; + } + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index e69de29bb..692518041 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -0,0 +1,304 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import io.micronaut.http.HttpStatus; +import io.micronaut.http.exceptions.HttpStatusException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.model.workflow.ImportContext; +import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import org.breedinginsight.brapps.importer.services.ImportStatusService; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; +import org.breedinginsight.brapps.importer.services.processors.experiment.WorkflowEnum; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.PendingData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessContext; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.CommitPendingImportObjectsStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateExistingPendingImportObjectsStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.PopulateNewPendingImportObjectsStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow.steps.ValidatePendingImportObjectsStep; +import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentPhenotypeService; +import org.breedinginsight.services.exceptions.ValidatorException; + +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; + +import javax.inject.Singleton; +import java.util.Optional; + +@Slf4j +@Getter +@Singleton +public class CreateNewExperimentWorkflow implements ExperimentWorkflow { + private final WorkflowEnum workflow; + private final PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep; + private final PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep; + private final CommitPendingImportObjectsStep commitPendingImportObjectsStep; // BAD + private final ValidatePendingImportObjectsStep validatePendingImportObjectsStep; + private final ImportStatusService statusService; + private final ExperimentPhenotypeService experimentPhenotypeService; // BAD + + @Inject + public CreateNewExperimentWorkflow( + PopulateExistingPendingImportObjectsStep populateExistingPendingImportObjectsStep, + PopulateNewPendingImportObjectsStep populateNewPendingImportObjectsStep, + CommitPendingImportObjectsStep commitPendingImportObjectsStep, + ValidatePendingImportObjectsStep validatePendingImportObjectsStep, + ImportStatusService statusService, + ExperimentPhenotypeService experimentPhenotypeService + ) { + this.populateExistingPendingImportObjectsStep = populateExistingPendingImportObjectsStep; + this.populateNewPendingImportObjectsStep = populateNewPendingImportObjectsStep; + this.commitPendingImportObjectsStep = commitPendingImportObjectsStep; + this.validatePendingImportObjectsStep = validatePendingImportObjectsStep; + this.statusService = statusService; + this.experimentPhenotypeService = experimentPhenotypeService; + this.workflow = WorkflowEnum.NEW_OBSERVATION; + } + + private ImportPreviewResponse runWorkflow(ImportContext context) throws Exception { + ImportUpload upload = context.getUpload(); + boolean commit = context.isCommit(); + List importRows = context.getImportRows(); + ProcessedData processedData = new ProcessedData(); + + // Make sure the file does not contain obs unit ids before proceeding + if (containsObsUnitIDs(context)) { + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "ObsUnitIDs are detected"); + } + + statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); + + ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); + ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); + populateNewPendingImportObjectsStep.process(processContext, phenotypeData); + ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context, processContext.getPendingData(), phenotypeData, processedData); + + // short circuit if there were validation errors + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + + // TODO: move to experiment import service + ImportPreviewResponse response = buildImportPreviewResponse(importRows, processContext.getPendingData(), processedData, upload); + + statusService.updateMappedData(upload, response, "Finished mapping data to brapi objects"); + + // preview data + if (!commit) { + statusService.updateOk(upload); + return response; + } + + // commit data + long totalObjects = getNewObjectCount(response); + statusService.startUpload(upload, totalObjects, "Starting upload to brapi service"); + statusService.updateMessage(upload, "Creating new experiment objects in brapi service"); + + commitPendingImportObjectsStep.process(processContext, processedData); + + statusService.finishUpload(upload, totalObjects, "Completed upload to brapi service"); + return response; + } + + /** + * Process the import service context and returns an Optional ImportWorkflowResult. + * + * @param context The import service context to be processed. If null, then it skips processing but returns the result with no-preview. + * @return An Optional ImportWorkflowResult which contains the workflow and import preview response (if available). + * If the context is null, it returns the result with no-preview. + */ + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) + .build(); + + // No-preview result + ImportWorkflowResult workflowResult = ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .caughtException(Optional.empty()) + .build(); + + // Skip this workflow unless creating a new experiment + if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return Optional.of(workflowResult); + } + + // TODO: unify usage of single import context type throughout + ImportContext importContext = ImportContext.from(context); + + // Start processing the import... + ImportPreviewResponse response; + try { + response = runWorkflow(importContext); + workflowResult.setImportPreviewResponse(Optional.of(response)); + } catch(Exception e) { + workflowResult.setCaughtException(Optional.of(e)); + } + + return Optional.of(workflowResult); + } + + @Override + public int getOrder() { + return 1; + } + + // TODO: move to shared area + private ImportPreviewResponse buildImportPreviewResponse(List importRows, PendingData pendingData, ProcessedData processedData, + ImportUpload upload) { + + Map mappedBrAPIImport = processedData.getMappedBrAPIImport(); + Map statistics = generateStatisticsMap(pendingData, importRows); + + ImportPreviewResponse response = new ImportPreviewResponse(); + response.setStatistics(statistics); + List mappedBrAPIImportList = new ArrayList<>(mappedBrAPIImport.values()); + response.setRows(mappedBrAPIImportList); + response.setDynamicColumnNames(upload.getDynamicColumnNamesList()); + return response; + } + + // TODO: move to shared area + private long getNewObjectCount(ImportPreviewResponse response) { + // get total number of new brapi objects to create + long totalObjects = 0; + for (ImportPreviewStatistics stats : response.getStatistics().values()) { + totalObjects += stats.getNewObjectCount(); + } + return totalObjects; + } + + private boolean containsObsUnitIDs(ImportContext importContext) { + List importRows = importContext.getImportRows(); + return importRows.stream() + .anyMatch(row -> { + ExperimentObservation expRow = (ExperimentObservation) row; + return StringUtils.isNotBlank(expRow.getObsUnitID()); + }); + } + + // TODO: move to shared area: experiment import service + private Map generateStatisticsMap(PendingData pendingData, List importRows) { + // Data for stats. + HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names + HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's + HashSet gidCounter = new HashSet<>(); // set of unique GID's + + Map> observationByHash = pendingData.getObservationByHash(); + + for (BrAPIImport row : importRows) { + ExperimentObservation importRow = (ExperimentObservation) row; + // Collect date for stats. + addIfNotNull(environmentNameCounter, importRow.getEnv()); + addIfNotNull(obsUnitsIDCounter, ExperimentUtilities.createObservationUnitKey(importRow)); + addIfNotNull(gidCounter, importRow.getGid()); + } + + int numNewObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numExistingObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + int numMutatedObservations = Math.toIntExact( + observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() + .newObjectCount(environmentNameCounter.size()) + .build(); + ImportPreviewStatistics obdUnitStats = ImportPreviewStatistics.builder() + .newObjectCount(obsUnitsIDCounter.size()) + .build(); + ImportPreviewStatistics gidStats = ImportPreviewStatistics.builder() + .newObjectCount(gidCounter.size()) + .build(); + ImportPreviewStatistics observationStats = ImportPreviewStatistics.builder() + .newObjectCount(numNewObservations) + .build(); + ImportPreviewStatistics existingObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numExistingObservations) + .build(); + ImportPreviewStatistics mutatedObservationStats = ImportPreviewStatistics.builder() + .newObjectCount(numMutatedObservations) + .build(); + + return Map.of( + "Environments", environmentStats, + "Observation_Units", obdUnitStats, + "GIDs", gidStats, + "Observations", observationStats, + "Existing_Observations", existingObservationStats, + "Mutated_Observations", mutatedObservationStats + ); + } + + // TODO: move to common area + private void addIfNotNull(HashSet set, String setValue) { + if (setValue != null) { + set.add(setValue); + } + } +} +