BI-1464 - Backend: Create Sub-Entity Dataset

src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java

@@ -9,12 +9,14 @@
 import io.micronaut.security.annotation.Secured;
 import io.micronaut.security.rules.SecurityRule;
 import lombok.extern.slf4j.Slf4j;
+import org.brapi.client.v2.model.exceptions.ApiException;
 import org.breedinginsight.api.auth.ProgramSecured;
 import org.breedinginsight.api.auth.ProgramSecuredRoleGroup;
 import org.breedinginsight.api.model.v1.response.Response;
 import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery;
 import org.breedinginsight.brapi.v2.services.BrAPITrialService;
 import org.breedinginsight.model.Dataset;
+import org.breedinginsight.model.DatasetMetadata;
 import org.breedinginsight.model.DownloadFile;
 import org.breedinginsight.model.Program;
 import org.breedinginsight.services.ProgramService;
@@ -23,6 +25,7 @@
 
 import javax.inject.Inject;
 import javax.validation.Valid;
+import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
 
@@ -89,4 +92,30 @@ public HttpResponse<Response<Dataset>> getDatasetData(
         }
     }
 
+    /**
+     * Retrieves the datasets for a given program and experiment.
+     *
+     * @param programId The UUID of the program.
+     * @param experimentId The UUID of the experiment.
+     * @return An HttpResponse with a Response object containing a list of DatasetMetadata.
+     * @throws DoesNotExistException if the program does not exist.
+     * @throws ApiException if an error occurs while retrieving the datasets.
+     */
+    @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/datasets")
+    @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL})
+    @Produces(MediaType.APPLICATION_JSON)
+    public HttpResponse<Response<List<DatasetMetadata>>> getDatasets(
+            @PathVariable("programId") UUID programId,
+            @PathVariable("experimentId") UUID experimentId) throws DoesNotExistException, ApiException {
+
+        Optional<Program> programOptional = programService.getById(programId);
+        if (programOptional.isEmpty()) {
+            return HttpResponse.status(HttpStatus.NOT_FOUND, "Program does not exist");
+        }
+
+        Response<List<DatasetMetadata>> response = new Response(experimentService.getDatasetsMetadata(programOptional.get(), experimentId));
+        return HttpResponse.ok(response);
+
+    }
+
 }

src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java

@@ -44,6 +44,7 @@ public final class BrAPIAdditionalInfoFields {
     public static final String ENVIRONMENT_NUMBER = "environmentNumber";
     public static final String STUDY_NAME = "studyName";
     public static final String OBSERVATION_DATASET_ID = "observationDatasetId";
+    public static final String DATASETS = "datasets";
     public static final String FEMALE_PARENT_UNKNOWN = "femaleParentUnknown";
     public static final String MALE_PARENT_UNKNOWN = "maleParentUnknown";
 	public static final String TREATMENTS = "treatments";

src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java

@@ -1,11 +1,16 @@
 package org.breedinginsight.brapi.v2.services;
 
+import com.google.gson.Gson;
+import com.google.gson.JsonArray;
+import com.google.gson.reflect.TypeToken;
 import io.micronaut.context.annotation.Property;
 import io.micronaut.http.MediaType;
 import io.micronaut.http.server.exceptions.InternalServerException;
 import io.micronaut.http.server.types.files.StreamedFile;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.brapi.client.v2.BrAPIClient;
+import org.brapi.client.v2.JSON;
 import org.brapi.client.v2.model.exceptions.ApiException;
 import org.brapi.v2.model.BrAPIExternalReference;
 import org.brapi.v2.model.core.*;
@@ -30,6 +35,7 @@
 import org.breedinginsight.services.TraitService;
 import org.breedinginsight.services.exceptions.DoesNotExistException;
 import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns;
+import org.breedinginsight.utilities.DatasetUtil;
 import org.breedinginsight.utilities.IntOrderComparator;
 import org.breedinginsight.utilities.FileUtil;
 import org.breedinginsight.utilities.Utilities;
@@ -40,6 +46,7 @@
 import java.io.IOException;
 import java.io.PipedInputStream;
 import java.io.PipedOutputStream;
+import java.lang.reflect.Type;
 import java.time.OffsetDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
@@ -327,6 +334,22 @@ public Dataset getDatasetData(Program program, UUID experimentId, UUID datsetId,
         return dataset;
     }
 
+    /**
+     * Retrieves the metadata of datasets associated with a program and experiment.
+     *
+     * @param program The program object representing the program that the datasets belong to.
+     * @param experimentId The UUID of the experiment that the datasets are associated with.
+     * @return A list of DatasetMetadata objects containing the metadata of the datasets.
+     * @throws DoesNotExistException If the trial does not exist for the program and experimentId combination.
+     * @throws ApiException If there is an error retrieving the trial or parsing the datasets metadata.
+     */
+    public List<DatasetMetadata> getDatasetsMetadata(Program program, UUID experimentId) throws DoesNotExistException, ApiException {
+        BrAPITrial trial = trialDAO.getTrialById(program.getId(), experimentId).orElseThrow(() -> new DoesNotExistException("Trial does not exist"));
+        JsonArray datasetsJson = trial.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS);
+        List<DatasetMetadata> datasets = DatasetUtil.datasetsFromJson(datasetsJson);
+        return datasets;
+    }
+
     private void addBrAPIObsToRecords(
             List<BrAPIObservation> dataset,
             BrAPITrial experiment,

src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java

@@ -33,10 +33,7 @@
 import org.breedinginsight.brapps.importer.model.config.*;
 import org.breedinginsight.brapps.importer.model.imports.BrAPIImport;
 import org.breedinginsight.brapps.importer.services.ExternalReferenceSource;
-import org.breedinginsight.model.BrAPIConstants;
-import org.breedinginsight.model.Program;
-import org.breedinginsight.model.ProgramLocation;
-import org.breedinginsight.model.User;
+import org.breedinginsight.model.*;
 import org.breedinginsight.utilities.Utilities;
 
 import java.math.BigInteger;
@@ -284,8 +281,9 @@ public BrAPIObservationUnit constructBrAPIObservationUnit(
 
         BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition();
         BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship();
-        level.setLevelName("plot");  //BreedBase only accepts "plot" or "plant"
+        level.setLevelName(getExpUnit());
         level.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal));
+        level.setLevelOrder(DatasetLevel.EXP_UNIT.getValue());
         position.setObservationLevel(level);
         observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit());
 
@@ -368,6 +366,69 @@ public BrAPIObservationUnit constructBrAPIObservationUnit(
         return observationUnit;
     }
 
+    /**
+     * Constructs a BrAPI sub-observation unit based on the given parameters. Only a subset of the information is
+     * included as the exp unit level contains all the relevant top-level details and is linked from this unit
+     *
+     * Uses only the following methods from ExperimentObservation:
+     * - getSubObsUnit
+     * - getSubUnitId
+     *
+     * Existing ExpUnit data is from passed in BrAPI object which is looked up based on ObsUnitID, no other values
+     * in import file are required for creating a sub observation unit dataset
+     */
+    public BrAPIObservationUnit constructBrAPISubObservationUnit(
+            Program program,
+            String seqVal,
+            boolean commit,
+            BrAPIObservationUnit expUnit,
+            String referenceSource,
+            UUID datasetId,
+            UUID id
+    ) {
+
+        BrAPIObservationUnit observationUnit = new BrAPIObservationUnit();
+        if (commit) {
+            observationUnit.setObservationUnitName(Utilities.appendProgramKey(getSubObsUnit(), program.getKey(), seqVal));
+            observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, UUID.fromString(expUnit.getTrialDbId()),
+                    datasetId, UUID.fromString(expUnit.getStudyDbId()), id));
+        } else {
+            observationUnit.setObservationUnitName(getSubObsUnit());
+        }
+        observationUnit.setStudyName(expUnit.getStudyName());
+
+        observationUnit.setGermplasmName(expUnit.getGermplasmName());
+        String gid = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString();
+        observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, gid);
+
+        BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition();
+
+        // observationLevel entry for Sub-Obs Unit
+        BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship();
+        level.setLevelName(getSubObsUnit());
+        level.setLevelCode(Utilities.appendProgramKey(getSubUnitId(), program.getKey(), seqVal));
+        level.setLevelOrder(DatasetLevel.SUB_OBS_UNIT.getValue());
+        position.setObservationLevel(level);
+
+        // keep this in case we decide to rename levels in future
+        observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getSubObsUnit());
+
+        // observationLevelRelationships for top-level Exp Unit linking
+        List<BrAPIObservationUnitLevelRelationship> levelRelationships = new ArrayList<>();
+        BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship();
+
+        // set name without added keys
+        expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString());
+        expUnitLevel.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal));
+        expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue());
+        levelRelationships.add(expUnitLevel);
+
+        // TODO: Do replicate and block matter for field book?
+
+        observationUnit.setObservationUnitPosition(position);
+        return observationUnit;
+    }
+
     public BrAPIObservation constructBrAPIObservation(
             String value,
             String variableName,

src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java

@@ -19,6 +19,7 @@
 
 import com.google.gson.Gson;
 import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
 import io.micronaut.context.annotation.Property;
 import io.micronaut.context.annotation.Prototype;
@@ -65,6 +66,7 @@
 import org.breedinginsight.services.exceptions.MissingRequiredInfoException;
 import org.breedinginsight.services.exceptions.UnprocessableEntityException;
 import org.breedinginsight.services.exceptions.ValidatorException;
+import org.breedinginsight.utilities.DatasetUtil;
 import org.breedinginsight.utilities.Utilities;
 import org.jooq.DSLContext;
 import tech.tablesaw.api.Table;
@@ -132,6 +134,12 @@ public class ExperimentProcessor implements Processor {
     private Map<String, PendingImportObject<BrAPIObservationUnit>> observationUnitByNameNoScope = null;
     private Map<String, PendingImportObject<BrAPIObservationUnit>> pendingObsUnitByOUId = new HashMap<>();
 
+    // for use when creating sub obs units
+    private Map<String, PendingImportObject<BrAPIObservationUnit>> obsUnitByExpObsUnitId = null;
+    private Map<String, PendingImportObject<BrAPIObservationUnit>> subObsUnitBySubObsUnitId = null;
+
+    private ExperimentProcessorWorkflow workflow = new ExperimentProcessorWorkflow();
+
     private final Map<String, PendingImportObject<BrAPIObservation>> observationByHash = new HashMap<>();
     private Map<String, BrAPIObservation> existingObsByObsHash = new HashMap<>();
     // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the initNewBrapiData() method
@@ -175,6 +183,12 @@ public String getName() {
         return NAME;
     }
 
+    @Override
+    public void initialize(List<BrAPIImport> importRows) {
+        // determine experiment import workflow based on file contents
+        workflow.determineWorkflow(importRows);
+    }
+
     /**
      * Initialize the Map<String, PendingImportObject> objects with existing BrAPI Data.
      *
@@ -184,6 +198,8 @@ public String getName() {
     @Override
     public void getExistingBrapiData(List<BrAPIImport> importRows, Program program) {
 
+
+
         List<ExperimentObservation> experimentImportRows = importRows.stream()
                                                                      .map(trialImport -> (ExperimentObservation) trialImport)
                                                                      .collect(Collectors.toList());
@@ -1132,6 +1148,7 @@ private PendingImportObject<BrAPIGermplasm> getGidPIO(ExperimentObservation impo
 
     private PendingImportObject<BrAPIObservationUnit> fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException {
         PendingImportObject<BrAPIObservationUnit> pio;
+        // TODO: Should be based on ObsUnitID
         String key = createObservationUnitKey(importRow);
         if (hasAllReferenceUnitIds) {
             pio = pendingObsUnitByOUId.get(importRow.getObsUnitID());
@@ -1148,6 +1165,7 @@ private PendingImportObject<BrAPIObservationUnit> fetchOrCreateObsUnitPIO(Progra
             UUID trialID = trialPIO.getId();
             UUID datasetId = null;
             if (commit) {
+                // TODO: BI-1464 get dataset id from array
                 datasetId = UUID.fromString(trialPIO.getBrAPIObject()
                         .getAdditionalInfo().getAsJsonObject()
                         .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString());
@@ -1284,7 +1302,9 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr
         PendingImportObject<BrAPIListDetails> pio;
         PendingImportObject<BrAPITrial> trialPIO = hasAllReferenceUnitIds ?
                 getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle());
-        String name = String.format("Observation Dataset [%s-%s]",
+        String datasetName = StringUtils.isNotBlank(importRow.getSubObsUnit()) ? importRow.getSubObsUnit() : importRow.getExpUnit();
+        String name = String.format("%s Observation Dataset [%s-%s]",
+                datasetName,
                 program.getKey(),
                 trialPIO.getBrAPIObject()
                         .getAdditionalInfo()
@@ -1301,7 +1321,25 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr
                     program,
                     trialPIO.getId().toString());
             pio = new PendingImportObject<BrAPIListDetails>(ImportObjectState.NEW, newDataset, id);
+
+            // TODO:BI-1464 remove old id and update code to use datasets array - existing will need migration
             trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString());
+
+            JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS);
+            // If datasets property does not yet exist, create it
+
+            List<DatasetMetadata> datasets = DatasetUtil.datasetsFromJson(datasetsJson);
+            DatasetMetadata dataset = DatasetMetadata.builder()
+                    .name(datasetName)
+                    .id(id)
+                    .level(StringUtils.isNotBlank(importRow.getSubObsUnit()) ? DatasetLevel.SUB_OBS_UNIT : DatasetLevel.EXP_UNIT)
+                    .build();
+
+            log.debug(dataset.getName());
+            datasets.add(dataset);
+            datasetsJson = DatasetUtil.jsonArrayFromDatasets(datasets);
+            trialPIO.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, datasetsJson);
+
             if (ImportObjectState.EXISTING == trialPIO.getState()) {
                 trialPIO.setState(ImportObjectState.MUTATED);
             }
@@ -1615,6 +1653,13 @@ private Map<String, PendingImportObject<BrAPIObservationUnit>> initializeObserva
         try {
             List<BrAPIObservationUnit> existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program);
 
+            // TODO: grab from externalReferences
+            /*
+            observationUnitByObsUnitId = existingObsUnits.stream()
+                    .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId,
+                            (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false)));
+             */
+
             String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName());
             if (existingObsUnits.size() == rowByObsUnitId.size()) {
                 existingObsUnits.forEach(brAPIObservationUnit -> {
@@ -2067,6 +2112,7 @@ private Map<String, PendingImportObject<BrAPIListDetails>> initializeObsVarDatas
 
         Optional<PendingImportObject<BrAPITrial>> trialPIO = getTrialPIO(experimentImportRows);
 
+        // TODO: BI-1464 use value in datasets array instead
         if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) {
             String datasetId = trialPIO.get().getBrAPIObject()
                     .getAdditionalInfo()