From 434d98ac2a4e4cff7b90358caaf0e8ddd90bdf82 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 22 Feb 2024 14:10:27 -0500 Subject: [PATCH 1/4] Work in progress on datasets api --- .../v1/controller/ExperimentController.java | 29 +++++ .../constants/BrAPIAdditionalInfoFields.java | 1 + .../brapi/v2/services/BrAPITrialService.java | 23 ++++ .../processors/ExperimentProcessor.java | 20 +++- .../org/breedinginsight/model/Dataset.java | 17 +++ .../breedinginsight/model/DatasetLevel.java | 34 ++++++ .../model/DatasetMetadata.java | 38 ++++++ .../utilities/DatasetUtil.java | 28 +++++ .../brapi/v2/DatasetsIntegrationTest.java | 113 ++++++++++++++++++ 9 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/breedinginsight/model/DatasetLevel.java create mode 100644 src/main/java/org/breedinginsight/model/DatasetMetadata.java create mode 100644 src/main/java/org/breedinginsight/utilities/DatasetUtil.java create mode 100644 src/test/java/org/breedinginsight/brapi/v2/DatasetsIntegrationTest.java diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 017197219..0a166ae32 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -9,12 +9,14 @@ import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.api.model.v1.response.Response; import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.model.Dataset; +import org.breedinginsight.model.DatasetMetadata; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; @@ -23,6 +25,7 @@ import javax.inject.Inject; import javax.validation.Valid; +import java.util.List; import java.util.Optional; import java.util.UUID; @@ -89,4 +92,30 @@ public HttpResponse> getDatasetData( } } + /** + * Retrieves the datasets for a given program and experiment. + * + * @param programId The UUID of the program. + * @param experimentId The UUID of the experiment. + * @return An HttpResponse with a Response object containing a list of DatasetMetadata. + * @throws DoesNotExistException if the program does not exist. + * @throws ApiException if an error occurs while retrieving the datasets. + */ + @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/datasets") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + @Produces(MediaType.APPLICATION_JSON) + public HttpResponse>> getDatasets( + @PathVariable("programId") UUID programId, + @PathVariable("experimentId") UUID experimentId) throws DoesNotExistException, ApiException { + + Optional programOptional = programService.getById(programId); + if (programOptional.isEmpty()) { + return HttpResponse.status(HttpStatus.NOT_FOUND, "Program does not exist"); + } + + Response> response = new Response(experimentService.getDatasetsMetadata(programOptional.get(), experimentId)); + return HttpResponse.ok(response); + + } + } diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 87fbfdda9..0ac5afea9 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -44,6 +44,7 @@ public final class BrAPIAdditionalInfoFields { public static final String ENVIRONMENT_NUMBER = "environmentNumber"; public static final String STUDY_NAME = "studyName"; public static final String OBSERVATION_DATASET_ID = "observationDatasetId"; + public static final String DATASETS = "datasets"; public static final String FEMALE_PARENT_UNKNOWN = "femaleParentUnknown"; public static final String MALE_PARENT_UNKNOWN = "maleParentUnknown"; public static final String TREATMENTS = "treatments"; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 98c07a23e..ddb3cfa4c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,11 +1,16 @@ package org.breedinginsight.brapi.v2.services; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.reflect.TypeToken; import io.micronaut.context.annotation.Property; import io.micronaut.http.MediaType; import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; @@ -30,6 +35,7 @@ import org.breedinginsight.services.TraitService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.IntOrderComparator; import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; @@ -40,6 +46,7 @@ import java.io.IOException; import java.io.PipedInputStream; import java.io.PipedOutputStream; +import java.lang.reflect.Type; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; @@ -327,6 +334,22 @@ public Dataset getDatasetData(Program program, UUID experimentId, UUID datsetId, return dataset; } + /** + * Retrieves the metadata of datasets associated with a program and experiment. + * + * @param program The program object representing the program that the datasets belong to. + * @param experimentId The UUID of the experiment that the datasets are associated with. + * @return A list of DatasetMetadata objects containing the metadata of the datasets. + * @throws DoesNotExistException If the trial does not exist for the program and experimentId combination. + * @throws ApiException If there is an error retrieving the trial or parsing the datasets metadata. + */ + public List getDatasetsMetadata(Program program, UUID experimentId) throws DoesNotExistException, ApiException { + BrAPITrial trial = trialDAO.getTrialById(program.getId(), experimentId).orElseThrow(() -> new DoesNotExistException("Trial does not exist")); + JsonArray datasetsJson = trial.getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + List datasets = DatasetUtil.datasetsFromJson(datasetsJson); + return datasets; + } + private void addBrAPIObsToRecords( List dataset, BrAPITrial experiment, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c68932453..233a77725 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -19,6 +19,7 @@ import com.google.gson.Gson; import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; @@ -64,6 +65,7 @@ import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import tech.tablesaw.api.Table; @@ -1145,7 +1147,9 @@ private void addObsVarsToDatasetDetails(PendingImportObject pi private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) { PendingImportObject pio; PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); - String name = String.format("Observation Dataset [%s-%s]", + String datasetName = StringUtils.isNotBlank(importRow.getSubObsUnit()) ? importRow.getSubObsUnit() : importRow.getExpUnit(); + String name = String.format("%s Observation Dataset [%s-%s]", + datasetName, program.getKey(), trialPIO.getBrAPIObject() .getAdditionalInfo() @@ -1163,6 +1167,19 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr trialPIO.getId().toString()); pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); + + // TODO:BI-1464 add datasetId to datasets array + JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + List datasets = DatasetUtil.datasetsFromJson(datasetsJson); + DatasetMetadata dataset = DatasetMetadata.builder() + .name(datasetName) + .id(id) + .level(StringUtils.isNotBlank(importRow.getSubObsUnit()) ? DatasetLevel.SUB_OBS_UNIT : DatasetLevel.EXP_UNIT) + .build(); + datasets.add(dataset); + datasetsJson = DatasetUtil.jsonArrayFromDatasets(datasets); + trialPIO.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, datasetsJson); + if (ImportObjectState.EXISTING == trialPIO.getState()) { trialPIO.setState(ImportObjectState.MUTATED); } @@ -1580,6 +1597,7 @@ private Map> initializeObsVarDatas Optional> trialPIO = getTrialPIO(experimentImportRows); + // TODO: BI-1464 if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { String datasetId = trialPIO.get().getBrAPIObject() .getAdditionalInfo() diff --git a/src/main/java/org/breedinginsight/model/Dataset.java b/src/main/java/org/breedinginsight/model/Dataset.java index 638c946f1..096eae2db 100644 --- a/src/main/java/org/breedinginsight/model/Dataset.java +++ b/src/main/java/org/breedinginsight/model/Dataset.java @@ -1,3 +1,20 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.breedinginsight.model; import com.fasterxml.jackson.annotation.JsonInclude; diff --git a/src/main/java/org/breedinginsight/model/DatasetLevel.java b/src/main/java/org/breedinginsight/model/DatasetLevel.java new file mode 100644 index 000000000..e505f1916 --- /dev/null +++ b/src/main/java/org/breedinginsight/model/DatasetLevel.java @@ -0,0 +1,34 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.model; + +public enum DatasetLevel { + EXP_UNIT(0), + SUB_OBS_UNIT(1); + + private final int literal; + + DatasetLevel(int literal) { + this.literal = literal; + } + + public int getLiteral() { + return literal; + } + +} diff --git a/src/main/java/org/breedinginsight/model/DatasetMetadata.java b/src/main/java/org/breedinginsight/model/DatasetMetadata.java new file mode 100644 index 000000000..6bfa51f2a --- /dev/null +++ b/src/main/java/org/breedinginsight/model/DatasetMetadata.java @@ -0,0 +1,38 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.model; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; +import java.util.UUID; + +@Getter +@Setter +@Accessors(chain=true) +@ToString +@SuperBuilder +@NoArgsConstructor +public class DatasetMetadata { + private String name; + private UUID id; + private DatasetLevel level; +} diff --git a/src/main/java/org/breedinginsight/utilities/DatasetUtil.java b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java new file mode 100644 index 000000000..8af258f2d --- /dev/null +++ b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java @@ -0,0 +1,28 @@ +package org.breedinginsight.utilities; + +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.reflect.TypeToken; +import org.brapi.client.v2.JSON; +import org.breedinginsight.model.DatasetMetadata; + +import java.lang.reflect.Type; +import java.util.List; + +public class DatasetUtil { + + public static Type listDatasetType = new TypeToken>() {}.getType(); + public static Gson gson = new JSON().getGson(); + + public static List datasetsFromJson(JsonArray datasetsJsonArray) { + if (datasetsJsonArray != null) { + return gson.fromJson(datasetsJsonArray, listDatasetType); + } + + return List.of(); + } + + public static JsonArray jsonArrayFromDatasets(List datasets) { + return gson.toJsonTree(datasets, listDatasetType).getAsJsonArray(); + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/DatasetsIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/DatasetsIntegrationTest.java new file mode 100644 index 000000000..2f3f7b381 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/DatasetsIntegrationTest.java @@ -0,0 +1,113 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import io.kowalski.fannypack.FannyPack; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.client.exceptions.HttpClientResponseException; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; + +import javax.inject.Inject; + +import java.util.List; + +import static io.micronaut.http.HttpRequest.GET; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class DatasetsIntegrationTest extends BrAPITest { + + @Inject + @Client("/${micronaut.bi.api.version}") + RxHttpClient client; + + private String nonExistingId = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"; + + @Inject + private DSLContext dsl; + + @Inject + private ProgramDAO programDAO; + @Inject + private UserDAO userDAO; + + private User testUser; + private Program program; + + + @BeforeAll + public void setup() { + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + dsl.execute(securityFp.get("InsertPrograms")); + + List programs = programDAO.getAll(); + program = programs.get(0); + + // Insert system roles + testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId().toString()); + } + + @Test + public void getDatasetsInvalidProgram() { + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/experiments/%s/datasets", nonExistingId, nonExistingId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + public void getDatasetsInvalidExperiment() { + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/experiments/%s/datasets", program.getId().toString(), nonExistingId)) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = call.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + + +} From d0d787e62e82903f9ee66ea81335e04ed78cbe3f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 26 Feb 2024 11:19:04 -0500 Subject: [PATCH 2/4] Add new entry to datasets property --- .../importer/services/processors/ExperimentProcessor.java | 5 +++++ src/main/java/org/breedinginsight/utilities/DatasetUtil.java | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 233a77725..be64f3a6a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1170,12 +1170,17 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr // TODO:BI-1464 add datasetId to datasets array JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); + // If datasets property does not yet exist, create it + + List datasets = DatasetUtil.datasetsFromJson(datasetsJson); DatasetMetadata dataset = DatasetMetadata.builder() .name(datasetName) .id(id) .level(StringUtils.isNotBlank(importRow.getSubObsUnit()) ? DatasetLevel.SUB_OBS_UNIT : DatasetLevel.EXP_UNIT) .build(); + + log.debug(dataset.getName()); datasets.add(dataset); datasetsJson = DatasetUtil.jsonArrayFromDatasets(datasets); trialPIO.getBrAPIObject().getAdditionalInfo().add(BrAPIAdditionalInfoFields.DATASETS, datasetsJson); diff --git a/src/main/java/org/breedinginsight/utilities/DatasetUtil.java b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java index 8af258f2d..abbcf224b 100644 --- a/src/main/java/org/breedinginsight/utilities/DatasetUtil.java +++ b/src/main/java/org/breedinginsight/utilities/DatasetUtil.java @@ -7,6 +7,7 @@ import org.breedinginsight.model.DatasetMetadata; import java.lang.reflect.Type; +import java.util.ArrayList; import java.util.List; public class DatasetUtil { @@ -19,7 +20,7 @@ public static List datasetsFromJson(JsonArray datasetsJsonArray return gson.fromJson(datasetsJsonArray, listDatasetType); } - return List.of(); + return new ArrayList<>(); } public static JsonArray jsonArrayFromDatasets(List datasets) { From 9f5a8f4cf775becfd9a29890ea3eac050132f833 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 29 Feb 2024 17:46:18 -0500 Subject: [PATCH 3/4] Some work on sub observations --- .../ExperimentObservation.java | 71 ++++++++++++++++-- .../processors/ExperimentProcessor.java | 75 ++++++++++++++++++- .../breedinginsight/model/DatasetLevel.java | 10 +-- src/main/resources/application.yml | 2 + 4 files changed, 145 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 579c34a31..3673a4290 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -33,10 +33,7 @@ import org.breedinginsight.brapps.importer.model.config.*; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.model.BrAPIConstants; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.model.User; +import org.breedinginsight.model.*; import org.breedinginsight.utilities.Utilities; import java.math.BigInteger; @@ -284,8 +281,9 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); - level.setLevelName("plot"); //BreedBase only accepts "plot" or "plant" + level.setLevelName(getExpUnit()); level.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); + level.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); position.setObservationLevel(level); observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getExpUnit()); @@ -368,6 +366,69 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( return observationUnit; } + /** + * Constructs a BrAPI sub-observation unit based on the given parameters. Only a subset of the information is + * included as the exp unit level contains all the relevant top-level details and is linked from this unit + * + * Uses only the following methods from ExperimentObservation: + * - getSubObsUnit + * - getSubUnitId + * + * Existing ExpUnit data is from passed in BrAPI object which is looked up based on ObsUnitID, no other values + * in import file are required for creating a sub observation unit dataset + */ + public BrAPIObservationUnit constructBrAPISubObservationUnit( + Program program, + String seqVal, + boolean commit, + BrAPIObservationUnit expUnit, + String referenceSource, + UUID datasetId, + UUID id + ) { + + BrAPIObservationUnit observationUnit = new BrAPIObservationUnit(); + if (commit) { + observationUnit.setObservationUnitName(Utilities.appendProgramKey(getSubObsUnit(), program.getKey(), seqVal)); + observationUnit.setExternalReferences(getObsUnitExternalReferences(program, referenceSource, UUID.fromString(expUnit.getTrialDbId()), + datasetId, UUID.fromString(expUnit.getStudyDbId()), id)); + } else { + observationUnit.setObservationUnitName(getSubObsUnit()); + } + observationUnit.setStudyName(expUnit.getStudyName()); + + observationUnit.setGermplasmName(expUnit.getGermplasmName()); + String gid = expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString(); + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, gid); + + BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); + + // observationLevel entry for Sub-Obs Unit + BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); + level.setLevelName(getSubObsUnit()); + level.setLevelCode(Utilities.appendProgramKey(getSubUnitId(), program.getKey(), seqVal)); + level.setLevelOrder(DatasetLevel.SUB_OBS_UNIT.getValue()); + position.setObservationLevel(level); + + // keep this in case we decide to rename levels in future + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL, getSubObsUnit()); + + // observationLevelRelationships for top-level Exp Unit linking + List levelRelationships = new ArrayList<>(); + BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); + + // set name without added keys + expUnitLevel.setLevelName(expUnit.getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + expUnitLevel.setLevelCode(Utilities.appendProgramKey(getExpUnitId(), program.getKey(), seqVal)); + expUnitLevel.setLevelOrder(DatasetLevel.EXP_UNIT.getValue()); + levelRelationships.add(expUnitLevel); + + // TODO: Do replicate and block matter for field book? + + observationUnit.setObservationUnitPosition(position); + return observationUnit; + } + public BrAPIObservation constructBrAPIObservation( String value, String variableName, diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index be64f3a6a..a158da09f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -129,6 +129,66 @@ public class ExperimentProcessor implements Processor { // initialized by getExistingBrapiData() ) private Map> observationUnitByNameNoScope = null; + // for use when creating sub obs units + private Map> obsUnitByExpObsUnitId = null; + private Map> subObsUnitBySubObsUnitId = null; + + /** + * Operational modes of experiment processor dependent on import file contents + * + * CREATE_NEW_EXPERIMENT: + * ENTRY: + * - WHEN no ExpUnit ObsUnitIDs column present in file + * AND Exp Title file column contains new title + * CONDITION: + * - All data validations pass + * ACTION: + * - Create BrAPI objects Trial, Study, ObservationUnit, Optional: [Location, Observation, List] + * APPEND_EXPERIMENT_ENV: (Is creating obs units also required?) + * ENTRY: + * - WHEN no ExpUnit ObsUnitIDs column present in file + * AND Exp Title file column contains existing title + * AND Env file column contains new env + * CONDITION: + * - All data validations pass + * ACTION: + * - Create BrAPI objects Study, ObservationUnit, Optional: [Location, Observation, List] + * APPEND_EXPERIMENT_OBS: (this might be combined with next mode) + * ENTRY: + * - WHEN ExpUnit ObsUnitIds column present in file + * AND Sub-ObsUnit ObsUnitIds column not present in file + * AND Sub-ObsUnit column contains only empty values + * AND one or more phenotype columns are present in file + * ACTION: + * - Create BrAPI objects Study, ObservationUnit, Optional: [Location, Observation, List] + * UPDATE_EXPERIMENT_OBS: + * - WHEN ExpUnit ObsUnitIds column present in file + * AND Sub-ObsUnit ObsUnitIds column not present in file + * AND Sub-ObsUnit column contains only empty values + * AND one or more phenotype columns are present in file + * AND phenotype values for observation units already exist + * + * UPDATE_EXP_UNIT_DATA: + * - WHEN ExpUnit ObsUnitIds column present in file + * AND Sub-ObsUnit ObsUnitIds column not present in file + * AND Sub-ObsUnit column contains only empty values + * CREATE_SUB_OBS_UNIT_DATA: + * - WHEN ExpUnit ObsUnitIds column present in file + * AND Sub-ObsUnit ObsUnitIds column not present in file + * AND Sub-ObsUnit column rows contains values + * UPDATE_SUB_OBS_UNIT_DATA: + * - WHEN ExpUnit ObsUnitIds column present in file + * AND Sub-ObsUnit ObsUnitIds column present in file + */ + private enum Mode { + CREATE_EXP_UNIT_DATA, + UPDATE_EXP_UNIT_DATA, + CREATE_SUB_OBS_UNIT_DATA, + UPDATE_SUB_OBS_UNIT_DATA + } + + private Mode mode; + private final Map> observationByHash = new HashMap<>(); private Map existingObsByObsHash = new HashMap<>(); // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method @@ -1007,6 +1067,7 @@ private PendingImportObject getGidPOI(ExperimentObservation impo private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException { PendingImportObject pio; + // TODO: Should be based on ObsUnitID String key = createObservationUnitKey(importRow); if (this.observationUnitByNameNoScope.containsKey(key)) { pio = observationUnitByNameNoScope.get(key); @@ -1021,6 +1082,7 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra UUID trialID = trialPIO.getId(); UUID datasetId = null; if (commit) { + // TODO: BI-1464 get dataset id from array datasetId = UUID.fromString(trialPIO.getBrAPIObject() .getAdditionalInfo().getAsJsonObject() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); @@ -1166,13 +1228,13 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr program, trialPIO.getId().toString()); pio = new PendingImportObject(ImportObjectState.NEW, newDataset, id); + + // TODO:BI-1464 remove old id and update code to use datasets array - existing will need migration trialPIO.getBrAPIObject().putAdditionalInfoItem("observationDatasetId", id.toString()); - // TODO:BI-1464 add datasetId to datasets array JsonArray datasetsJson = trialPIO.getBrAPIObject().getAdditionalInfo().getAsJsonArray(BrAPIAdditionalInfoFields.DATASETS); // If datasets property does not yet exist, create it - List datasets = DatasetUtil.datasetsFromJson(datasetsJson); DatasetMetadata dataset = DatasetMetadata.builder() .name(datasetName) @@ -1462,6 +1524,13 @@ private Map> initializeObserva try { List existingObsUnits = brAPIObservationUnitDAO.getObservationUnitsById(rowByObsUnitId.keySet(), program); + // TODO: grab from externalReferences + /* + observationUnitByObsUnitId = existingObsUnits.stream() + .collect(Collectors.toMap(BrAPIObservationUnit::getObservationUnitDbId, + (BrAPIObservationUnit unit) -> new PendingImportObject<>(unit, false))); + */ + String refSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); if (existingObsUnits.size() == rowByObsUnitId.size()) { existingObsUnits.forEach(brAPIObservationUnit -> { @@ -1602,7 +1671,7 @@ private Map> initializeObsVarDatas Optional> trialPIO = getTrialPIO(experimentImportRows); - // TODO: BI-1464 + // TODO: BI-1464 use value in datasets array instead if (trialPIO.isPresent() && trialPIO.get().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { String datasetId = trialPIO.get().getBrAPIObject() .getAdditionalInfo() diff --git a/src/main/java/org/breedinginsight/model/DatasetLevel.java b/src/main/java/org/breedinginsight/model/DatasetLevel.java index e505f1916..691f30565 100644 --- a/src/main/java/org/breedinginsight/model/DatasetLevel.java +++ b/src/main/java/org/breedinginsight/model/DatasetLevel.java @@ -21,14 +21,14 @@ public enum DatasetLevel { EXP_UNIT(0), SUB_OBS_UNIT(1); - private final int literal; + private final int value; - DatasetLevel(int literal) { - this.literal = literal; + DatasetLevel(int value) { + this.value = value; } - public int getLiteral() { - return literal; + public int getValue() { + return value; } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index aec211b82..019a0336a 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -31,6 +31,8 @@ micronaut: mixed: true threshold: '10MB' max-request-size: '100MB' + serialization: + enum-values: true bi: api: version: v1 From 0318d225be9e492e470a7265287323d180141615 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 12 Mar 2024 13:21:12 -0400 Subject: [PATCH 4/4] Added experiment workflow --- .../processors/ExperimentProcessor.java | 64 ++------------- .../ExperimentProcessorWorkflow.java | 82 +++++++++++++++++++ .../processors/GermplasmProcessor.java | 5 ++ .../processors/LocationProcessor.java | 3 + .../processors/ObservationProcessor.java | 4 + .../processors/ObservationUnitProcessor.java | 4 + .../services/processors/Processor.java | 6 ++ .../services/processors/ProcessorManager.java | 1 + .../processors/SampleSubmissionProcessor.java | 4 + .../services/processors/StudyProcessor.java | 4 + .../services/processors/TrialProcessor.java | 3 + 11 files changed, 125 insertions(+), 55 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessorWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index a158da09f..5e6d64108 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -133,61 +133,7 @@ public class ExperimentProcessor implements Processor { private Map> obsUnitByExpObsUnitId = null; private Map> subObsUnitBySubObsUnitId = null; - /** - * Operational modes of experiment processor dependent on import file contents - * - * CREATE_NEW_EXPERIMENT: - * ENTRY: - * - WHEN no ExpUnit ObsUnitIDs column present in file - * AND Exp Title file column contains new title - * CONDITION: - * - All data validations pass - * ACTION: - * - Create BrAPI objects Trial, Study, ObservationUnit, Optional: [Location, Observation, List] - * APPEND_EXPERIMENT_ENV: (Is creating obs units also required?) - * ENTRY: - * - WHEN no ExpUnit ObsUnitIDs column present in file - * AND Exp Title file column contains existing title - * AND Env file column contains new env - * CONDITION: - * - All data validations pass - * ACTION: - * - Create BrAPI objects Study, ObservationUnit, Optional: [Location, Observation, List] - * APPEND_EXPERIMENT_OBS: (this might be combined with next mode) - * ENTRY: - * - WHEN ExpUnit ObsUnitIds column present in file - * AND Sub-ObsUnit ObsUnitIds column not present in file - * AND Sub-ObsUnit column contains only empty values - * AND one or more phenotype columns are present in file - * ACTION: - * - Create BrAPI objects Study, ObservationUnit, Optional: [Location, Observation, List] - * UPDATE_EXPERIMENT_OBS: - * - WHEN ExpUnit ObsUnitIds column present in file - * AND Sub-ObsUnit ObsUnitIds column not present in file - * AND Sub-ObsUnit column contains only empty values - * AND one or more phenotype columns are present in file - * AND phenotype values for observation units already exist - * - * UPDATE_EXP_UNIT_DATA: - * - WHEN ExpUnit ObsUnitIds column present in file - * AND Sub-ObsUnit ObsUnitIds column not present in file - * AND Sub-ObsUnit column contains only empty values - * CREATE_SUB_OBS_UNIT_DATA: - * - WHEN ExpUnit ObsUnitIds column present in file - * AND Sub-ObsUnit ObsUnitIds column not present in file - * AND Sub-ObsUnit column rows contains values - * UPDATE_SUB_OBS_UNIT_DATA: - * - WHEN ExpUnit ObsUnitIds column present in file - * AND Sub-ObsUnit ObsUnitIds column present in file - */ - private enum Mode { - CREATE_EXP_UNIT_DATA, - UPDATE_EXP_UNIT_DATA, - CREATE_SUB_OBS_UNIT_DATA, - UPDATE_SUB_OBS_UNIT_DATA - } - - private Mode mode; + private ExperimentProcessorWorkflow workflow = new ExperimentProcessorWorkflow(); private final Map> observationByHash = new HashMap<>(); private Map existingObsByObsHash = new HashMap<>(); @@ -228,6 +174,12 @@ public String getName() { return NAME; } + @Override + public void initialize(List importRows) { + // determine experiment import workflow based on file contents + workflow.determineWorkflow(importRows); + } + /** * Initialize the Map objects with existing BrAPI Data. * @@ -237,6 +189,8 @@ public String getName() { @Override public void getExistingBrapiData(List importRows, Program program) { + + List experimentImportRows = importRows.stream() .map(trialImport -> (ExperimentObservation) trialImport) .collect(Collectors.toList()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessorWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessorWorkflow.java new file mode 100644 index 000000000..c4c97e847 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessorWorkflow.java @@ -0,0 +1,82 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; + +import java.util.List; + +@Slf4j +public class ExperimentProcessorWorkflow { + + public enum Workflows { + UNSPECIFIED, + CREATE_EXP_ENV_PHENOTYPES, + APPEND_UPDATE_EXPUNIT_PHENOTYPES, + APPEND_UPDATE_SUBOBSUNIT_PHENOTYPES + } + + private Workflows workflow = Workflows.UNSPECIFIED; + + public ExperimentProcessorWorkflow() { + } + + public void determineWorkflow(List importRows) { + + boolean hasExpUnitObsUnitIDs = importRows.stream() + .anyMatch(row -> { + ExperimentObservation expRow = (ExperimentObservation) row; + return StringUtils.isNotBlank(expRow.getObsUnitID()); + }); + + if (hasExpUnitObsUnitIDs) { + long distinctCount = importRows.stream() + .map(row -> { + ExperimentObservation expRow = (ExperimentObservation) row; + return expRow.getObsUnitID(); + }) + .distinct() + .count(); + + if (distinctCount != importRows.size()) { + // If have ExpUnit ObsUnitIDs and there are duplicates -> Append / Update SubObsUnit Phenotypes + setWorkflow(Workflows.APPEND_UPDATE_SUBOBSUNIT_PHENOTYPES); + } else { + // If have ExpUnit ObsUnitIDs and all are unique -> Append / Update ExpUnit Phenotypes + setWorkflow(Workflows.APPEND_UPDATE_EXPUNIT_PHENOTYPES); + } + + } else { + // No ObsUnitIDs so creating experiment or appending env + setWorkflow(Workflows.CREATE_EXP_ENV_PHENOTYPES); + } + } + + public void setWorkflow(Workflows workflow) { + log.debug("Workflow: " + workflow); + this.workflow = workflow; + } + + public Workflows getWorkflow() { + return workflow; + } + +} \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index de5eb65ae..9e9da1698 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -111,6 +111,11 @@ public GermplasmProcessor(BrAPIGermplasmService brAPIGermplasmService, DSLContex this.brAPIGermplasmService = brAPIGermplasmService; } + @Override + public void initialize(List importRows) { + } + + @Override public void getExistingBrapiData(List importRows, Program program) throws ApiException { // Get all of our objects specified in the data file by their unique attributes diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java index 5a3d5af0a..47c55774d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/LocationProcessor.java @@ -57,6 +57,9 @@ public LocationProcessor(ProgramLocationService locationService) { this.locationService = locationService; } + @Override + public void initialize(List importRows) { + } public void getExistingBrapiData(List importRows, Program program) { List uniqueLocationNames = importRows.stream() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java index 7fcefcd3d..edc344919 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java @@ -64,6 +64,10 @@ public ObservationProcessor(BrAPIObservationVariableDAO brAPIVariableDAO, this.brAPIObservationDAO = brAPIObservationDAO; } + @Override + public void initialize(List importRows) { + } + public void getExistingBrapiData(List importRows, Program program) { // will skip existing observations, no error reported diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java index 6f1e6723a..8071ff5db 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java @@ -57,6 +57,10 @@ public ObservationUnitProcessor(BrAPIObservationUnitDAO brAPIObservationUnitDAO) this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; } + @Override + public void initialize(List importRows) { + } + public void getExistingBrapiData(List importRows, Program program) { // get unique observation unit names diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java index 820626676..fc57630d4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java @@ -31,6 +31,12 @@ public interface Processor { + /** + * Given importRows, configure any initial processor state like modes of operation + * @param importRows + */ + void initialize(List importRows); + /** * Given importRows, get all existing objects in the brapi service. First phase of processing followed by process. * @param importRows diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java index 1961b33d1..3f10bd369 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java @@ -57,6 +57,7 @@ public ImportPreviewResponse process(List importRows, List