From d008c1599890884e61bc105d1c40b2178dad95fa Mon Sep 17 00:00:00 2001 From: HMS17 Date: Fri, 12 Dec 2025 08:49:07 -0500 Subject: [PATCH 1/2] [BI-2234] - Improve exp dataset export file --- .../brapi/v2/services/BrAPITrialService.java | 76 +++++++++++++++---- .../experiment/ExperimentFileColumns.java | 15 +++- 2 files changed, 72 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 158ce952a..b38af3de2 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -36,10 +36,7 @@ import org.breedinginsight.services.TraitService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; -import org.breedinginsight.utilities.DatasetUtil; -import org.breedinginsight.utilities.IntOrderComparator; -import org.breedinginsight.utilities.FileUtil; -import org.breedinginsight.utilities.Utilities; +import org.breedinginsight.utilities.*; import org.jetbrains.annotations.NotNull; import javax.inject.Inject; @@ -154,9 +151,6 @@ public DownloadFile exportObservations( new ArrayList<>(Arrays.asList(params.getEnvironments().split(","))) : new ArrayList<>(); FileType fileType = params.getFileExtension(); - // make columns present in all exports - List columns = ExperimentFileColumns.getOrderedColumns(); - // add columns for requested dataset obsvars and timestamps log.debug(logHash + ": fetching experiment for export"); BrAPITrial experiment = getExperiment(program, experimentId); @@ -188,8 +182,29 @@ public DownloadFile exportObservations( Utilities.generateApiExceptionLogMessage(err), err); } + boolean isSubObs = isSubEntityDataset(ous); + + List columns; + + //make columns present in all exports + if (isSubObs){ + columns = ExperimentFileColumns.getOrderedColumnsSubEntity(); + } else { + columns = ExperimentFileColumns.getOrderedColumns(); + } + //add obsUnitID as dynamic column with observation level appended to header - String observationLvl = ous.get(0).getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); + if (isSubObs) { + //need to add top level obs unit ids as well + BrAPIObservationUnitLevelRelationship topLevel = ous.get(0).getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> (x.getLevelOrder() != null && x.getLevelOrder().equals(0))).findFirst().orElse(null); + if (topLevel != null) { + String topObservationLvl = StringUtils.capitalize(topLevel.getLevelName()); + columns = dynamicUpdateObsUnitIDLabel(columns, topObservationLvl); + } + } + String observationLvl = ous.get(0).getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); columns = dynamicUpdateObsUnitIDLabel(columns, observationLvl); if (params.getDatasetId() != null) { @@ -220,7 +235,8 @@ public DownloadFile exportObservations( params.isIncludeTimestamps(), obsVars, studyDbIdByOUId, - programGermplasmByDbId + programGermplasmByDbId, + isSubObs ); // make export rows for OUs without observations @@ -230,7 +246,7 @@ public DownloadFile exportObservations( // Map Observation Unit to the Study it belongs to. studyDbIdByOUId.put(ouId, ou.getStudyDbId()); if (!rowByOUId.containsKey(ouId)) { - rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId)); + rowByOUId.put(ouId, createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId, isSubObs)); } } } @@ -591,7 +607,8 @@ private void addBrAPIObsToRecords( boolean includeTimestamp, List obsVars, Map studyDbIdByOUId, - Map programGermplasmByDbId) throws ApiException, DoesNotExistException { + Map programGermplasmByDbId, + boolean isSubObs) throws ApiException, DoesNotExistException { Map varByDbId = new HashMap<>(); obsVars.forEach(var -> varByDbId.put(var.getObservationVariableDbId(), var)); for (BrAPIObservation obs: dataset) { @@ -609,7 +626,7 @@ private void addBrAPIObsToRecords( } else { // otherwise make a new row - Map row = createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId); + Map row = createExportRow(experiment, program, ou, studyByDbId, programGermplasmByDbId, isSubObs); addObsVarDataToRow(row, obs, includeTimestamp, var, program); rowByOUId.put(ouId, row); } @@ -727,7 +744,8 @@ private Map createExportRow( Program program, BrAPIObservationUnit ou, Map studyByDbId, - Map programGermplasmByDbId) throws ApiException, DoesNotExistException { + Map programGermplasmByDbId, + boolean isSubEntity) throws ApiException, DoesNotExistException { HashMap row = new HashMap<>(); // get OU id, germplasm, and study @@ -750,7 +768,6 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.TEST_CHECK, testCheck); row.put(ExperimentObservation.Columns.EXP_TITLE, Utilities.removeProgramKey(experiment.getTrialName(), program.getKey())); row.put(ExperimentObservation.Columns.EXP_DESCRIPTION, experiment.getTrialDescription()); - row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); @@ -776,7 +793,6 @@ private Map createExportRow( BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); - row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); // get replicate number Optional repLevel = ou.getObservationUnitPosition() @@ -810,6 +826,28 @@ private Map createExportRow( String observationLvl = ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString(); row.put(observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX, ouId); + if (isSubEntity) { + BrAPIObservationUnitLevelRelationship topLevel = ou.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> (x.getLevelOrder() != null && x.getLevelOrder().equals(0))).findFirst().orElse(null); + + if (topLevel != null) { + String topLvlName = StringUtils.capitalize(topLevel.getLevelName()); + row.put(ExperimentObservation.Columns.EXP_UNIT, topLvlName); + + String topLvlOuId = Utilities.removeProgramKeyAndUnknownAdditionalData(topLevel.getLevelCode(), program.getKey()); + row.put(topLvlName + " " + OBSERVATION_UNIT_ID_SUFFIX, topLvlOuId); + } + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.EXP_UNIT_ID).getAsString()); + + row.put(ExperimentObservation.Columns.SUB_OBS_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + row.put(ExperimentObservation.Columns.SUB_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + + } else { + row.put(ExperimentObservation.Columns.EXP_UNIT, ou.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.OBSERVATION_LEVEL).getAsString()); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + } + return row; } @@ -888,6 +926,10 @@ private List filterDatasetByEnvironment( .collect(Collectors.toList()); } + private boolean isSubEntityDataset(List ous){ + return (ous.get(0).getObservationUnitPosition().getObservationLevelRelationships().size() > 2); + } + private void sortDefaultForObservationUnit(List ous) { Comparator studyNameComparator = Comparator.comparing(BrAPIObservationUnit::getStudyName, new IntOrderComparator()); Comparator ouNameComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); @@ -899,7 +941,9 @@ private void sortDefaultForExportRows(@NotNull List> exportR Comparator> expUnitIdComparator = Comparator.comparing(row -> (row.get(Columns.EXP_UNIT_ID).toString()), new IntOrderComparator()); - exportRows.sort(envComparator.thenComparing(expUnitIdComparator)); + Comparator> subUnitIdComparator = Comparator.comparing(row -> (row.get(Columns.SUB_UNIT_ID).toString()), new IntOrderComparator()); + + exportRows.sort(envComparator.thenComparing(expUnitIdComparator).thenComparing(subUnitIdComparator)); } public BrAPIStudy getEnvironment(Program program, UUID envId) throws ApiException { diff --git a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java index 39f7b22c0..dd2e4789f 100644 --- a/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java +++ b/src/main/java/org/breedinginsight/services/parsers/experiment/ExperimentFileColumns.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.stream.Collectors; -import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; +import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.*; public enum ExperimentFileColumns { @@ -34,13 +34,13 @@ public enum ExperimentFileColumns { EXP_TITLE(ExperimentObservation.Columns.EXP_TITLE, Column.ColumnDataType.STRING), EXP_DESCRIPTION(ExperimentObservation.Columns.EXP_DESCRIPTION, Column.ColumnDataType.STRING), EXP_UNIT(ExperimentObservation.Columns.EXP_UNIT, Column.ColumnDataType.STRING), - //SUB_OBS_UNIT(ExperimentObservation.Columns.SUB_OBS_UNIT, Column.ColumnDataType.STRING), + SUB_OBS_UNIT(ExperimentObservation.Columns.SUB_OBS_UNIT, Column.ColumnDataType.STRING), EXP_TYPE(ExperimentObservation.Columns.EXP_TYPE, Column.ColumnDataType.STRING), ENV(ExperimentObservation.Columns.ENV, Column.ColumnDataType.STRING), ENV_LOCATION(ExperimentObservation.Columns.ENV_LOCATION, Column.ColumnDataType.STRING), ENV_YEAR(ExperimentObservation.Columns.ENV_YEAR, Column.ColumnDataType.INTEGER), EXP_UNIT_ID(ExperimentObservation.Columns.EXP_UNIT_ID, Column.ColumnDataType.STRING), - //SUB_UNIT_ID(ExperimentObservation.Columns.SUB_UNIT_ID, Column.ColumnDataType.STRING), + SUB_UNIT_ID(ExperimentObservation.Columns.SUB_UNIT_ID, Column.ColumnDataType.STRING), REP_NUM(ExperimentObservation.Columns.REP_NUM, Column.ColumnDataType.INTEGER), BLOCK_NUM(ExperimentObservation.Columns.BLOCK_NUM, Column.ColumnDataType.INTEGER), ROW(ExperimentObservation.Columns.ROW, Column.ColumnDataType.STRING), @@ -52,6 +52,7 @@ public enum ExperimentFileColumns { TREATMENT_FACTORS(ExperimentObservation.Columns.TREATMENT_FACTORS, Column.ColumnDataType.STRING); private final Column column; + private static List subEntityOnlyColumns = Arrays.asList(SUB_OBS_UNIT, SUB_UNIT_ID); ExperimentFileColumns(String value, Column.ColumnDataType dataType) { this.column = new Column(value, dataType); @@ -63,6 +64,14 @@ public String toString() { } public static List getOrderedColumns() { + //Don't include subentity columns + return Arrays.stream(ExperimentFileColumns.values()) + .filter(val -> !(subEntityOnlyColumns.contains(val))) + .map(value -> value.column) + .collect(Collectors.toList()); + } + + public static List getOrderedColumnsSubEntity() { return Arrays.stream(ExperimentFileColumns.values()) .map(value -> value.column) .collect(Collectors.toList()); From 104127bf6b57a30f84e2d60fa0e86e13942eaa2a Mon Sep 17 00:00:00 2001 From: HMS17 Date: Mon, 15 Dec 2025 10:14:43 -0500 Subject: [PATCH 2/2] [BI-2234] - Code review helper method --- .../brapi/v2/services/BrAPITrialService.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index b38af3de2..9ca961920 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -135,6 +135,13 @@ private long countGermplasm(UUID programId, String trialDbId) throws ApiExceptio return obUnits.stream().map(BrAPIObservationUnit::getGermplasmDbId).distinct().count(); } + private BrAPIObservationUnitLevelRelationship getTopLevel(BrAPIObservationUnit ou) { + BrAPIObservationUnitLevelRelationship topLevel = ou.getObservationUnitPosition() + .getObservationLevelRelationships().stream() + .filter(x -> (x.getLevelOrder() != null && x.getLevelOrder().equals(0))).findFirst().orElse(null); + return topLevel; + } + public DownloadFile exportObservations( Program program, UUID experimentId, @@ -196,9 +203,7 @@ public DownloadFile exportObservations( //add obsUnitID as dynamic column with observation level appended to header if (isSubObs) { //need to add top level obs unit ids as well - BrAPIObservationUnitLevelRelationship topLevel = ous.get(0).getObservationUnitPosition() - .getObservationLevelRelationships().stream() - .filter(x -> (x.getLevelOrder() != null && x.getLevelOrder().equals(0))).findFirst().orElse(null); + BrAPIObservationUnitLevelRelationship topLevel = getTopLevel(ous.get(0)); if (topLevel != null) { String topObservationLvl = StringUtils.capitalize(topLevel.getLevelName()); columns = dynamicUpdateObsUnitIDLabel(columns, topObservationLvl); @@ -827,9 +832,7 @@ private Map createExportRow( row.put(observationLvl + " " + OBSERVATION_UNIT_ID_SUFFIX, ouId); if (isSubEntity) { - BrAPIObservationUnitLevelRelationship topLevel = ou.getObservationUnitPosition() - .getObservationLevelRelationships().stream() - .filter(x -> (x.getLevelOrder() != null && x.getLevelOrder().equals(0))).findFirst().orElse(null); + BrAPIObservationUnitLevelRelationship topLevel = getTopLevel(ou); if (topLevel != null) { String topLvlName = StringUtils.capitalize(topLevel.getLevelName());