From c18b75ad2ec60fef9ac7510de0059885cb818702 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 6 May 2024 15:33:00 +0000 Subject: [PATCH 01/55] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 4b9b4d2ac..c580c09a5 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+729 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2be3bf139db0bc693f8d51d541383cfa8e2b506f +version=v0.10.0+731 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/480c3b7791584f425970fb7e00950c11847ae0eb From e26da32b3636a13d98fec750c041d517dbce6b88 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 6 May 2024 11:48:13 -0400 Subject: [PATCH 02/55] Update version.properties --- src/main/resources/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index c580c09a5..02f8f5ace 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+731 +version=v0.9.1+731 versionInfo=https://github.com/Breeding-Insight/bi-api/commit/480c3b7791584f425970fb7e00950c11847ae0eb From 8a95cda598da5c2b179075a175edb62db4978a9f Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 6 May 2024 15:48:26 +0000 Subject: [PATCH 03/55] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 02f8f5ace..694f986e0 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.1+731 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/480c3b7791584f425970fb7e00950c11847ae0eb +version=v0.9.1+733 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/e26da32b3636a13d98fec750c041d517dbce6b88 From 86922f289bb6f8ef74768a1d2607259e0b0e351e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 8 May 2024 18:36:09 +0000 Subject: [PATCH 04/55] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 694f986e0..93a4ec11d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.1+733 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/e26da32b3636a13d98fec750c041d517dbce6b88 +version=v0.9.1+739 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ed1fd023abd6980fb4af05e801cfafb11a2b381e From e7e46b70afc2f6a6b0acdee36ad8e0bc4e01299f Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 13 May 2024 15:12:37 -0400 Subject: [PATCH 05/55] [BI-2127] return Lat, Long, Elevation, and RTK --- .../brapi/v2/services/BrAPITrialService.java | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index caab88879..e669e46df 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,5 +1,9 @@ package org.breedinginsight.brapi.v2.services; +import com.github.filosganga.geogson.model.Coordinates; +import com.github.filosganga.geogson.model.Point; +import com.github.filosganga.geogson.model.positions.SinglePosition; +import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; import io.micronaut.http.MediaType; import io.micronaut.http.server.exceptions.InternalServerException; @@ -465,10 +469,19 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); + + Coordinates coordinates = extractCoordinates(ou); + row.put( ExperimentObservation.Columns.LAT, coordinates==null? null : String.valueOf(coordinates.getLat()) ); + row.put( ExperimentObservation.Columns.LONG, coordinates==null? null : String.valueOf(coordinates.getLon()) ); + row.put( ExperimentObservation.Columns.ELEVATION, coordinates==null? null : String.valueOf(coordinates.getAlt()) ); + BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + JsonObject additionalInfo = ou.getAdditionalInfo(); + row.put(ExperimentObservation.Columns.RTK, additionalInfo.get(BrAPIAdditionalInfoFields.RTK).getAsString()); + // get replicate number Optional repLevel = ou.getObservationUnitPosition() .getObservationLevelRelationships().stream() @@ -484,9 +497,12 @@ private Map createExportRow( .findFirst(); blockLevel.ifPresent(brAPIObservationUnitLevelRelationship -> row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); - if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null && - ou.getObservationUnitPosition().getPositionCoordinateY() != null) { + if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null + ) { row.put(ExperimentObservation.Columns.ROW, ou.getObservationUnitPosition().getPositionCoordinateX()); + } + if (ou.getObservationUnitPosition() != null && + ou.getObservationUnitPosition().getPositionCoordinateY() != null) { row.put(ExperimentObservation.Columns.COLUMN, ou.getObservationUnitPosition().getPositionCoordinateY()); } if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { @@ -499,7 +515,22 @@ private Map createExportRow( return row; } - + private Coordinates extractCoordinates(BrAPIObservationUnit ou){ + Coordinates coordinates = null; + if ( ou.getObservationUnitPosition()!=null + && ou.getObservationUnitPosition().getGeoCoordinates()!=null + && ou.getObservationUnitPosition().getGeoCoordinates().getGeometry()!=null + && ou.getObservationUnitPosition().getGeoCoordinates().getGeometry().positions()!=null + ) + { + Object o = ou.getObservationUnitPosition().getGeoCoordinates().getGeometry().positions(); + if (o instanceof SinglePosition){ + SinglePosition sp = (SinglePosition)o; + coordinates= sp.coordinates(); + } + } + return coordinates; + } private void addObsVarColumns( List columns, From 1a909b6131f6de47b60e177da515e4336d33b904 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 13 May 2024 16:03:10 -0400 Subject: [PATCH 06/55] [BI-2127] protect against nulls --- .../brapi/v2/services/BrAPITrialService.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index e669e46df..381514182 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,7 +1,6 @@ package org.breedinginsight.brapi.v2.services; import com.github.filosganga.geogson.model.Coordinates; -import com.github.filosganga.geogson.model.Point; import com.github.filosganga.geogson.model.positions.SinglePosition; import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; @@ -470,18 +469,23 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); + // Lat, Long, Elevation Coordinates coordinates = extractCoordinates(ou); row.put( ExperimentObservation.Columns.LAT, coordinates==null? null : String.valueOf(coordinates.getLat()) ); row.put( ExperimentObservation.Columns.LONG, coordinates==null? null : String.valueOf(coordinates.getLon()) ); row.put( ExperimentObservation.Columns.ELEVATION, coordinates==null? null : String.valueOf(coordinates.getAlt()) ); + // RTK + JsonObject additionalInfo = ou.getAdditionalInfo(); + String rtk = ( additionalInfo==null || additionalInfo.get(BrAPIAdditionalInfoFields.RTK) ==null ) + ? null + : additionalInfo.get(BrAPIAdditionalInfoFields.RTK).getAsString(); + row.put(ExperimentObservation.Columns.RTK, rtk); + BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - JsonObject additionalInfo = ou.getAdditionalInfo(); - row.put(ExperimentObservation.Columns.RTK, additionalInfo.get(BrAPIAdditionalInfoFields.RTK).getAsString()); - // get replicate number Optional repLevel = ou.getObservationUnitPosition() .getObservationLevelRelationships().stream() @@ -497,14 +501,13 @@ private Map createExportRow( .findFirst(); blockLevel.ifPresent(brAPIObservationUnitLevelRelationship -> row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); - if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null - ) { + + //Row and Column + if ( ou.getObservationUnitPosition() != null ) { row.put(ExperimentObservation.Columns.ROW, ou.getObservationUnitPosition().getPositionCoordinateX()); - } - if (ou.getObservationUnitPosition() != null && - ou.getObservationUnitPosition().getPositionCoordinateY() != null) { row.put(ExperimentObservation.Columns.COLUMN, ou.getObservationUnitPosition().getPositionCoordinateY()); } + if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor()); } else { From 5a2100cbe6272636597fe67e7e175b8bf342113d Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 14 May 2024 10:24:28 -0400 Subject: [PATCH 07/55] [BI-2127] protect against a double beign NaN (not yet set) before converting to a string --- .../brapi/v2/services/BrAPITrialService.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 381514182..7c18824e7 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -471,9 +471,9 @@ private Map createExportRow( // Lat, Long, Elevation Coordinates coordinates = extractCoordinates(ou); - row.put( ExperimentObservation.Columns.LAT, coordinates==null? null : String.valueOf(coordinates.getLat()) ); - row.put( ExperimentObservation.Columns.LONG, coordinates==null? null : String.valueOf(coordinates.getLon()) ); - row.put( ExperimentObservation.Columns.ELEVATION, coordinates==null? null : String.valueOf(coordinates.getAlt()) ); + row.put( ExperimentObservation.Columns.LAT, coordinates==null? null : doubleToString(coordinates.getLat()) ); + row.put( ExperimentObservation.Columns.LONG, coordinates==null? null : doubleToString(coordinates.getLon()) ); + row.put( ExperimentObservation.Columns.ELEVATION, coordinates==null? null : doubleToString(coordinates.getAlt()) ); // RTK JsonObject additionalInfo = ou.getAdditionalInfo(); @@ -518,6 +518,9 @@ private Map createExportRow( return row; } + private String doubleToString(double val){ + return Double.isNaN(val) ? null : String.valueOf( val ); + } private Coordinates extractCoordinates(BrAPIObservationUnit ou){ Coordinates coordinates = null; if ( ou.getObservationUnitPosition()!=null From a52eebd51c245227883849524d9809a2be0b96ad Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 15 May 2024 12:04:45 -0400 Subject: [PATCH 08/55] [BI-2128] implemented fix and integration tests --- .../processors/ExperimentProcessor.java | 3 +- .../importer/ExperimentFileImportTest.java | 96 +++++++++++++++++-- 2 files changed, 89 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f27d89338..95e196c5b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1244,7 +1244,8 @@ private void fetchOrCreateObservationPIO(Program program, } if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(key, value, column, rowNum)){ + // Update observation value only if it is changed and new value is not blank. + if (!isObservationMatched(key, value, column, rowNum) && StringUtils.isNotBlank(value)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 6bc203d64..09953b2f7 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -85,7 +85,6 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentFileImportTest extends BrAPITest { - private static final String OVERWRITE = "overwrite"; private FannyPack securityFp; private String mappingId; @@ -822,6 +821,75 @@ public void importNewObservationDataByObsUnitId(boolean commit) { } } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { + List traits = importTestUtils.createTraits(1); + Program program = createProgram("Overwrite Attempt With Blank Obs"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), "1"); // Valid observation value. + + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + + // Fetch the ObsUnitId to use in the overwrite upload. + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + assertRowSaved(newExp, program, traits); + + Map newObsVar = new HashMap<>(); + newObsVar.put(Columns.GERMPLASM_GID, "1"); + newObsVar.put(Columns.TEST_CHECK, "T"); + newObsVar.put(Columns.EXP_TITLE, "Test Exp"); + newObsVar.put(Columns.EXP_UNIT, "Plot"); + newObsVar.put(Columns.EXP_TYPE, "Phenotyping"); + newObsVar.put(Columns.ENV, "New Env"); + newObsVar.put(Columns.ENV_LOCATION, "Location A"); + newObsVar.put(Columns.ENV_YEAR, "2023"); + newObsVar.put(Columns.EXP_UNIT_ID, "a-1"); + newObsVar.put(Columns.REP_NUM, "1"); + newObsVar.put(Columns.BLOCK_NUM, "1"); + newObsVar.put(Columns.ROW, "1"); + newObsVar.put(Columns.COLUMN, "1"); + newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); // Indicates this is an overwrite. + newObsVar.put(traits.get(0).getObservationVariableName(), ""); // Empty string should be no op. + + Map requestBody = new HashMap<>(); + requestBody.put("overwrite", "true"); + requestBody.put("overwriteReason", "testing"); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), requestBody, commit, client, program, mappingId); + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + // Verify that the overwrite attempt with blank observation value did not overwrite the original value. + assertRowSaved(newExp, program, traits); + if(commit) { + assertRowSaved(newExp, program, traits); + } else { + assertValidPreviewRow(newExp, row, program, traits); + } + } @ParameterizedTest @ValueSource(booleans = {true, false}) @@ -1086,15 +1154,15 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { /* Scenario: - - an experiment was created with observations - - do a second upload with additional observations for the experiment. Make sure the original observations are blank values - - verify the second set of observations get uploaded successfully + - Create an experiment with valid observations. + - Upload a second file with (1) a blank observation, (2) a changed valid observation, and (3) a new observation for the experiment. + - Verify that (1) the blank observation makes no change, (2) the changed observation is overwritten, and (3) new observations are appended to the experiment. */ @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { - List traits = importTestUtils.createTraits(2); + List traits = importTestUtils.createTraits(3); Program program = createProgram("Exp with additional Uploads (blank) "+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); @@ -1110,7 +1178,9 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(Columns.BLOCK_NUM, "1"); newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - newExp.put(traits.get(0).getObservationVariableName(), "1"); + String originalValue = "1"; // Convenience variable, this value is reused. + newExp.put(traits.get(0).getObservationVariableName(), originalValue); + newExp.put(traits.get(1).getObservationVariableName(), "2"); importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); @@ -1140,10 +1210,15 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); - newObservation.put(traits.get(0).getObservationVariableName(), ""); - newObservation.put(traits.get(1).getObservationVariableName(), "2"); + newObservation.put(traits.get(0).getObservationVariableName(), ""); // This blank value should not overwrite. + newObservation.put(traits.get(1).getObservationVariableName(), "3"); // This valid value should overwrite. + newObservation.put(traits.get(2).getObservationVariableName(), "4"); // This valid new observation should be appended. - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + Map requestBody = new HashMap<>(); + requestBody.put("overwrite", "true"); + requestBody.put("overwriteReason", "testing"); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), requestBody, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -1154,6 +1229,9 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + // The blank value should not have produced an overwrite. + // This change is to make the "expected" value correct. + newObservation.put(traits.get(0).getObservationVariableName(), originalValue); if(commit) { assertRowSaved(newObservation, program, traits); } else { From c3af14fe0e56d8f42d86667ec06708f254099c60 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 15 May 2024 12:14:10 -0400 Subject: [PATCH 09/55] [BI-2128] added test description --- .../brapps/importer/ExperimentFileImportTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 09953b2f7..2d884bac8 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -821,6 +821,12 @@ public void importNewObservationDataByObsUnitId(boolean commit) { } } + /* + Scenario: + - an experiment was created with observations + - an overwrite operation is attempted with blank observation values + - verify blank obervation values do not overwrite original values + */ @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows From fa8c7cb586f4bf4110a505e10e090fecee0cd714 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 15 May 2024 13:11:44 -0400 Subject: [PATCH 10/55] [BI-2128] fixed typo --- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 2d884bac8..48bffd4c2 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -825,7 +825,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { Scenario: - an experiment was created with observations - an overwrite operation is attempted with blank observation values - - verify blank obervation values do not overwrite original values + - verify blank observation values do not overwrite original values */ @ParameterizedTest @ValueSource(booleans = {true, false}) From aa138b6978a02f8922e17bce5b74efcca8dd2fb2 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 16 May 2024 13:45:17 +0000 Subject: [PATCH 11/55] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 93a4ec11d..111d06bd2 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.1+739 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ed1fd023abd6980fb4af05e801cfafb11a2b381e +version=v0.9.1+741 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fa8c7cb586f4bf4110a505e10e090fecee0cd714 From c67e67aac2b91c6ff95748b930c0979faf468c1e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 16 May 2024 13:47:38 +0000 Subject: [PATCH 12/55] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 50c54602e..f1fe6063d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+737 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/72bf86b660d7fcb7b0b0d3479a8fdc72f64e1a5f +version=v0.10.0+743 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/626d73785f28db9b297e0e260a635e76786a7868 From 2e41d4c75b49e7988a660d51495e21a02cd5fd81 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 17 May 2024 16:26:40 -0400 Subject: [PATCH 13/55] [BI-2127] made code more readable --- .../brapi/v2/services/BrAPITrialService.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 7c18824e7..dd9bde018 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -471,9 +471,15 @@ private Map createExportRow( // Lat, Long, Elevation Coordinates coordinates = extractCoordinates(ou); - row.put( ExperimentObservation.Columns.LAT, coordinates==null? null : doubleToString(coordinates.getLat()) ); - row.put( ExperimentObservation.Columns.LONG, coordinates==null? null : doubleToString(coordinates.getLon()) ); - row.put( ExperimentObservation.Columns.ELEVATION, coordinates==null? null : doubleToString(coordinates.getAlt()) ); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getLat())) + .ifPresent(lat -> row.put(ExperimentObservation.Columns.LAT, lat)); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getLon())) + .ifPresent(lon -> row.put(ExperimentObservation.Columns.LONG, lon)); + Optional.ofNullable(coordinates) + .map(c -> doubleToString(c.getAlt())) + .ifPresent(elevation -> row.put(ExperimentObservation.Columns.ELEVATION, elevation)); // RTK JsonObject additionalInfo = ou.getAdditionalInfo(); From 453cf79701e6a6094a4931091470426f5fa81bff Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 22 May 2024 16:03:38 +0000 Subject: [PATCH 14/55] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index f1fe6063d..b8a9bf8f8 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+743 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/626d73785f28db9b297e0e260a635e76786a7868 +version=v0.10.0+745 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6662651d581bb80396250f3ff38bc9443c4a4744 From 5564ca1e97a7509d0ef6edc353462471ee61ee47 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 18:17:05 -0400 Subject: [PATCH 15/55] create micronaut composite pattern classes for experiment import workflow --- .../model/imports/ImportServiceContext.java | 1 + .../ExperimentImportService.java | 27 ++++----- .../importer/model/workflow/Action.java | 29 +++++++++ .../model/workflow/ImportWorkflow.java | 12 ++++ .../model/workflow/ImportWorkflowResult.java | 33 +++++++++++ .../importer/services/FileImportService.java | 10 ++-- .../experiment/ExperimentImportWorkflow.java | 59 +++++++++++++++++++ .../AppendOverwriteObservationWorkflow.java | 53 +++++++++++++++++ .../workflow/NewExperimentWorkflow.java | 54 +++++++++++++++++ .../workflow/NewEnvironmentWorkflow.java | 53 +++++++++++++++++ 10 files changed, 309 insertions(+), 22 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index 45393f67c..c5a479e90 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -32,6 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { + private String action; private Workflow workflow; private List brAPIImports; private Table data; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index c6f68b251..de7d54961 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -23,9 +23,11 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import tech.tablesaw.api.Table; @@ -34,6 +36,7 @@ import javax.inject.Provider; import javax.inject.Singleton; import java.util.List; +import java.util.Optional; @Singleton @Slf4j @@ -43,12 +46,16 @@ public class ExperimentImportService implements BrAPIImportService { private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; + private final ExperimentImportWorkflow workflow; @Inject - public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider) + public ExperimentImportService(Provider experimentProcessorProvider, + Provider processorManagerProvider, + ExperimentImportWorkflow workflow) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; + this.workflow = workflow; } @Override @@ -70,23 +77,11 @@ public String getMissingColumnMsg(String columnName) { public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - ImportPreviewResponse response = null; - List processors = List.of(experimentProcessorProvider.get()); - - if (context.getWorkflow() != null) { - log.info("Workflow: " + context.getWorkflow().getName()); + if (!context.getAction().isEmpty()) { + log.info("Workflow: " + context.getAction()); } - // TODO: change to calling workflow process instead of processor manager - response = processorManagerProvider.get().process(context.getBrAPIImports(), - processors, - context.getData(), - context.getProgram(), - context.getUpload(), - context.getUser(), - context.isCommit()); - return response; - + return workflow.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java new file mode 100644 index 000000000..65c3d0cbb --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java @@ -0,0 +1,29 @@ +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; + +@Getter +@Setter +@ToString +@AllArgsConstructor +public class Action { + private String name; + private int order; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java new file mode 100644 index 000000000..c5dfc1957 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java @@ -0,0 +1,12 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; + +import java.util.Optional; + +@FunctionalInterface +public interface ImportWorkflow extends Ordered { + Optional process(ImportServiceContext context); +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java new file mode 100644 index 000000000..2e61a6d46 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -0,0 +1,33 @@ +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ + +package org.breedinginsight.brapps.importer.model.workflow; + +import lombok.*; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; + +import java.util.Optional; + +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +public class ImportWorkflowResult { + private Action action; + private Optional importPreviewResponse; +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index ef6c5f884..59438c696 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -435,14 +435,12 @@ private void processFile(UUID workflowId, List finalBrAPIImportList // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - Workflow workflow = null; - if (workflowId != null) { - Optional optionalWorkflow = workflowFactory.getWorkflow(workflowId); - workflow = optionalWorkflow.orElse(null); - } + + final Workflow[] workflow = {null}; + Optional.ofNullable(workflowId).ifPresent(id -> workflow[0] = workflowFactory.getWorkflow(id).orElse(null)); ImportServiceContext context = ImportServiceContext.builder() - .workflow(workflow) + .workflow(workflow[0]) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java new file mode 100644 index 000000000..e8a7d45da --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java @@ -0,0 +1,59 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment; + +import io.micronaut.context.annotation.Primary; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; + +import javax.inject.Singleton; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +@Primary +@Singleton +public class ExperimentImportWorkflow implements ImportWorkflow { + private final List workflows; + + public ExperimentImportWorkflow(List workflows) { + this.workflows = workflows; + } + + @Override + public Optional process(ImportServiceContext context) { + return workflows.stream() + .map(workflow->workflow.process(context)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + } + public List getWorkflows() { + return workflows.stream() + .map(workflow->workflow.process(null)) + .filter(Optional::isPresent) + .map(Optional::get) + .map(result->result.getAction()) + .collect(Collectors.toList()); + } + + public enum ImportAction { + APPEND_OVERWRITE("append-overwrite-observation"), + NEW_OBSERVATION("new-observation"), + APPEND_ENVIRONMENT("append-environment"); + + private String urlFragment; + + ImportAction(String urlFragment) { + this.urlFragment = urlFragment; + } + + public String getUrlFragment() { + return urlFragment; + } + + public boolean isEqual(String value) { + return Optional.ofNullable(urlFragment.equals(value)).orElse(false); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java new file mode 100644 index 000000000..ca5e932ef --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java @@ -0,0 +1,53 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; + +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; + +import javax.inject.Singleton; +import java.util.Optional; + +@Singleton +public class AppendOverwriteObservationWorkflow implements ImportWorkflow { + private final ExperimentImportWorkflow.ImportAction action; + + public AppendOverwriteObservationWorkflow(){ + this.action = ExperimentImportWorkflow.ImportAction.APPEND_OVERWRITE; + } + + @Override + public Optional process(ImportServiceContext context) { + // Workflow processing the context + Action workflow = new Action(action.getUrlFragment(), this.getOrder()); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .action(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending or overwriting observation data + if (context != null && !action.isEqual(context.getAction())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; + } + + @Override + public int getOrder() { + return 2; + } + + public ExperimentImportWorkflow.ImportAction getAction() { + return this.action; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java new file mode 100644 index 000000000..c29fe9fd5 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java @@ -0,0 +1,54 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; + +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; + +import javax.inject.Singleton; +import java.util.Optional; + +@Singleton +public class NewExperimentWorkflow implements ImportWorkflow { + private final ExperimentImportWorkflow.ImportAction action; + + public NewExperimentWorkflow(){ + this.action = ExperimentImportWorkflow.ImportAction.NEW_OBSERVATION; + } + + @Override + public Optional process(ImportServiceContext context) { + // Workflow processing the context + Action workflow = new Action(action.getUrlFragment(), this.getOrder()); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .action(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless creating a new experiment + if (context != null && !action.isEqual(context.getAction())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; + } + + @Override + public int getOrder() { + return 1; + } + + public ExperimentImportWorkflow.ImportAction getAction() { + return this.action; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java new file mode 100644 index 000000000..57ee9790e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java @@ -0,0 +1,53 @@ +package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; + +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; + +import javax.inject.Singleton; +import java.util.Optional; + +@Singleton +public class NewEnvironmentWorkflow implements ImportWorkflow { + private final ExperimentImportWorkflow.ImportAction action; + + public NewEnvironmentWorkflow(){ + this.action = ExperimentImportWorkflow.ImportAction.APPEND_ENVIRONMENT; + } + + @Override + public Optional process(ImportServiceContext context) { + // Workflow processing the context + Action workflow = new Action(action.getUrlFragment(), this.getOrder()); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .action(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending a new environment + if (context != null && !action.isEqual(context.getAction())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; + } + + @Override + public int getOrder() { + return 3; + } + + public ExperimentImportWorkflow.ImportAction getAction() { + return this.action; + } +} From f726611cef1f5f3b3d82f76fd9b0c3ad5e8c8a01 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 18:52:39 -0400 Subject: [PATCH 16/55] create FileImportService method for GET workflows --- .../importer/controllers/ImportController.java | 13 ++++++++++--- .../importer/model/imports/BrAPIImportService.java | 4 ++++ .../ExperimentImportService.java | 5 +++++ .../brapps/importer/services/FileImportService.java | 9 +++++++-- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index 5c9ac0ca1..e9e9612e1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,6 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; +import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; @@ -214,16 +215,22 @@ public HttpResponse>> getSystemMappings(@Nu @Produces(MediaType.APPLICATION_JSON) @AddMetadata @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { - List workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + List workflows = null; + try { + workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); + } catch (DoesNotExistException e) { + log.error(e.getMessage(), e); + return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } List metadataStatus = new ArrayList<>(); metadataStatus.add(new Status(StatusCode.INFO, "Successful Query")); Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(workflows)); + Response> response = new Response(metadata, new DataResponse<>(workflows)); return HttpResponse.ok(response); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index d06454c30..997155d93 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -18,10 +18,14 @@ package org.breedinginsight.brapps.importer.model.imports; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.Action; + +import java.util.List; public interface BrAPIImportService { String getImportTypeId(); BrAPIImport getImportClass(); + List getWorkflows(); default String getInvalidIntegerMsg(String columnName) { return String.format("Column name \"%s\" must be integer type, but non-integer type provided.", columnName); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index de7d54961..760464d93 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; @@ -72,6 +73,10 @@ public String getImportTypeId() { public String getMissingColumnMsg(String columnName) { return "Column heading does not match template or ontology"; } + @Override + public List getWorkflows() { + return workflow.getWorkflows(); + } @Override public ImportPreviewResponse process(ImportServiceContext context) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 59438c696..7d9ebc0c6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -43,6 +43,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; +import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.workflow.WorkflowFactory; @@ -584,8 +585,12 @@ public List getSystemMappingByName(String name) { return importMappings; } - public List getWorkflowsForSystemMapping(UUID mappingId) { - return importMappingWorkflowDAO.getWorkflowsByImportMappingId(mappingId); + public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException { + ImportMapping mappingConfig = importMappingDAO.getMapping(mappingId) + .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); + BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) + .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); + return importService.getWorkflows(); } From f6952dd78f45955706e5adbe25eed24ab7b641e2 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 19:22:18 -0400 Subject: [PATCH 17/55] delete unused classes --- .../controllers/UploadController.java | 12 +-- .../daos/ImportMappingWorkflowDAO.java | 68 --------------- .../importer/services/FileImportService.java | 24 ++---- .../services/workflow/WorkflowFactory.java | 83 ------------------- 4 files changed, 11 insertions(+), 176 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index 053f1dc3c..5ee6eb062 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -158,15 +158,15 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/preview") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/preview") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable UUID workflowId, @PathVariable UUID uploadId) { + @PathVariable String workflow, @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, null, false); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, null, false); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { @@ -184,16 +184,16 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/commit") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/commit") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable UUID workflowId, @PathVariable UUID uploadId, + @PathVariable String workflow, @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, userInput, true); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java deleted file mode 100644 index 1b60bda45..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/ImportMappingWorkflowDAO.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.daos; - -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.dao.db.tables.daos.ImporterMappingWorkflowDao; -import org.jooq.Configuration; -import org.jooq.DSLContext; - -import javax.inject.Inject; -import java.util.List; -import java.util.Optional; -import java.util.UUID; - -import static org.breedinginsight.dao.db.Tables.*; - -public class ImportMappingWorkflowDAO extends ImporterMappingWorkflowDao { - - private DSLContext dsl; - - @Inject - public ImportMappingWorkflowDAO(Configuration config, DSLContext dsl) { - super(config); - this.dsl = dsl; - } - - /** - * Retrieves a list of ImportMappingWorkflow objects associated with the given mappingId. They are ordered by - * position for proper ordering on the front end. - * - * @param mappingId The UUID of the mapping to retrieve the workflows for. - * @return A list of ImportMappingWorkflow objects. - */ - public List getWorkflowsByImportMappingId(UUID mappingId) { - return dsl.select() - .from(IMPORTER_MAPPING_WORKFLOW) - .where(IMPORTER_MAPPING_WORKFLOW.MAPPING_ID.eq(mappingId)) - .orderBy(IMPORTER_MAPPING_WORKFLOW.POSITION.asc()) - .fetch(ImportMappingWorkflow::parseSQLRecord); - } - - /** - * Retrieves a workflow by its ID. - * - * @param workflowId The ID of the workflow to retrieve. - * @return An Optional containing the ImportMappingWorkflow if found, otherwise an empty Optional. - */ - public Optional getWorkflowById(UUID workflowId) { - return Optional.ofNullable(fetchOneById(workflowId)) - .map(ImportMappingWorkflow::new); - } - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 7d9ebc0c6..ffaa68614 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.micronaut.http.HttpStatus; -import io.micronaut.http.annotation.PathVariable; import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.multipart.CompletedFileUpload; import io.micronaut.http.server.exceptions.InternalServerException; @@ -33,7 +32,6 @@ import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingProgramDAO; -import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; @@ -44,12 +42,8 @@ import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.workflow.WorkflowFactory; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; -import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.ProgramService; @@ -88,14 +82,12 @@ public class FileImportService { private final ImportDAO importDAO; private final DSLContext dsl; private final ImportMappingProgramDAO importMappingProgramDAO; - private final ImportMappingWorkflowDAO importMappingWorkflowDAO; - private final WorkflowFactory workflowFactory; @Inject FileImportService(ProgramUserService programUserService, ProgramService programService, MimeTypeParser mimeTypeParser, ImportMappingDAO importMappingDAO, ObjectMapper objectMapper, MappingManager mappingManager, ImportConfigManager configManager, ImportDAO importDAO, DSLContext dsl, ImportMappingProgramDAO importMappingProgramDAO, - ImportMappingWorkflowDAO importMappingWorkflowDAO, WorkflowFactory workflowFactory, UserService userService) { + UserService userService) { this.programUserService = programUserService; this.programService = programService; this.mimeTypeParser = mimeTypeParser; @@ -107,8 +99,6 @@ public class FileImportService { this.dsl = dsl; this.importMappingProgramDAO = importMappingProgramDAO; this.userService = userService; - this.importMappingWorkflowDAO = importMappingWorkflowDAO; - this.workflowFactory = workflowFactory; } public List getAllImportTypeConfigs() { @@ -334,7 +324,7 @@ public ImportResponse uploadData(UUID programId, UUID mappingId, AuthenticatedUs return response; } - public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowId, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws + public ImportResponse updateUpload(UUID programId, UUID uploadId, String workflow, AuthenticatedUser actingUser, Map userInput, Boolean commit) throws DoesNotExistException, UnprocessableEntityException, AuthorizationException { Program program = validateRequest(programId, actingUser); @@ -384,7 +374,7 @@ public ImportResponse updateUpload(UUID programId, UUID uploadId, UUID workflowI } else { brAPIImportList = mappingManager.map(mappingConfig, data); } - processFile(workflowId, brAPIImportList, data, program, upload, user, commit, importService, actingUser); + processFile(workflow, brAPIImportList, data, program, upload, user, commit, importService, actingUser); } catch (UnprocessableEntityException e) { log.error(e.getMessage(), e); ImportProgress progress = upload.getProgress(); @@ -430,18 +420,14 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(UUID workflowId, List finalBrAPIImportList, Table data, Program program, + private void processFile(String workflow, List finalBrAPIImportList, Table data, Program program, ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { - - final Workflow[] workflow = {null}; - Optional.ofNullable(workflowId).ifPresent(id -> workflow[0] = workflowFactory.getWorkflow(id).orElse(null)); - ImportServiceContext context = ImportServiceContext.builder() - .workflow(workflow[0]) + .action(workflow) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java b/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java deleted file mode 100644 index 17d6fe64e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/workflow/WorkflowFactory.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.services.workflow; - -import io.micronaut.context.ApplicationContext; -import io.micronaut.context.annotation.Factory; -import io.micronaut.context.exceptions.NoSuchBeanException; -import io.micronaut.inject.qualifiers.Qualifiers; -import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.daos.ImportMappingWorkflowDAO; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; - -import javax.inject.Inject; -import java.util.Optional; -import java.util.UUID; - -@Factory -@Slf4j -public class WorkflowFactory { - - private final ImportMappingWorkflowDAO importMappingWorkflowDAO; - private final ApplicationContext applicationContext; - - @Inject - public WorkflowFactory(ImportMappingWorkflowDAO importMappingWorkflowDAO, - ApplicationContext applicationContext) { - this.importMappingWorkflowDAO = importMappingWorkflowDAO; - this.applicationContext = applicationContext; - } - - /** - * Produces the appropriate workflow instance based on the import context - * - * @param context the import context - * @return an Optional containing the workflow if id is not null, otherwise an empty Optional - * - * @throws IllegalStateException - * @throws NoSuchBeanException - */ - public Optional getWorkflow(UUID workflowId) { - - if (workflowId != null) { - // construct workflow from db record - Optional workflowOptional = importMappingWorkflowDAO.getWorkflowById(workflowId); - - ImportMappingWorkflow importMappingworkflow = workflowOptional.orElseThrow(() -> { - String msg = "Must have record in db for workflowId"; - log.error(msg); - return new IllegalStateException(msg); - }); - - // newer versions of micronaut have fancier ways to do this using annotations with provider but as - // far as I can tell it's not available in 2.5 - Workflow workflow; - try { - workflow = applicationContext.getBean(Workflow.class, Qualifiers.byName(importMappingworkflow.getBean())); - } catch (NoSuchBeanException e) { - log.error("Could not find workflow class implementation for bean: " + importMappingworkflow.getBean()); - throw e; - } - - return Optional.of(workflow); - } - - return Optional.empty(); - } -} From 5e31aa6f363f7b1767b49fe59b79c793cd10a373 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 20:33:43 -0400 Subject: [PATCH 18/55] clean up and renaming --- .../controllers/ImportController.java | 9 +-- .../model/imports/BrAPIImportService.java | 4 +- .../model/imports/ImportServiceContext.java | 5 +- .../ExperimentImportService.java | 22 ++--- .../importer/model/workflow/Action.java | 29 ------- .../model/workflow/ImportWorkflow.java | 35 ++++++-- .../model/workflow/ImportWorkflowResult.java | 2 +- .../importer/model/workflow/Workflow.java | 37 ++------- .../importer/services/FileImportService.java | 6 +- ...tWorkflow.java => ExperimentWorkflow.java} | 26 +++--- .../AppendOverwriteObservationWorkflow.java | 53 ------------ .../AppendOverwritePhenotypesWorkflow.java | 81 ++++++++++--------- .../workflow/CreateNewExperimentWorkflow.java | 79 +++++++++--------- .../workflow/NewExperimentWorkflow.java | 54 ------------- .../CreateNewEnvironmentWorkflow.java | 81 ++++++++++--------- .../workflow/NewEnvironmentWorkflow.java | 53 ------------ .../V1.22.0__add_experiment_workflows.sql | 51 ------------ 17 files changed, 195 insertions(+), 432 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ExperimentImportWorkflow.java => ExperimentWorkflow.java} (61%) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java delete mode 100644 src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java index e9e9612e1..1a16dadd7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/ImportController.java @@ -37,8 +37,7 @@ import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportMappingWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.ImportConfigManager; import org.breedinginsight.brapps.importer.model.config.ImportConfigResponse; import org.breedinginsight.brapps.importer.services.FileImportService; @@ -215,9 +214,9 @@ public HttpResponse>> getSystemMappings(@Nu @Produces(MediaType.APPLICATION_JSON) @AddMetadata @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { + public HttpResponse>> getWorkflowsForSystemMapping(@PathVariable UUID mappingId) { - List workflows = null; + List workflows = null; try { workflows = fileImportService.getWorkflowsForSystemMapping(mappingId); } catch (DoesNotExistException e) { @@ -230,7 +229,7 @@ public HttpResponse>> getWorkflowsForSystemMapping Pagination pagination = new Pagination(workflows.size(), workflows.size(), 1, 0); Metadata metadata = new Metadata(pagination, metadataStatus); - Response> response = new Response(metadata, new DataResponse<>(workflows)); + Response> response = new Response(metadata, new DataResponse<>(workflows)); return HttpResponse.ok(response); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 997155d93..3f25d99e6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -18,14 +18,14 @@ package org.breedinginsight.brapps.importer.model.imports; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import java.util.List; public interface BrAPIImportService { String getImportTypeId(); BrAPIImport getImportClass(); - List getWorkflows(); + List getWorkflows(); default String getInvalidIntegerMsg(String columnName) { return String.format("Column name \"%s\" must be integer type, but non-integer type provided.", columnName); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index c5a479e90..4d052cd33 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -19,10 +19,10 @@ import lombok.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import tech.tablesaw.api.Table; + import java.util.List; @Getter @@ -32,8 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { - private String action; - private Workflow workflow; + private String workflow; private List brAPIImports; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 760464d93..df60e9782 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -18,26 +18,18 @@ package org.breedinginsight.brapps.importer.model.imports.experimentObservation; import lombok.extern.slf4j.Slf4j; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; -import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import tech.tablesaw.api.Table; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; import java.util.List; -import java.util.Optional; @Singleton @Slf4j @@ -47,12 +39,12 @@ public class ExperimentImportService implements BrAPIImportService { private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; - private final ExperimentImportWorkflow workflow; + private final ExperimentWorkflow workflow; @Inject public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider, - ExperimentImportWorkflow workflow) + ExperimentWorkflow workflow) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; @@ -74,7 +66,7 @@ public String getMissingColumnMsg(String columnName) { return "Column heading does not match template or ontology"; } @Override - public List getWorkflows() { + public List getWorkflows() { return workflow.getWorkflows(); } @@ -82,8 +74,8 @@ public List getWorkflows() { public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - if (!context.getAction().isEmpty()) { - log.info("Workflow: " + context.getAction()); + if (!context.getWorkflow().isEmpty()) { + log.info("Workflow: " + context.getWorkflow()); } return workflow.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java deleted file mode 100644 index 65c3d0cbb..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Action.java +++ /dev/null @@ -1,29 +0,0 @@ -///* -// * See the NOTICE file distributed with this work for additional information -// * regarding copyright ownership. -// * -// * Licensed under the Apache License, Version 2.0 (the "License"); -// * you may not use this file except in compliance with the License. -// * You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ - -package org.breedinginsight.brapps.importer.model.workflow; - -import lombok.*; - -@Getter -@Setter -@ToString -@AllArgsConstructor -public class Action { - private String name; - private int order; -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java index c5dfc1957..b26a886bd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java @@ -1,12 +1,31 @@ -package org.breedinginsight.brapps.importer.model.workflow; +///* +// * See the NOTICE file distributed with this work for additional information +// * regarding copyright ownership. +// * +// * Licensed under the Apache License, Version 2.0 (the "License"); +// * you may not use this file except in compliance with the License. +// * You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, software +// * distributed under the License is distributed on an "AS IS" BASIS, +// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// * See the License for the specific language governing permissions and +// * limitations under the License. +// */ -import io.micronaut.core.order.Ordered; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +package org.breedinginsight.brapps.importer.model.workflow; -import java.util.Optional; +import lombok.*; -@FunctionalInterface -public interface ImportWorkflow extends Ordered { - Optional process(ImportServiceContext context); +@Getter +@Setter +@Builder +@ToString +@AllArgsConstructor +public class ImportWorkflow { + private String urlFragment; + private String displayName; + private int order; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java index 2e61a6d46..9304c1ce2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -28,6 +28,6 @@ @ToString @AllArgsConstructor public class ImportWorkflowResult { - private Action action; + private ImportWorkflow workflow; private Optional importPreviewResponse; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 9ea0e7bf9..8bc19120e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -1,36 +1,11 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.model.workflow; -public interface Workflow { +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; - /** - * Processes the given import context and returns the processed data. - * - * @param context the import context containing the necessary data for processing - * @return the processed data - */ - ProcessedData process(ImportContext context); +import java.util.Optional; - /** - * Retrieves the name of the Workflow for logging display purposes. - * - * @return the name of the Workflow - */ - String getName(); +@FunctionalInterface +public interface Workflow extends Ordered { + Optional process(ImportServiceContext context); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index ffaa68614..6a1914ab9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -41,7 +41,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.response.ImportResponse; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; -import org.breedinginsight.brapps.importer.model.workflow.Action; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingEntity; import org.breedinginsight.dao.db.tables.pojos.ImporterMappingProgramEntity; import org.breedinginsight.model.Program; @@ -427,7 +427,7 @@ private void processFile(String workflow, List finalBrAPIImportList CompletableFuture.supplyAsync(() -> { try { ImportServiceContext context = ImportServiceContext.builder() - .action(workflow) + .workflow(workflow) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) @@ -571,7 +571,7 @@ public List getSystemMappingByName(String name) { return importMappings; } - public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException { + public List getWorkflowsForSystemMapping(UUID mappingId) throws DoesNotExistException { ImportMapping mappingConfig = importMappingDAO.getMapping(mappingId) .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java similarity index 61% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java index e8a7d45da..04bbc4fc1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentImportWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java @@ -2,8 +2,8 @@ import io.micronaut.context.annotation.Primary; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.Action; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import javax.inject.Singleton; @@ -13,10 +13,10 @@ @Primary @Singleton -public class ExperimentImportWorkflow implements ImportWorkflow { - private final List workflows; +public class ExperimentWorkflow implements Workflow { + private final List workflows; - public ExperimentImportWorkflow(List workflows) { + public ExperimentWorkflow(List workflows) { this.workflows = workflows; } @@ -28,29 +28,33 @@ public Optional process(ImportServiceContext context) { .map(Optional::get) .findFirst(); } - public List getWorkflows() { + public List getWorkflows() { return workflows.stream() .map(workflow->workflow.process(null)) .filter(Optional::isPresent) .map(Optional::get) - .map(result->result.getAction()) + .map(result->result.getWorkflow()) .collect(Collectors.toList()); } - public enum ImportAction { - APPEND_OVERWRITE("append-overwrite-observation"), - NEW_OBSERVATION("new-observation"), - APPEND_ENVIRONMENT("append-environment"); + public enum Workflow { + NEW_OBSERVATION("new-experiment","Create new experiment"), + APPEND_OVERWRITE("append-dataset", "Append experimental dataset"), + APPEND_ENVIRONMENT("append-environment", "Create new experimental environment"); private String urlFragment; + private String displayName; + + Workflow(String urlFragment, String displayName) { - ImportAction(String urlFragment) { this.urlFragment = urlFragment; + this.displayName = displayName; } public String getUrlFragment() { return urlFragment; } + public String getDisplayName() { return displayName; } public boolean isEqual(String value) { return Optional.ofNullable(urlFragment.equals(value)).orElse(false); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java deleted file mode 100644 index ca5e932ef..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwriteObservationWorkflow.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; - -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; - -import javax.inject.Singleton; -import java.util.Optional; - -@Singleton -public class AppendOverwriteObservationWorkflow implements ImportWorkflow { - private final ExperimentImportWorkflow.ImportAction action; - - public AppendOverwriteObservationWorkflow(){ - this.action = ExperimentImportWorkflow.ImportAction.APPEND_OVERWRITE; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - Action workflow = new Action(action.getUrlFragment(), this.getOrder()); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .action(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending or overwriting observation data - if (context != null && !action.isEqual(context.getAction())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 2; - } - - public ExperimentImportWorkflow.ImportAction getAction() { - return this.action; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 21df19be6..44fb575bf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -1,50 +1,55 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.services.processors.experiment.append.workflow; -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; -import javax.inject.Named; - -/** - * This class represents a workflow for appending and overwriting phenotypes. The bean name must match the appropriate - * bean column value in the import_mapping_workflow db table - */ +import javax.inject.Singleton; +import java.util.Optional; -@Prototype -@Named("AppendOverwritePhenotypesWorkflow") +@Getter +@Singleton public class AppendOverwritePhenotypesWorkflow implements Workflow { + private final ExperimentWorkflow.Workflow workflow; + + public AppendOverwritePhenotypesWorkflow(){ + this.workflow = ExperimentWorkflow.Workflow.APPEND_OVERWRITE; + } + @Override - public ProcessedData process(ImportContext context) { - // TODO - return null; + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .urlFragment(getWorkflow().getUrlFragment()) + .displayName(getWorkflow().getDisplayName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending or overwriting observation data + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; } - /** - * Retrieves the name of the workflow. This is used for logging display purposes. - * - * @return the name of the workflow - */ @Override - public String getName() { - return "AppendOverwritePhenotypesWorkflow"; + public int getOrder() { + return 2; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 6f6ab661a..3c593ac74 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -1,50 +1,55 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; -import javax.inject.Named; +import javax.inject.Singleton; +import java.util.Optional; -/** - * This class represents a workflow for creating a new experiment. The bean name must match the appropriate bean column - * value in the import_mapping_workflow db table - */ -@Prototype -@Named("CreateNewExperimentWorkflow") +@Getter +@Singleton public class CreateNewExperimentWorkflow implements Workflow { + private final ExperimentWorkflow.Workflow workflow; + + public CreateNewExperimentWorkflow(){ + this.workflow = ExperimentWorkflow.Workflow.NEW_OBSERVATION; + } @Override - public ProcessedData process(ImportContext context) { - // TODO - return null; + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .urlFragment(getWorkflow().getUrlFragment()) + .displayName(getWorkflow().getDisplayName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless creating a new experiment + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; } - /** - * Retrieves the name of the workflow. This is used for logging display purposes. - * - * @return the name of the workflow - */ @Override - public String getName() { - return "CreateNewExperimentWorkflow"; + public int getOrder() { + return 1; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java deleted file mode 100644 index c29fe9fd5..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/NewExperimentWorkflow.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.create.workflow; - -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; - -import javax.inject.Singleton; -import java.util.Optional; - -@Singleton -public class NewExperimentWorkflow implements ImportWorkflow { - private final ExperimentImportWorkflow.ImportAction action; - - public NewExperimentWorkflow(){ - this.action = ExperimentImportWorkflow.ImportAction.NEW_OBSERVATION; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - Action workflow = new Action(action.getUrlFragment(), this.getOrder()); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .action(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless creating a new experiment - if (context != null && !action.isEqual(context.getAction())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 1; - } - - public ExperimentImportWorkflow.ImportAction getAction() { - return this.action; - } -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java index 5776ab59b..fe0b90c1f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -1,50 +1,55 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; -import io.micronaut.context.annotation.Prototype; -import org.breedinginsight.brapps.importer.model.workflow.ImportContext; -import org.breedinginsight.brapps.importer.model.workflow.ProcessedData; +import lombok.Getter; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; -import javax.inject.Named; - -/** - * This class represents a workflow for adding new environments to an existing experiment. The bean name must match - * the appropriate bean column value in the import_mapping_workflow db table - */ +import javax.inject.Singleton; +import java.util.Optional; -@Prototype -@Named("CreateNewEnvironmentWorkflow") +@Getter +@Singleton public class CreateNewEnvironmentWorkflow implements Workflow { + private final ExperimentWorkflow.Workflow workflow; + + public CreateNewEnvironmentWorkflow(){ + this.workflow = ExperimentWorkflow.Workflow.APPEND_ENVIRONMENT; + } + @Override - public ProcessedData process(ImportContext context) { - // TODO - return null; + public Optional process(ImportServiceContext context) { + // Workflow processing the context + ImportWorkflow workflow = ImportWorkflow.builder() + .urlFragment(getWorkflow().getUrlFragment()) + .displayName(getWorkflow().getDisplayName()) + .build(); + + // No-preview result + Optional result = Optional.of(ImportWorkflowResult.builder() + .workflow(workflow) + .importPreviewResponse(Optional.empty()) + .build()); + + // Skip this workflow unless appending a new environment + if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + return Optional.empty(); + } + + // Skip processing if no context, but return no-preview result for this workflow + if (context == null) { + return result; + } + + // Start processing the import... + return result; } - /** - * Retrieves the name of the workflow. This is used for logging display purposes. - * - * @return the name of the workflow - */ @Override - public String getName() { - return "CreateNewEnvironmentWorkflow"; + public int getOrder() { + return 3; } + } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java deleted file mode 100644 index 57ee9790e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/NewEnvironmentWorkflow.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; - -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.Action; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentImportWorkflow; - -import javax.inject.Singleton; -import java.util.Optional; - -@Singleton -public class NewEnvironmentWorkflow implements ImportWorkflow { - private final ExperimentImportWorkflow.ImportAction action; - - public NewEnvironmentWorkflow(){ - this.action = ExperimentImportWorkflow.ImportAction.APPEND_ENVIRONMENT; - } - - @Override - public Optional process(ImportServiceContext context) { - // Workflow processing the context - Action workflow = new Action(action.getUrlFragment(), this.getOrder()); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .action(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending a new environment - if (context != null && !action.isEqual(context.getAction())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 3; - } - - public ExperimentImportWorkflow.ImportAction getAction() { - return this.action; - } -} diff --git a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql deleted file mode 100644 index 2c9d4d547..000000000 --- a/src/main/resources/db/migration/V1.22.0__add_experiment_workflows.sql +++ /dev/null @@ -1,51 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - Table maps workflows to import mappings and provides required configuration options - - mapping_id - link to importer_mapping that this provides a workflow for - name - name that will be displayed on front end - bean - must match @Named("") annotation on Workflow class - position - for ordering records explicitly, wanted for front end default option and order - */ -CREATE TABLE importer_mapping_workflow -( - like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, - mapping_id UUID NOT NULL, - name TEXT NOT NULL, - bean TEXT NOT NULL, - position INTEGER NOT NULL -); - -ALTER TABLE importer_mapping_workflow - ADD FOREIGN KEY (mapping_id) REFERENCES importer_mapping (id); - -DO -$$ -DECLARE - exp_mapping_id UUID; -BEGIN - exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); - -INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean, position) -VALUES - (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow', 0), - (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow', 1), - (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow', 2); -END -$$; \ No newline at end of file From 708793dae1c11e78ece376f1a5fdd3813fcc821d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 22 May 2024 20:38:09 -0400 Subject: [PATCH 19/55] add overrides to import services --- .../model/imports/germplasm/GermplasmImportService.java | 6 ++++++ .../model/imports/sample/SampleSubmissionImportService.java | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index ce52f483f..3c3a65b4d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -57,6 +58,11 @@ public GermplasmImport getImportClass() { return new GermplasmImport(); } + @Override + public List getWorkflows() { + return null; + } + @Override public String getImportTypeId() { return IMPORT_TYPE_ID; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 2c8e8b7b5..55848b0a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -23,6 +23,7 @@ import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.Processor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.SampleSubmissionProcessor; @@ -59,6 +60,11 @@ public BrAPIImport getImportClass() { return new SampleSubmissionImport(); } + @Override + public List getWorkflows() { + return null; + } + @Override public ImportPreviewResponse process(ImportServiceContext context) throws Exception { List processors = List.of(sampleProcessorProvider.get()); From 68bdacb282b335af1874db9ede5ad421de4806ae Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 23 May 2024 06:33:16 -0400 Subject: [PATCH 20/55] create ExperimentWorkflow interface that extends Workflow interface --- .../ExperimentImportService.java | 12 ++++++------ .../importer/model/workflow/ExperimentWorkflow.java | 6 ++++++ .../brapps/importer/model/workflow/Workflow.java | 11 ----------- ...orkflow.java => ExperimentWorkflowNavigator.java} | 8 ++++---- .../workflow/AppendOverwritePhenotypesWorkflow.java | 10 +++++----- .../create/workflow/CreateNewExperimentWorkflow.java | 10 +++++----- .../workflow/CreateNewEnvironmentWorkflow.java | 10 +++++----- 7 files changed, 31 insertions(+), 36 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java rename src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/{ExperimentWorkflow.java => ExperimentWorkflowNavigator.java} (85%) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index df60e9782..c6d191bf0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -24,7 +24,7 @@ import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Inject; import javax.inject.Provider; @@ -39,16 +39,16 @@ public class ExperimentImportService implements BrAPIImportService { private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; - private final ExperimentWorkflow workflow; + private final ExperimentWorkflowNavigator workflowNavigator; @Inject public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider, - ExperimentWorkflow workflow) + ExperimentWorkflowNavigator workflowNavigator) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; - this.workflow = workflow; + this.workflowNavigator = workflowNavigator; } @Override @@ -67,7 +67,7 @@ public String getMissingColumnMsg(String columnName) { } @Override public List getWorkflows() { - return workflow.getWorkflows(); + return workflowNavigator.getWorkflows(); } @Override @@ -78,7 +78,7 @@ public ImportPreviewResponse process(ImportServiceContext context) log.info("Workflow: " + context.getWorkflow()); } - return workflow.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); + return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java new file mode 100644 index 000000000..c34794bab --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ExperimentWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface ExperimentWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java deleted file mode 100644 index 8bc19120e..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.breedinginsight.brapps.importer.model.workflow; - -import io.micronaut.core.order.Ordered; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; - -import java.util.Optional; - -@FunctionalInterface -public interface Workflow extends Ordered { - Optional process(ImportServiceContext context); -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java similarity index 85% rename from src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java rename to src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index 04bbc4fc1..bcf12d543 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -3,7 +3,7 @@ import io.micronaut.context.annotation.Primary; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import javax.inject.Singleton; @@ -13,10 +13,10 @@ @Primary @Singleton -public class ExperimentWorkflow implements Workflow { - private final List workflows; +public class ExperimentWorkflowNavigator implements ExperimentWorkflow { + private final List workflows; - public ExperimentWorkflow(List workflows) { + public ExperimentWorkflowNavigator(List workflows) { this.workflows = workflows; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 44fb575bf..e64a2f88a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -4,19 +4,19 @@ import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Singleton; import java.util.Optional; @Getter @Singleton -public class AppendOverwritePhenotypesWorkflow implements Workflow { - private final ExperimentWorkflow.Workflow workflow; +public class AppendOverwritePhenotypesWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; public AppendOverwritePhenotypesWorkflow(){ - this.workflow = ExperimentWorkflow.Workflow.APPEND_OVERWRITE; + this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_OVERWRITE; } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 3c593ac74..022727c5a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -4,19 +4,19 @@ import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Singleton; import java.util.Optional; @Getter @Singleton -public class CreateNewExperimentWorkflow implements Workflow { - private final ExperimentWorkflow.Workflow workflow; +public class CreateNewExperimentWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; public CreateNewExperimentWorkflow(){ - this.workflow = ExperimentWorkflow.Workflow.NEW_OBSERVATION; + this.workflow = ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION; } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java index fe0b90c1f..a5695e7f8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -4,19 +4,19 @@ import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.Workflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; import javax.inject.Singleton; import java.util.Optional; @Getter @Singleton -public class CreateNewEnvironmentWorkflow implements Workflow { - private final ExperimentWorkflow.Workflow workflow; +public class CreateNewEnvironmentWorkflow implements ExperimentWorkflow { + private final ExperimentWorkflowNavigator.Workflow workflow; public CreateNewEnvironmentWorkflow(){ - this.workflow = ExperimentWorkflow.Workflow.APPEND_ENVIRONMENT; + this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_ENVIRONMENT; } @Override From 0e022f12675712ad57574d4b7a27e2e5cac6700f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 23 May 2024 06:48:39 -0400 Subject: [PATCH 21/55] create interfaces for germplasm and sample workflows --- .../model/workflow/ImportMappingWorkflow.java | 56 ------------------- 1 file changed, 56 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java deleted file mode 100644 index 8dc5800bc..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportMappingWorkflow.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapps.importer.model.workflow; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; -import lombok.experimental.Accessors; -import lombok.experimental.SuperBuilder; - -import org.breedinginsight.dao.db.tables.pojos.ImporterMappingWorkflowEntity; -import org.jooq.Record; - -import static org.breedinginsight.dao.db.tables.ImporterMappingWorkflowTable.IMPORTER_MAPPING_WORKFLOW; - -@Getter -@Setter -@Accessors(chain=true) -@ToString -@NoArgsConstructor -@SuperBuilder() -public class ImportMappingWorkflow extends ImporterMappingWorkflowEntity { - - - public ImportMappingWorkflow(ImporterMappingWorkflowEntity importMappingWorkflowEntity) { - this.setId(importMappingWorkflowEntity.getId()); - this.setName(importMappingWorkflowEntity.getName()); - this.setBean(importMappingWorkflowEntity.getBean()); - this.setPosition(importMappingWorkflowEntity.getPosition()); - } - public static ImportMappingWorkflow parseSQLRecord(Record record) { - - return ImportMappingWorkflow.builder() - .id(record.getValue(IMPORTER_MAPPING_WORKFLOW.ID)) - .name(record.getValue(IMPORTER_MAPPING_WORKFLOW.NAME)) - .bean(record.getValue(IMPORTER_MAPPING_WORKFLOW.BEAN)) - .position(record.getValue(IMPORTER_MAPPING_WORKFLOW.POSITION)) - .build(); - } -} From f2a1034abf7ab01abf784255e4dd09f60c7c77b0 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 23 May 2024 08:14:51 -0400 Subject: [PATCH 22/55] create interfaces --- .../importer/model/workflow/GermplasmWorkflow.java | 6 ++++++ .../model/workflow/SampleSubmissionWorkflow.java | 6 ++++++ .../brapps/importer/model/workflow/Workflow.java | 11 +++++++++++ 3 files changed, 23 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java new file mode 100644 index 000000000..f26342d3b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/GermplasmWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface GermplasmWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java new file mode 100644 index 000000000..19be9bdcc --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/SampleSubmissionWorkflow.java @@ -0,0 +1,6 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +@FunctionalInterface +public interface SampleSubmissionWorkflow extends Workflow { + +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java new file mode 100644 index 000000000..8bc19120e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -0,0 +1,11 @@ +package org.breedinginsight.brapps.importer.model.workflow; + +import io.micronaut.core.order.Ordered; +import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; + +import java.util.Optional; + +@FunctionalInterface +public interface Workflow extends Ordered { + Optional process(ImportServiceContext context); +} From 1d63a71339140339df4888ceb140299bedb0e7e2 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 17 May 2024 11:54:49 -0400 Subject: [PATCH 23/55] [BI-2127] merged with BI-2127 into release/0.9.1 --- .../brapi/v2/services/BrAPITrialService.java | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index caab88879..60f427c5d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,5 +1,9 @@ package org.breedinginsight.brapi.v2.services; + +import com.github.filosganga.geogson.model.Coordinates; +import com.github.filosganga.geogson.model.positions.SinglePosition; +import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; import io.micronaut.http.MediaType; import io.micronaut.http.server.exceptions.InternalServerException; @@ -465,6 +469,20 @@ private Map createExportRow( row.put(ExperimentObservation.Columns.EXP_TYPE, experiment.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE).getAsString()); row.put(ExperimentObservation.Columns.ENV, Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), program.getKey())); row.put(ExperimentObservation.Columns.ENV_LOCATION, Utilities.removeProgramKey(study.getLocationName(), program.getKey())); + + // Lat, Long, Elevation + Coordinates coordinates = extractCoordinates(ou); + row.put( ExperimentObservation.Columns.LAT, coordinates==null? null : doubleToString(coordinates.getLat()) ); + row.put( ExperimentObservation.Columns.LONG, coordinates==null? null : doubleToString(coordinates.getLon()) ); + row.put( ExperimentObservation.Columns.ELEVATION, coordinates==null? null : doubleToString(coordinates.getAlt()) ); + + // RTK + JsonObject additionalInfo = ou.getAdditionalInfo(); + String rtk = ( additionalInfo==null || additionalInfo.get(BrAPIAdditionalInfoFields.RTK) ==null ) + ? null + : additionalInfo.get(BrAPIAdditionalInfoFields.RTK).getAsString(); + row.put(ExperimentObservation.Columns.RTK, rtk); + BrAPISeason season = seasonDAO.getSeasonById(study.getSeasons().get(0), program.getId()); row.put(ExperimentObservation.Columns.ENV_YEAR, season.getYear()); row.put(ExperimentObservation.Columns.EXP_UNIT_ID, Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); @@ -484,11 +502,13 @@ private Map createExportRow( .findFirst(); blockLevel.ifPresent(brAPIObservationUnitLevelRelationship -> row.put(ExperimentObservation.Columns.BLOCK_NUM, Integer.parseInt(brAPIObservationUnitLevelRelationship.getLevelCode()))); - if (ou.getObservationUnitPosition() != null && ou.getObservationUnitPosition().getPositionCoordinateX() != null && - ou.getObservationUnitPosition().getPositionCoordinateY() != null) { + + //Row and Column + if ( ou.getObservationUnitPosition() != null ) { row.put(ExperimentObservation.Columns.ROW, ou.getObservationUnitPosition().getPositionCoordinateX()); row.put(ExperimentObservation.Columns.COLUMN, ou.getObservationUnitPosition().getPositionCoordinateY()); } + if (ou.getTreatments() != null && !ou.getTreatments().isEmpty()) { row.put(ExperimentObservation.Columns.TREATMENT_FACTORS, ou.getTreatments().get(0).getFactor()); } else { @@ -498,8 +518,26 @@ private Map createExportRow( return row; } + private String doubleToString(double val){ + return Double.isNaN(val) ? null : String.valueOf( val ); + } - + private Coordinates extractCoordinates(BrAPIObservationUnit ou){ + Coordinates coordinates = null; + if ( ou.getObservationUnitPosition()!=null + && ou.getObservationUnitPosition().getGeoCoordinates()!=null + && ou.getObservationUnitPosition().getGeoCoordinates().getGeometry()!=null + && ou.getObservationUnitPosition().getGeoCoordinates().getGeometry().positions()!=null + ) + { + Object o = ou.getObservationUnitPosition().getGeoCoordinates().getGeometry().positions(); + if (o instanceof SinglePosition){ + SinglePosition sp = (SinglePosition)o; + coordinates= sp.coordinates(); + } + } + return coordinates; + } private void addObsVarColumns( List columns, From 8a28a2dfe633f52ee7a93b2641e296d4bdd3d165 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 24 May 2024 14:21:30 +0000 Subject: [PATCH 24/55] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 111d06bd2..b2f683567 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.1+741 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fa8c7cb586f4bf4110a505e10e090fecee0cd714 +version=v0.9.1+747 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1d63a71339140339df4888ceb140299bedb0e7e2 From 9104a3e456e9044104d3541750e29a0746be2c4b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 29 May 2024 19:01:01 -0400 Subject: [PATCH 25/55] restore germplasmAccessionNumber to GermplasmTemplateMap --- ...rmplasm_restore_accession_number_field.sql | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/main/resources/db/migration/V1.22.0__germplasm_restore_accession_number_field.sql diff --git a/src/main/resources/db/migration/V1.22.0__germplasm_restore_accession_number_field.sql b/src/main/resources/db/migration/V1.22.0__germplasm_restore_accession_number_field.sql new file mode 100644 index 000000000..31530da81 --- /dev/null +++ b/src/main/resources/db/migration/V1.22.0__germplasm_restore_accession_number_field.sql @@ -0,0 +1,21 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +update importer_mapping +set mapping = '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent GID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent GID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "b910adfe-a474-47a0-8410-514578898436", "value": {"fileFieldName": "Synonyms"}, "objectId": "synonyms"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "bc09c6e1-866f-45c3-a285-a25859e8c982", "value": {"fileFieldName": "GID"}, "objectId": "germplasmAccessionNumber"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', + file = '[{"Germplasm Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2"}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "2", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "1", "Synonyms": "test1;test2"}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "7", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "2", "Synonyms": "test1;test2"}, {"Name": "BITest BI003", "Source": "Ithaca Nursery", "Entry No": "8", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "7", "Female Parent Entry No": "2", "Synonyms": "test1;test2"}, {"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "3", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2"}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "640e8b58-1b1c-44a6-91a6-85b2b773376b", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2"}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "5", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent GID": "", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "2", "Synonyms": "test1;test2"}]' +where name = 'GermplasmTemplateMap'; \ No newline at end of file From f50cad473c04a7351bfcf9cd7b4af321e41bca1e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 30 May 2024 18:56:11 +0000 Subject: [PATCH 26/55] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index b2f683567..ad14d8045 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.1+747 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1d63a71339140339df4888ceb140299bedb0e7e2 +version=v0.9.1+749 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/4e51c9f68656304da9a17b99103b0bd740e80f6f From 402eb5de2dd58f1fa8076db344a88e150a5ed390 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 31 May 2024 10:40:55 -0400 Subject: [PATCH 27/55] create DomainImportService abstract class --- .../model/imports/DomainImportService.java | 72 +++++++++++++++++++ .../ExperimentImportService.java | 32 +++------ .../importer/model/workflow/Workflow.java | 6 ++ 3 files changed, 86 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java new file mode 100644 index 000000000..5f89de64a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -0,0 +1,72 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.extern.slf4j.Slf4j; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; +import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; + +@Singleton +@Slf4j +public abstract class DomainImportService implements BrAPIImportService { + + private final Provider experimentProcessorProvider; + private final Provider processorManagerProvider; + private final Workflow workflowNavigator; + + @Inject + public DomainImportService(Provider experimentProcessorProvider, + Provider processorManagerProvider) + { + this.experimentProcessorProvider = experimentProcessorProvider; + this.processorManagerProvider = processorManagerProvider; + this.workflowNavigator = getNavigator(); + } + + protected abstract Workflow getNavigator(); + @Override + public String getMissingColumnMsg(String columnName) { + return "Column heading does not match template or ontology"; + } + @Override + public List getWorkflows() { + return workflowNavigator.getWorkflows(); + } + + @Override + public ImportPreviewResponse process(ImportServiceContext context) + throws Exception { + + if (!context.getWorkflow().isEmpty()) { + log.info("Workflow: " + context.getWorkflow()); + } + + return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); + } +} + diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index c6d191bf0..60fe4bb3b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -19,9 +19,11 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.imports.DomainImportService; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; @@ -33,12 +35,9 @@ @Singleton @Slf4j -public class ExperimentImportService implements BrAPIImportService { +public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - - private final Provider experimentProcessorProvider; - private final Provider processorManagerProvider; private final ExperimentWorkflowNavigator workflowNavigator; @Inject @@ -46,10 +45,13 @@ public ExperimentImportService(Provider experimentProcessor Provider processorManagerProvider, ExperimentWorkflowNavigator workflowNavigator) { - this.experimentProcessorProvider = experimentProcessorProvider; - this.processorManagerProvider = processorManagerProvider; + super(experimentProcessorProvider, processorManagerProvider); this.workflowNavigator = workflowNavigator; } + @Override + public Workflow getNavigator() { + return this.workflowNavigator; + } @Override public ExperimentObservation getImportClass() { @@ -61,24 +63,6 @@ public String getImportTypeId() { return IMPORT_TYPE_ID; } - @Override - public String getMissingColumnMsg(String columnName) { - return "Column heading does not match template or ontology"; - } - @Override - public List getWorkflows() { - return workflowNavigator.getWorkflows(); - } - - @Override - public ImportPreviewResponse process(ImportServiceContext context) - throws Exception { - if (!context.getWorkflow().isEmpty()) { - log.info("Workflow: " + context.getWorkflow()); - } - - return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); - } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index 8bc19120e..17f48ee57 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -3,9 +3,15 @@ import io.micronaut.core.order.Ordered; import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; @FunctionalInterface public interface Workflow extends Ordered { Optional process(ImportServiceContext context); + default List getWorkflows() { + // Default implementation for getWorkflows method + return new ArrayList<>(); + } } From f9a078393be64419751ad42862a8e06f44b97498 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 5 Jun 2024 22:08:40 +0000 Subject: [PATCH 28/55] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 837e4609c..9eb5a3f90 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6662651d581bb80396250f3ff38bc9443c4a4744 +version=v0.10.0+752 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/7fd32a0140ea299da63eaa3d3b14bf75725a0086 From bce2c7fbcce36ee87698257b521b9f87ffd2aa95 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:11:36 -0400 Subject: [PATCH 29/55] change Workflow field names --- .../model/workflow/ImportWorkflow.java | 4 ++-- .../ExperimentWorkflowNavigator.java | 18 +++++++++--------- .../AppendOverwritePhenotypesWorkflow.java | 4 ++-- .../workflow/CreateNewExperimentWorkflow.java | 4 ++-- .../workflow/CreateNewEnvironmentWorkflow.java | 4 ++-- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java index b26a886bd..30dbac06a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflow.java @@ -25,7 +25,7 @@ @ToString @AllArgsConstructor public class ImportWorkflow { - private String urlFragment; - private String displayName; + private String id; + private String name; private int order; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index bcf12d543..b43b859f5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -42,22 +42,22 @@ public enum Workflow { APPEND_OVERWRITE("append-dataset", "Append experimental dataset"), APPEND_ENVIRONMENT("append-environment", "Create new experimental environment"); - private String urlFragment; - private String displayName; + private String id; + private String name; - Workflow(String urlFragment, String displayName) { + Workflow(String id, String name) { - this.urlFragment = urlFragment; - this.displayName = displayName; + this.id = id; + this.name = name; } - public String getUrlFragment() { - return urlFragment; + public String getId() { + return id; } - public String getDisplayName() { return displayName; } + public String getName() { return name; } public boolean isEqual(String value) { - return Optional.ofNullable(urlFragment.equals(value)).orElse(false); + return Optional.ofNullable(id.equals(value)).orElse(false); } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index e64a2f88a..6ff074059 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -23,8 +23,8 @@ public AppendOverwritePhenotypesWorkflow(){ public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() - .urlFragment(getWorkflow().getUrlFragment()) - .displayName(getWorkflow().getDisplayName()) + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) .build(); // No-preview result diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 022727c5a..503e4e5f0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -23,8 +23,8 @@ public CreateNewExperimentWorkflow(){ public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() - .urlFragment(getWorkflow().getUrlFragment()) - .displayName(getWorkflow().getDisplayName()) + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) .build(); // No-preview result diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java index a5695e7f8..1b7eba9ce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java @@ -23,8 +23,8 @@ public CreateNewEnvironmentWorkflow(){ public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() - .urlFragment(getWorkflow().getUrlFragment()) - .displayName(getWorkflow().getDisplayName()) + .id(getWorkflow().getId()) + .name(getWorkflow().getName()) .build(); // No-preview result From 6060fcc990177a9c7e779af2cc66a07937cf33e4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:29:14 -0400 Subject: [PATCH 30/55] use processorManager until Workflow business logic in place --- .../importer/model/imports/DomainImportService.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 5f89de64a..d826033fb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -21,6 +21,7 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; @@ -35,6 +36,7 @@ @Slf4j public abstract class DomainImportService implements BrAPIImportService { + // TODO: delete processor fields once WorkflowNavigator is used private final Provider experimentProcessorProvider; private final Provider processorManagerProvider; private final Workflow workflowNavigator; @@ -66,7 +68,15 @@ public ImportPreviewResponse process(ImportServiceContext context) log.info("Workflow: " + context.getWorkflow()); } - return workflowNavigator.process(context).flatMap(r->r.getImportPreviewResponse()).orElse(null); + // TODO: return results from WorkflowNavigator once processing logic is in separate workflows + // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + return processorManagerProvider.get().process(context.getBrAPIImports(), + List.of(experimentProcessorProvider.get()), + context.getData(), + context.getProgram(), + context.getUpload(), + context.getUser(), + context.isCommit()); } } From 90f478becd85cd15db0d47a989ee399fb38d2256 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 16:29:33 -0400 Subject: [PATCH 31/55] fix workflow navigator injection --- .../importer/model/imports/DomainImportService.java | 8 ++++---- .../experimentObservation/ExperimentImportService.java | 9 ++------- .../model/imports/germplasm/GermplasmImportService.java | 3 ++- .../imports/sample/SampleSubmissionImportService.java | 3 ++- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index d826033fb..13719e18d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -41,16 +41,16 @@ public abstract class DomainImportService implements BrAPIImportService { private final Provider processorManagerProvider; private final Workflow workflowNavigator; - @Inject + public DomainImportService(Provider experimentProcessorProvider, - Provider processorManagerProvider) + Provider processorManagerProvider, + Workflow workflowNavigator) { this.experimentProcessorProvider = experimentProcessorProvider; this.processorManagerProvider = processorManagerProvider; - this.workflowNavigator = getNavigator(); + this.workflowNavigator = workflowNavigator; } - protected abstract Workflow getNavigator(); @Override public String getMissingColumnMsg(String columnName) { return "Column heading does not match template or ontology"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 60fe4bb3b..793506c73 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -38,19 +38,14 @@ public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - private final ExperimentWorkflowNavigator workflowNavigator; + // TODO: delete processor fields once WorkflowNavigator is used @Inject public ExperimentImportService(Provider experimentProcessorProvider, Provider processorManagerProvider, ExperimentWorkflowNavigator workflowNavigator) { - super(experimentProcessorProvider, processorManagerProvider); - this.workflowNavigator = workflowNavigator; - } - @Override - public Workflow getNavigator() { - return this.workflowNavigator; + super(experimentProcessorProvider, processorManagerProvider, workflowNavigator); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index 3c3a65b4d..0caebe65e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -34,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; @Singleton @@ -60,7 +61,7 @@ public GermplasmImport getImportClass() { @Override public List getWorkflows() { - return null; + return new ArrayList<>(); } @Override diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 55848b0a1..eb7328ecf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -34,6 +34,7 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.ArrayList; import java.util.List; @Singleton @@ -62,7 +63,7 @@ public BrAPIImport getImportClass() { @Override public List getWorkflows() { - return null; + return new ArrayList<>(); } @Override From 62cf9ac438684991bb48358ca46eea1418a280eb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:07:33 -0400 Subject: [PATCH 32/55] set the order field for workflows --- .../experiment/ExperimentWorkflowNavigator.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index b43b859f5..63bfcedbd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -29,12 +29,21 @@ public Optional process(ImportServiceContext context) { .findFirst(); } public List getWorkflows() { - return workflows.stream() + // Each workflow returns in the field workflow the metadata about the workflow that processed the import context. + // Loop over all workflows, processing a null context, to collect just the metadata + List workflowSummaryList = workflows.stream() .map(workflow->workflow.process(null)) .filter(Optional::isPresent) .map(Optional::get) .map(result->result.getWorkflow()) .collect(Collectors.toList()); + + // The order field for each workflow is set to the order in the list + for (int i = 0; i < workflowSummaryList.size(); i++) { + workflowSummaryList.get(i).setOrder(i); + } + + return workflowSummaryList; } public enum Workflow { From 57f5cfbfc10f937e1099fd5a017997a7102db78f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:43:08 -0400 Subject: [PATCH 33/55] fix npe --- .../brapps/importer/model/imports/DomainImportService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 13719e18d..af65af238 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -64,7 +64,7 @@ public List getWorkflows() { public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - if (!context.getWorkflow().isEmpty()) { + if (context.getWorkflow() != null && !context.getWorkflow().isEmpty()) { log.info("Workflow: " + context.getWorkflow()); } From 79189ba768ad3219c57ea950b04a543409e6b77f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:18:23 -0400 Subject: [PATCH 34/55] Messing with dependency issue --- .../experimentObservation/ExperimentImportService.java | 9 +++++++++ .../steps/PopulateNewPendingImportObjectsStep.java | 8 +------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 149da0e77..013d55012 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.DomainImportService; +import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; @@ -26,12 +27,14 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; +import java.util.List; @Singleton @Slf4j public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; + private final ExperimentWorkflowNavigator workflowNavigator; // TODO: delete processor fields once WorkflowNavigator is used @Inject @@ -40,6 +43,7 @@ public ExperimentImportService(Provider experimentProcessor ExperimentWorkflowNavigator workflowNavigator) { super(experimentProcessorProvider, processorManagerProvider, workflowNavigator); + this.workflowNavigator = workflowNavigator; } @Override @@ -51,5 +55,10 @@ public ExperimentObservation getImportClass() { public String getImportTypeId() { return IMPORT_TYPE_ID; } + + @Override + public List getWorkflows() throws Exception{ + return workflowNavigator.getWorkflows(); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index a471b84f3..f5ec2cc96 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -73,9 +73,7 @@ @Slf4j public class PopulateNewPendingImportObjectsStep { - private final ExperimentValidateService experimentValidateService; private final ExperimentSeasonService experimentSeasonService; - private final BrAPIObservationDAO brAPIObservationDAO; private final BrAPIObservationUnitDAO brAPIObservationUnitDAO; private final DSLContext dsl; private final Gson gson; @@ -84,14 +82,10 @@ public class PopulateNewPendingImportObjectsStep { private String BRAPI_REFERENCE_SOURCE; @Inject - public PopulateNewPendingImportObjectsStep(ExperimentValidateService experimentValidateService, - ExperimentSeasonService experimentSeasonService, - BrAPIObservationDAO brAPIObservationDAO, + public PopulateNewPendingImportObjectsStep(ExperimentSeasonService experimentSeasonService, BrAPIObservationUnitDAO brAPIObservationUnitDAO, DSLContext dsl) { - this.experimentValidateService = experimentValidateService; this.experimentSeasonService = experimentSeasonService; - this.brAPIObservationDAO = brAPIObservationDAO; this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; this.dsl = dsl; this.gson = new JSON().getGson(); From 0ef8cd5d21a559ea886683b3109dcea422acd147 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 14:57:51 -0400 Subject: [PATCH 35/55] Removed ununsed fileImportService injection and resolved DI issue --- .../brapps/importer/services/FileImportService.java | 5 +++++ .../brapps/importer/services/FileMappingUtil.java | 8 +------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 0c2a9309d..da5a68e10 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -576,6 +576,11 @@ public List getWorkflowsForSystemMapping(UUID mappingId) throws .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); + // NOTE: + // this is creating a workflow navigator to call getWorkflows + // workflowNavigator.getWorkflows(); + // getWorkflows is creating the ExperimentWorkflow which is injecting the file import service and + // resulting in a circular dependency return importService.getWorkflows(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java index cb93daf48..8d3ed8146 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java @@ -32,13 +32,7 @@ @Singleton public class FileMappingUtil { - public static final String EXPERIMENT_TEMPLATE_NAME = "ExperimentsTemplateMap"; - private FileImportService fileImportService; - - - @Inject - public FileMappingUtil(FileImportService fileImportService) { - this.fileImportService = fileImportService; + public FileMappingUtil() { } // Returns a list of integers to identify the target row of the relationship. -1 if no relationship was found From 1acd04bd343865d81edbf02d2af56b05bf04f8e7 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:23:25 -0400 Subject: [PATCH 36/55] Added in BI-2128 change --- .../workflow/steps/PopulateNewPendingImportObjectsStep.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index f5ec2cc96..ddb4193ab 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -567,7 +567,9 @@ private void fetchOrCreateObservationPIO(ProcessedPhenotypeData phenotypeData, key = getImportObservationHash(importRow, variableName); if (existingObsByObsHash.containsKey(key)) { - if (!isObservationMatched(phenotypeData, pendingData, key, value, column, rowNum)){ + // NOTE: BI-2128 change added after refactor branch + // Update observation value only if it is changed and new value is not blank. + if (!isObservationMatched(phenotypeData, pendingData, key, value, column, rowNum) && StringUtils.isNotBlank(value)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); From 6a4043d2202adfa7c97e06d65bbc4d14c5045fc3 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:36:42 -0400 Subject: [PATCH 37/55] Updated create only tests to use create workflow --- .../importer/ExperimentFileImportTest.java | 239 ++++++++++++------ .../brapps/importer/ImportTestUtils.java | 95 ++++++- 2 files changed, 243 insertions(+), 91 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 48bffd4c2..fbb02435e 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -24,6 +24,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; @@ -77,6 +78,7 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static io.micronaut.http.HttpRequest.GET; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.jupiter.api.Assertions.*; @@ -142,9 +144,11 @@ public class ExperimentFileImportTest extends BrAPITest { private BrAPISeasonDAO seasonDAO; private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) - (json, type, context) -> OffsetDateTime.parse(json.getAsString())) - .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) - .create(); + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) + .create(); + + private String newExperimentWorkflowId; @BeforeAll public void setup() { @@ -153,7 +157,27 @@ public void setup() { mappingId = (String) setupObjects.get("mappingId"); testUser = (BiUserEntity) setupObjects.get("testUser"); securityFp = (FannyPack) setupObjects.get("securityFp"); + newExperimentWorkflowId = getNewExperimentWorkflowId(); + } + + /** + * TODO: assumes new workflow is first in list, doesn't look at position property, would be more robust to + * look at that instead of assuming order + * @return + */ + public String getNewExperimentWorkflowId() { + // GET /import/mappings{?importName} + Flowable> call = client.exchange( + GET("/import/mappings/"+mappingId+"/workflows").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + + return JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); } /* @@ -193,16 +217,20 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String workflowId = "new-experiment"; + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + // TODO: remove this + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -251,16 +279,18 @@ public void importNewExpMultiNewEnvSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(2, previewRows.size()); JsonObject firstRow = previewRows.get(0).getAsJsonObject(); @@ -298,7 +328,8 @@ public void importExistingExpAndEnvErrorMessage() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + JsonObject expResult = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map dupExp = new HashMap<>(); dupExp.put(Columns.GERMPLASM_GID, "1"); @@ -315,16 +346,17 @@ public void importExistingExpAndEnvErrorMessage() { dupExp.put(Columns.ROW, "1"); dupExp.put(Columns.COLUMN, "1"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + expResult = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, expResult.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + expResult); + assertTrue(expResult.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); } @Test @@ -347,9 +379,10 @@ public void importNewEnvNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + JsonObject uploadResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId, newExperimentWorkflowId); - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + JsonArray previewRows = uploadResponse.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); @@ -381,43 +414,53 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit, newExperimentWorkflowId); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit, newExperimentWorkflowId); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit, newExperimentWorkflowId); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit, newExperimentWorkflowId); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit, newExperimentWorkflowId); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit, newExperimentWorkflowId); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit, newExperimentWorkflowId); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit, newExperimentWorkflowId); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); } @Test @@ -441,7 +484,8 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -491,7 +535,9 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -529,7 +575,8 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit, newExperimentWorkflowId); } @ParameterizedTest @@ -554,7 +601,8 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + //JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -594,7 +642,9 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + //uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyWorkflowFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); + } @ParameterizedTest @@ -617,21 +667,24 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + //importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId, newExperimentWorkflowId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + //Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + //JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, true, client, program, mappingId, newExperimentWorkflowId); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); @@ -1022,6 +1075,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { - a new experiment is created after the first experiment - verify the second experiment gets created successfully */ + //TODO: this one @Test @SneakyThrows public void importSecondExpAfterFirstExpWithObs() { @@ -1043,7 +1097,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + //JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + JsonObject resultA = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -1071,7 +1126,8 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + //JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + JsonObject resultB = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId, newExperimentWorkflowId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -1337,10 +1393,10 @@ private Map assertRowSaved(Map expected, Program assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { assertEquals(expected.get(Columns.TEST_CHECK), - ou.getObservationUnitPosition() - .getEntryType() - .name() - .substring(0, 1)); + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); } assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); @@ -1433,10 +1489,10 @@ private Map assertValidPreviewRow(Map expected, if(traits != null) { assertNotNull(actual.get("observations")); observations = StreamSupport.stream(actual.getAsJsonArray("observations") - .spliterator(), false) - .map(obs -> gson.fromJson(obs.getAsJsonObject() - .getAsJsonObject("brAPIObject"), BrAPIObservation.class)) - .collect(Collectors.toList()); + .spliterator(), false) + .map(obs -> gson.fromJson(obs.getAsJsonObject() + .getAsJsonObject("brAPIObject"), BrAPIObservation.class)) + .collect(Collectors.toList()); ret.put("observations", observations); } @@ -1444,10 +1500,10 @@ private Map assertValidPreviewRow(Map expected, assertEquals(expected.get(Columns.GERMPLASM_GID), germplasm.getAccessionNumber()); if(expected.containsKey(Columns.TEST_CHECK) && StringUtils.isNotBlank((String)expected.get(Columns.TEST_CHECK))) { assertEquals(expected.get(Columns.TEST_CHECK), - ou.getObservationUnitPosition() - .getEntryType() - .name() - .substring(0, 1)); + ou.getObservationUnitPosition() + .getEntryType() + .name() + .substring(0, 1)); } assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(trial.getTrialName(), program.getKey())); assertEquals(expected.get(Columns.EXP_TITLE), Utilities.removeProgramKey(study.getTrialName(), program.getKey())); @@ -1518,8 +1574,8 @@ private String yearToSeasonDbId(String year, UUID programId) throws ApiException for (BrAPISeason season : seasons) { if (null == season.getSeasonName() || season.getSeasonName() - .isBlank() || season.getSeasonName() - .equals(year)) { + .isBlank() || season.getSeasonName() + .equals(year)) { return season.getSeasonDbId(); } } @@ -1530,17 +1586,17 @@ private String yearToSeasonDbId(String year, UUID programId) throws ApiException private Program createProgram(String name, String abbv, String key, String referenceSource, List germplasm, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { SpeciesEntity validSpecies = speciesDAO.findAll().get(0); SpeciesRequest speciesRequest = SpeciesRequest.builder() - .commonName(validSpecies.getCommonName()) - .id(validSpecies.getId()) - .build(); + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); ProgramRequest programRequest1 = ProgramRequest.builder() - .name(name) - .abbreviation(abbv) - .documentationUrl("localhost:8080") - .objective("To test things") - .species(speciesRequest) - .key(key) - .build(); + .name(name) + .abbreviation(abbv) + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key(key) + .build(); TestUtils.insertAndFetchTestProgram(gson, client, programRequest1); @@ -1609,6 +1665,33 @@ private JsonObject uploadAndVerifyFailure(Program program, File file, String exp JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + private JsonObject uploadAndVerifyWorkflowFailure(Program program, File file, String expectedColumnError, boolean commit, String workflowId) throws InterruptedException, IOException { + + //Flowable> call = importTestUtils.uploadDataFile(file, null, true, client, program, mappingId); + //HttpResponse response = call.blockingFirst(); + //assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + //String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + //HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + + JsonObject result = importTestUtils.uploadAndFetchWorkflowNoStatusCheck(file, null, true, client, program, mappingId, newExperimentWorkflowId); + //JsonObject result = JsonParser.parseString(upload).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); assertEquals(1, rowErrors.size()); JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 12b79ac15..f5dd37f51 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -97,6 +97,38 @@ public Flowable> uploadDataFile(File file, Map> uploadWorkflowDataFile(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) { + + MultipartBody requestBody = MultipartBody.builder().addPart("file", file).build(); + + // Upload file + String uploadUrl = String.format("/programs/%s/import/mappings/%s/data", program.getId(), mappingId); + Flowable> call = client.exchange( + POST(uploadUrl, requestBody) + .contentType(MediaType.MULTIPART_FORM_DATA_TYPE) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + JsonObject result = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"); + String importId = result.get("importId").getAsString(); + + // Process data + String url = String.format("/programs/%s/import/mappings/%s/workflows/%s/data/%s/%s", program.getId(), mappingId, workflowId, importId, commit ? "commit" : "preview"); + Flowable> processCall = client.exchange( + PUT(url, userData) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + return processCall; + + } + public HttpResponse getUploadedFile(String importId, RxHttpClient client, Program program, String mappingId) throws InterruptedException { Flowable> call = client.exchange( GET(String.format("/programs/%s/import/mappings/%s/data/%s?mapping=true", program.getId(), mappingId, importId)) @@ -123,16 +155,16 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, // Species Species validSpecies = speciesService.getAll().get(0); SpeciesRequest speciesRequest = SpeciesRequest.builder() - .commonName(validSpecies.getCommonName()) - .id(validSpecies.getId()) - .build(); + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); // Insert program ProgramRequest program = ProgramRequest.builder() - .name("Test Program") - .species(speciesRequest) - .key("TEST") - .build(); + .name("Test Program") + .species(speciesRequest) + .key("TEST") + .build(); Program validProgram = this.insertAndFetchTestProgram(program, client, gson); // Get import @@ -141,18 +173,18 @@ public Map setup(RxHttpClient client, Gson gson, DSLContext dsl, ); HttpResponse response = call.blockingFirst(); String mappingId = JsonParser.parseString(response.body()).getAsJsonObject() - .getAsJsonObject("result") - .getAsJsonArray("data") - .get(0).getAsJsonObject().get("id").getAsString(); + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); BiUserEntity testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).get(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); return Map.of("program", validProgram, - "mappingId", mappingId, - "testUser", testUser, - "securityFp", securityFp); + "mappingId", mappingId, + "testUser", testUser, + "securityFp", securityFp); } @@ -170,6 +202,43 @@ public JsonObject uploadAndFetch(File file, Map userData, Boolea return result; } + public JsonObject uploadAndFetchWorkflow(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + return result; + } + + public JsonObject uploadAndFetchWorkflowNoStatusCheck(File file, + Map userData, + Boolean commit, + RxHttpClient client, + Program program, + String mappingId, + String workflowId) throws InterruptedException { + Flowable> call = uploadWorkflowDataFile(file, userData, commit, client, program, mappingId, workflowId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + return result; + } + public List createTraits(int numToCreate) { List traits = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { From 651cffc5dc8fc31e27a0fed557cdf1267ea1e0ee Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 15:52:11 -0400 Subject: [PATCH 38/55] Removed unused migration --- .../V1.23.0__add_experiment_workflows.sql | 51 ------------------- 1 file changed, 51 deletions(-) delete mode 100644 src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql diff --git a/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql b/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql deleted file mode 100644 index 2c9d4d547..000000000 --- a/src/main/resources/db/migration/V1.23.0__add_experiment_workflows.sql +++ /dev/null @@ -1,51 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - Table maps workflows to import mappings and provides required configuration options - - mapping_id - link to importer_mapping that this provides a workflow for - name - name that will be displayed on front end - bean - must match @Named("") annotation on Workflow class - position - for ordering records explicitly, wanted for front end default option and order - */ -CREATE TABLE importer_mapping_workflow -( - like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, - mapping_id UUID NOT NULL, - name TEXT NOT NULL, - bean TEXT NOT NULL, - position INTEGER NOT NULL -); - -ALTER TABLE importer_mapping_workflow - ADD FOREIGN KEY (mapping_id) REFERENCES importer_mapping (id); - -DO -$$ -DECLARE - exp_mapping_id UUID; -BEGIN - exp_mapping_id := (SELECT id FROM importer_mapping WHERE name = 'ExperimentsTemplateMap'); - -INSERT INTO public.importer_mapping_workflow (mapping_id, name, bean, position) -VALUES - (exp_mapping_id, 'Create new experiment', 'CreateNewExperimentWorkflow', 0), - (exp_mapping_id, 'Append experimental dataset', 'AppendOverwritePhenotypesWorkflow', 1), - (exp_mapping_id, 'Create new experimental environment', 'CreateNewEnvironmentWorkflow', 2); -END -$$; \ No newline at end of file From 39fb8cd9ccbddf39f8c759e6654eb7d84364a441 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:03:55 -0400 Subject: [PATCH 39/55] Removed append environment workflow --- .../ExperimentWorkflowNavigator.java | 3 +- .../CreateNewEnvironmentWorkflow.java | 54 ------------------- 2 files changed, 1 insertion(+), 56 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index a498a14b2..9b47c935d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -57,8 +57,7 @@ public List getWorkflows() throws Exception { public enum Workflow { NEW_OBSERVATION("new-experiment","Create new experiment"), - APPEND_OVERWRITE("append-dataset", "Append experimental dataset"), - APPEND_ENVIRONMENT("append-environment", "Create new experimental environment"); + APPEND_OVERWRITE("append-dataset", "Append experimental dataset"); private String id; private String name; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java deleted file mode 100644 index fdb2a9fcc..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/newenv/workflow/CreateNewEnvironmentWorkflow.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.breedinginsight.brapps.importer.services.processors.experiment.newenv.workflow; - -import lombok.Getter; -import org.breedinginsight.brapps.importer.model.imports.ImportServiceContext; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; -import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; -import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; - -import javax.inject.Singleton; -import java.util.Optional; - -@Getter -@Singleton -public class CreateNewEnvironmentWorkflow implements ExperimentWorkflow { - private final ExperimentWorkflowNavigator.Workflow workflow; - - public CreateNewEnvironmentWorkflow(){ - this.workflow = ExperimentWorkflowNavigator.Workflow.APPEND_ENVIRONMENT; - } - - public Optional process(ImportServiceContext context) { - // Workflow processing the context - ImportWorkflow workflow = ImportWorkflow.builder() - .id(getWorkflow().getId()) - .name(getWorkflow().getName()) - .build(); - - // No-preview result - Optional result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.empty()) - .build()); - - // Skip this workflow unless appending a new environment - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { - return Optional.empty(); - } - - // Skip processing if no context, but return no-preview result for this workflow - if (context == null) { - return result; - } - - // Start processing the import... - return result; - } - - @Override - public int getOrder() { - return 3; - } - -} From be8ea3844b378a1b4a9e71b9b5797ebe020c57ed Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:36:43 -0400 Subject: [PATCH 40/55] Fix error message --- .../create/workflow/CreateNewExperimentWorkflow.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index b3636a38e..29d7613f1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -101,9 +101,7 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio // Make sure the file does not contain obs unit ids before proceeding if (containsObsUnitIDs(context)) { - // TODO: get file name - throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Error detected in file, " + - upload.getUploadFileName() + ". ObsUnitIDs are detected. Import cannot proceed"); + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "ObsUnitIDs are detected"); } statusService.updateMessage(upload, "Checking existing experiment objects in brapi service and mapping data"); From d11ef3c63cc8c42c7860147be8a2767b317e96f3 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:42:13 -0400 Subject: [PATCH 41/55] Removed navigator stuff handled in domain import service --- .../experimentObservation/ExperimentImportService.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 013d55012..a9b6c62ac 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.DomainImportService; -import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentWorkflowNavigator; @@ -27,14 +26,12 @@ import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; -import java.util.List; @Singleton @Slf4j public class ExperimentImportService extends DomainImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; - private final ExperimentWorkflowNavigator workflowNavigator; // TODO: delete processor fields once WorkflowNavigator is used @Inject @@ -43,7 +40,6 @@ public ExperimentImportService(Provider experimentProcessor ExperimentWorkflowNavigator workflowNavigator) { super(experimentProcessorProvider, processorManagerProvider, workflowNavigator); - this.workflowNavigator = workflowNavigator; } @Override @@ -56,9 +52,5 @@ public String getImportTypeId() { return IMPORT_TYPE_ID; } - @Override - public List getWorkflows() throws Exception{ - return workflowNavigator.getWorkflows(); - } } From eebb4938883656a5f6cb46672d28939cf584d662 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 21 Jun 2024 16:48:16 -0400 Subject: [PATCH 42/55] Removed some development comments --- .../brapps/importer/model/workflow/ImportContext.java | 2 -- .../brapps/importer/model/workflow/ProcessedData.java | 3 --- 2 files changed, 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java index 53e3a2389..a7b6f7dc3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportContext.java @@ -40,8 +40,6 @@ public class ImportContext { private UUID workflowId; private ImportUpload upload; private List importRows; - // TODO: move this out potentially - //private Map mappedBrAPIImport; private Table data; private Program program; private User user; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index 677f35469..ac3c5ed61 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -28,9 +28,6 @@ @ToString @NoArgsConstructor public class ProcessedData { - // TODO: remove this, already in ImportPreviewResponse - //private Map statistics; - // TODO private Map mappedBrAPIImport; private ImportPreviewResponse importPreviewResponse; } \ No newline at end of file From 1225ab486a2b671ce539771ab729da2dbdf4f1ed Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 26 Jun 2024 16:33:33 -0400 Subject: [PATCH 43/55] Added db migration to change UNK breeding method class to Increase --- ....23.0__change_unk_breeding_method_class.sql | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/main/resources/db/migration/V1.23.0__change_unk_breeding_method_class.sql diff --git a/src/main/resources/db/migration/V1.23.0__change_unk_breeding_method_class.sql b/src/main/resources/db/migration/V1.23.0__change_unk_breeding_method_class.sql new file mode 100644 index 000000000..b4702d31b --- /dev/null +++ b/src/main/resources/db/migration/V1.23.0__change_unk_breeding_method_class.sql @@ -0,0 +1,18 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +update breeding_method set category = 'Increase' where abbreviation = 'UNK'; \ No newline at end of file From d30d04fe464ab5fd75681d39097a5967c3d26be4 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 26 Jun 2024 17:10:23 -0400 Subject: [PATCH 44/55] Removed trailing whitespace on Unknown breeding method --- .../db/migration/V1.23.0__change_unk_breeding_method_class.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V1.23.0__change_unk_breeding_method_class.sql b/src/main/resources/db/migration/V1.23.0__change_unk_breeding_method_class.sql index b4702d31b..7cd4be61e 100644 --- a/src/main/resources/db/migration/V1.23.0__change_unk_breeding_method_class.sql +++ b/src/main/resources/db/migration/V1.23.0__change_unk_breeding_method_class.sql @@ -15,4 +15,4 @@ * limitations under the License. */ -update breeding_method set category = 'Increase' where abbreviation = 'UNK'; \ No newline at end of file +update breeding_method set name = 'Unknown', category = 'Increase' where abbreviation = 'UNK'; \ No newline at end of file From b5dc3e86ae91d9947c30974f227cb5c3eb2a3175 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 09:49:27 -0400 Subject: [PATCH 45/55] Reference enum for new-experiment --- .../brapps/importer/model/imports/DomainImportService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index d31f0a8d0..9c7a3f261 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.model.workflow.ExperimentWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflow; import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import org.breedinginsight.brapps.importer.model.workflow.Workflow; @@ -71,7 +72,7 @@ public ImportPreviewResponse process(ImportServiceContext context) // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); - if ("new-experiment".equals(context.getWorkflow())) { + if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflow())) { return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); } else { return processorManagerProvider.get().process(context.getBrAPIImports(), From 2bbf7f3158795c268dcbd2279ed90743f8223978 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:13:42 -0400 Subject: [PATCH 46/55] Change variable name to reflect string type Co-authored-by: mlm483 <128052931+mlm483@users.noreply.github.com> --- .../brapps/importer/model/imports/DomainImportService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 9c7a3f261..3d956743a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -67,8 +67,8 @@ public ImportPreviewResponse process(ImportServiceContext context) throws Exception { Optional.ofNullable(context.getWorkflow()) - .filter(workflow -> !workflow.isEmpty()) - .ifPresent(workflow -> log.info("Workflow: " + workflow)); + .filter(workflowName -> !workflowName.isEmpty()) + .ifPresent(workflowName -> log.info("Workflow: " + workflowName)); // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); From ccc141472ccf40930141e2454df089cf55f9be0c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:20:40 -0400 Subject: [PATCH 47/55] Clarified comment --- .../experiment/create/model/ProcessedPhenotypeData.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java index cd6842476..c81e265cd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/model/ProcessedPhenotypeData.java @@ -23,7 +23,8 @@ import java.util.List; import java.util.Map; -// TODO: move to common higher level location +// TODO: move to common higher level location, could be used by both append and create workflows so being located +// in the create namespace won't make sense if we decide to do that in the future. @Getter @Setter @Builder From aa06c6e64c5b651757bbc5b7631d23c1f113b781 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:25:39 -0400 Subject: [PATCH 48/55] Removed old comment --- .../brapps/importer/services/FileImportService.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index da5a68e10..0c2a9309d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -576,11 +576,6 @@ public List getWorkflowsForSystemMapping(UUID mappingId) throws .orElseThrow(() -> new DoesNotExistException("Cannot find mapping config associated with upload.")); BrAPIImportService importService = configManager.getImportServiceById(mappingConfig.getImportTypeId()) .orElseThrow(() -> new DoesNotExistException("Config with that id does not exist")); - // NOTE: - // this is creating a workflow navigator to call getWorkflows - // workflowNavigator.getWorkflows(); - // getWorkflows is creating the ExperimentWorkflow which is injecting the file import service and - // resulting in a circular dependency return importService.getWorkflows(); } From dd6f34862c92bd90758cb8bbad839dac8c484205 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 10:32:54 -0400 Subject: [PATCH 49/55] Updated doc comment --- .../create/workflow/CreateNewExperimentWorkflow.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index 29d7613f1..f007af675 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -139,9 +139,12 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio } /** - * Retrieves the name of the workflow. This is used for logging display purposes. + * Process the import service context and returns an Optional ImportWorkflowResult. * - * @return the name of the workflow + * @param context The import service context to be processed. If null, then it skips processing but returns the result with no-preview. + * @return An Optional ImportWorkflowResult which contains the workflow and import preview response (if available). + * If the context is null, it returns the result with no-preview. + * @throws Exception If any error occurs during the processing. */ public Optional process(ImportServiceContext context) throws Exception { // Workflow processing the context From a6c35902c0bdbd3733a809f72d23a37a627bcfd0 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 1 Jul 2024 15:44:41 -0400 Subject: [PATCH 50/55] Cleaned up ProcessedData code --- .../model/workflow/ProcessedData.java | 1 - .../workflow/CreateNewExperimentWorkflow.java | 3 ++- .../PopulateNewPendingImportObjectsStep.java | 21 ++++++++++--------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java index ac3c5ed61..fc29774f0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ProcessedData.java @@ -29,5 +29,4 @@ @NoArgsConstructor public class ProcessedData { private Map mappedBrAPIImport; - private ImportPreviewResponse importPreviewResponse; } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index f007af675..d776e4917 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -98,6 +98,7 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio ImportUpload upload = context.getUpload(); boolean commit = context.isCommit(); List importRows = context.getImportRows(); + ProcessedData processedData = new ProcessedData(); // Make sure the file does not contain obs unit ids before proceeding if (containsObsUnitIDs(context)) { @@ -108,7 +109,7 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio ProcessedPhenotypeData phenotypeData = experimentPhenotypeService.extractPhenotypes(context); ProcessContext processContext = populateExistingPendingImportObjectsStep.process(context, phenotypeData); - ProcessedData processedData = populateNewPendingImportObjectsStep.process(processContext, phenotypeData); + populateNewPendingImportObjectsStep.process(processContext, phenotypeData); ValidationErrors validationErrors = validatePendingImportObjectsStep.process(context, processContext.getPendingData(), phenotypeData, processedData); // short circuit if there were validation errors diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index ddb4193ab..f20c8a1a1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -91,18 +91,19 @@ public PopulateNewPendingImportObjectsStep(ExperimentSeasonService experimentSea this.gson = new JSON().getGson(); } - public ProcessedData process(ProcessContext context, ProcessedPhenotypeData phenotypeData) + /** + * TODO: in the future returning ProcessedData rather than modifying in-place would be preferrable. + * + * @param context (modified in-place) + * @param phenotypeData + * @return + * @throws MissingRequiredInfoException + * @throws UnprocessableEntityException + * @throws ApiException + */ + public void process(ProcessContext context, ProcessedPhenotypeData phenotypeData) throws MissingRequiredInfoException, UnprocessableEntityException, ApiException { - - Table data = context.getImportContext().getData(); - ImportUpload upload = context.getImportContext().getUpload(); - ImportContext importContext = context.getImportContext(); - populatePendingImportObjects(context, phenotypeData); - - - // TODO: implement - return new ProcessedData(); } From 1a6de32dc4422765faf51876e4d151898734f542 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 2 Jul 2024 10:18:49 -0400 Subject: [PATCH 51/55] Changed from workflow to workflowId --- .../importer/controllers/UploadController.java | 12 ++++++------ .../importer/model/imports/DomainImportService.java | 8 ++++---- .../importer/model/imports/ImportServiceContext.java | 2 +- .../brapps/importer/services/FileImportService.java | 4 ++-- .../workflow/AppendOverwritePhenotypesWorkflow.java | 2 +- .../create/workflow/CreateNewExperimentWorkflow.java | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java index 5ee6eb062..eb2e2ec50 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java +++ b/src/main/java/org/breedinginsight/brapps/importer/controllers/UploadController.java @@ -158,15 +158,15 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/preview") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/preview") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> previewData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable String workflow, @PathVariable UUID uploadId) { + @PathVariable String workflowId, @PathVariable UUID uploadId) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, null, false); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, null, false); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { @@ -184,16 +184,16 @@ public HttpResponse> previewData(@PathVariable UUID pro } } - @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflow}/data/{uploadId}/commit") + @Put("programs/{programId}/import/mappings/{mappingId}/workflows/{workflowId}/data/{uploadId}/commit") @Produces(MediaType.APPLICATION_JSON) @AddMetadata @ProgramSecured(roles = {ProgramSecuredRole.BREEDER, ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> commitData(@PathVariable UUID programId, @PathVariable UUID mappingId, - @PathVariable String workflow, @PathVariable UUID uploadId, + @PathVariable String workflowId, @PathVariable UUID uploadId, @Body @Nullable Map userInput) { try { AuthenticatedUser actingUser = securityService.getUser(); - ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflow, actingUser, userInput, true); + ImportResponse result = fileImportService.updateUpload(programId, uploadId, workflowId, actingUser, userInput, true); Response response = new Response(result); return HttpResponse.ok(response).status(HttpStatus.ACCEPTED); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 3d956743a..4da8efb21 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -66,13 +66,13 @@ public List getWorkflows() throws Exception{ public ImportPreviewResponse process(ImportServiceContext context) throws Exception { - Optional.ofNullable(context.getWorkflow()) - .filter(workflowName -> !workflowName.isEmpty()) - .ifPresent(workflowName -> log.info("Workflow: " + workflowName)); + Optional.ofNullable(context.getWorkflowId()) + .filter(workflowId -> !workflowId.isEmpty()) + .ifPresent(workflowId -> log.info("Workflow: " + workflowId)); // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); - if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflow())) { + if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflowId())) { return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); } else { return processorManagerProvider.get().process(context.getBrAPIImports(), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java index 4d052cd33..90e8915f9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ImportServiceContext.java @@ -32,7 +32,7 @@ @AllArgsConstructor @NoArgsConstructor public class ImportServiceContext { - private String workflow; + private String workflowId; private List brAPIImports; private Table data; private Program program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 0c2a9309d..e24cfeef2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -420,14 +420,14 @@ public ImportUpload setDynamicColumns(ImportUpload newUpload, Table data, Import return newUpload; } - private void processFile(String workflow, List finalBrAPIImportList, Table data, Program program, + private void processFile(String workflowId, List finalBrAPIImportList, Table data, Program program, ImportUpload upload, User user, Boolean commit, BrAPIImportService importService, AuthenticatedUser actingUser) { // Spin off new process for processing the file CompletableFuture.supplyAsync(() -> { try { ImportServiceContext context = ImportServiceContext.builder() - .workflow(workflow) + .workflowId(workflowId) .brAPIImports(finalBrAPIImportList) .data(data) .program(program) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java index 55c34e88a..ea5c388cb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/append/workflow/AppendOverwritePhenotypesWorkflow.java @@ -33,7 +33,7 @@ public Optional process(ImportServiceContext context) { .build()); // Skip this workflow unless appending or overwriting observation data - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { return Optional.empty(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index d776e4917..ed95014a6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -163,7 +163,7 @@ public Optional process(ImportServiceContext context) thro .build()); // Skip this workflow unless creating a new experiment - if (context != null && !this.workflow.isEqual(context.getWorkflow())) { + if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { return Optional.empty(); } From 281cc6c7dabff7a444484f381fbec7161e176ea5 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 2 Jul 2024 15:31:32 +0000 Subject: [PATCH 52/55] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 9eb5a3f90..96d688433 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+752 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/7fd32a0140ea299da63eaa3d3b14bf75725a0086 +version=v0.10.0+754 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/4e9e1b800d5f90e3e13e3dfdf318fe25bbbfb813 From 28a22729f33ead44785bf09af99a22a4c59df0ae Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 26 Jun 2024 14:28:48 -0400 Subject: [PATCH 53/55] Swapped lat and lon as they were reversed --- .../imports/experimentObservation/ExperimentObservation.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 579c34a31..d69de11dd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -321,11 +321,11 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( try { double lat = Double.parseDouble(getLatitude()); double lon = Double.parseDouble(getLongitude()); - Point geoPoint = Point.from(lat, lon); + Point geoPoint = Point.from(lon, lat); if (getElevation() != null) { double elevation = Double.parseDouble(getElevation()); - geoPoint = Point.from(lat, lon, elevation); // geoPoint.withAlt(elevation) did not work + geoPoint = Point.from(lon, lat, elevation); // geoPoint.withAlt(elevation) did not work } BrApiGeoJSON coords = BrApiGeoJSON.builder() From 48a8817cdc23d496c805453aa54f2a5e85578bb9 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 2 Jul 2024 17:50:16 +0000 Subject: [PATCH 54/55] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 96d688433..4fafa232e 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.10.0+754 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/4e9e1b800d5f90e3e13e3dfdf318fe25bbbfb813 +version=v0.10.0+756 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a960d556180bd80ec3ca543555cee1b912fad236 From a488612e506cd767fa7c5acdc69f57b6ad35228b Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:20:07 -0400 Subject: [PATCH 55/55] Change exception handling --- .../model/imports/DomainImportService.java | 10 ++- .../model/workflow/ImportWorkflowResult.java | 1 + .../importer/model/workflow/Workflow.java | 4 +- .../ExperimentWorkflowNavigator.java | 63 ++++++++++--------- .../workflow/CreateNewExperimentWorkflow.java | 27 ++++---- 5 files changed, 60 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java index 4da8efb21..6cfffe73c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DomainImportService.java @@ -73,7 +73,15 @@ public ImportPreviewResponse process(ImportServiceContext context) // TODO: return results from WorkflowNavigator once processing logic is in separate workflows // return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); if (ExperimentWorkflowNavigator.Workflow.NEW_OBSERVATION.getId().equals(context.getWorkflowId())) { - return workflowNavigator.process(context).flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + Optional result = workflowNavigator.process(context); + + // Throw any exceptions caught during workflow processing + if (result.flatMap(ImportWorkflowResult::getCaughtException).isPresent()) { + throw result.flatMap(ImportWorkflowResult::getCaughtException).get(); + } + + return result.flatMap(ImportWorkflowResult::getImportPreviewResponse).orElse(null); + } else { return processorManagerProvider.get().process(context.getBrAPIImports(), List.of(experimentProcessorProvider.get()), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java index 9304c1ce2..f59b83b55 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/ImportWorkflowResult.java @@ -30,4 +30,5 @@ public class ImportWorkflowResult { private ImportWorkflow workflow; private Optional importPreviewResponse; + private Optional caughtException; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java index dec774e54..17f48ee57 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/workflow/Workflow.java @@ -9,8 +9,8 @@ @FunctionalInterface public interface Workflow extends Ordered { - Optional process(ImportServiceContext context) throws Exception; - default List getWorkflows() throws Exception { + Optional process(ImportServiceContext context); + default List getWorkflows() { // Default implementation for getWorkflows method return new ArrayList<>(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java index 9b47c935d..6acd4aad5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/ExperimentWorkflowNavigator.java @@ -7,9 +7,9 @@ import org.breedinginsight.brapps.importer.model.workflow.ImportWorkflowResult; import javax.inject.Singleton; -import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; @Primary @Singleton @@ -20,39 +20,46 @@ public ExperimentWorkflowNavigator(List workflows) { this.workflows = workflows; } + /** + * Process the import service context by executing a series of workflows in order + * + * This method iterates over the list of workflows provided, executing each workflow's process method + * with the given import service context. It then filters out empty results and returns the first non-empty result. + * + * @param context The import service context containing the data to be processed + * @return An Optional containing the first non-empty ImportWorkflowResult from the executed workflows, or an empty Optional if no non-empty result is found + */ @Override - public Optional process(ImportServiceContext context) throws Exception { - - // NOTE: Couldn't throw exception from map lambda - for (ExperimentWorkflow workflow : workflows) { - Optional result = workflow.process(context); - if (result.isPresent()) { - return result; - } - } - - return Optional.empty(); + public Optional process(ImportServiceContext context) { + /** + * Have each workflow in order process the context, returning the first non-empty result + */ + return workflows.stream() + .map(workflow->workflow.process(context)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); } - @Override - public List getWorkflows() throws Exception { - // Each workflow returns in the field workflow the metadata about the workflow that processed the import context. - // Loop over all workflows, processing a null context, to collect just the metadata - - // NOTE: Couldn't throw exception from map lambda - List workflowSummaryList = new ArrayList<>(); - - for (ExperimentWorkflow workflow : workflows) { - Optional result = workflow.process(null); - result.ifPresent(importWorkflowResult -> workflowSummaryList.add(importWorkflowResult.getWorkflow())); - } - - // The order field for each workflow is set to the order in the list + /** + * Retrieves a list of ImportWorkflow objects containing metadata about each workflow that processed the import context. + * + * @return List of ImportWorkflow objects with workflow metadata + */ + public List getWorkflows() { + List workflowSummaryList = workflows.stream() + .map(workflow -> workflow.process(null)) // Process each workflow with a null context + .filter(Optional::isPresent) // Filter out any workflows that do not return a result + .map(Optional::get) // Extract the result from Optional + .map(result -> result.getWorkflow()) // Retrieve the workflow metadata + .collect(Collectors.toList()); // Collect the workflow metadata into a list + + // Set the order field for each workflow based on its position in the list for (int i = 0; i < workflowSummaryList.size(); i++) { - workflowSummaryList.get(i).setOrder(i); + workflowSummaryList.get(i).setOrder(i); // Set the order for each workflow } - return workflowSummaryList; + return workflowSummaryList; // Return the list of workflow metadata } public enum Workflow { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java index ed95014a6..d0a4ca975 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/CreateNewExperimentWorkflow.java @@ -145,9 +145,8 @@ private ImportPreviewResponse runWorkflow(ImportContext context) throws Exceptio * @param context The import service context to be processed. If null, then it skips processing but returns the result with no-preview. * @return An Optional ImportWorkflowResult which contains the workflow and import preview response (if available). * If the context is null, it returns the result with no-preview. - * @throws Exception If any error occurs during the processing. */ - public Optional process(ImportServiceContext context) throws Exception { + public Optional process(ImportServiceContext context) { // Workflow processing the context ImportWorkflow workflow = ImportWorkflow.builder() .id(getWorkflow().getId()) @@ -155,12 +154,11 @@ public Optional process(ImportServiceContext context) thro .build(); // No-preview result - Optional result; - - result = Optional.of(ImportWorkflowResult.builder() + ImportWorkflowResult workflowResult = ImportWorkflowResult.builder() .workflow(workflow) .importPreviewResponse(Optional.empty()) - .build()); + .caughtException(Optional.empty()) + .build(); // Skip this workflow unless creating a new experiment if (context != null && !this.workflow.isEqual(context.getWorkflowId())) { @@ -169,21 +167,22 @@ public Optional process(ImportServiceContext context) thro // Skip processing if no context, but return no-preview result for this workflow if (context == null) { - return result; + return Optional.of(workflowResult); } // TODO: unify usage of single import context type throughout ImportContext importContext = ImportContext.from(context); // Start processing the import... - ImportPreviewResponse response = runWorkflow(importContext); - - result = Optional.of(ImportWorkflowResult.builder() - .workflow(workflow) - .importPreviewResponse(Optional.of(response)) - .build()); + ImportPreviewResponse response; + try { + response = runWorkflow(importContext); + workflowResult.setImportPreviewResponse(Optional.of(response)); + } catch(Exception e) { + workflowResult.setCaughtException(Optional.of(e)); + } - return result; + return Optional.of(workflowResult); } @Override