From e6374e595bbcbde3beb92696cb6441c80cdda98b Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Fri, 14 Mar 2025 14:14:34 -0400 Subject: [PATCH 01/59] [BI-2578] Support for unpaginated RQs to BrAPI server for cache loading --- .../brapi/v2/dao/BrAPIGermplasmDAO.java | 2 +- .../utilities/BrAPIDAOUtil.java | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 75226babf..9e83615a2 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -141,7 +141,7 @@ private Map fetchProgramGermplasm(UUID programId) throws BrAPIGermplasmSearchRequest germplasmSearch = new BrAPIGermplasmSearchRequest(); germplasmSearch.externalReferenceIDs(List.of(programId.toString())); germplasmSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); - return processGermplasmForDisplay(brAPIDAOUtil.search( + return processGermplasmForDisplay(brAPIDAOUtil.searchNoPaging( api::searchGermplasmPost, api::searchGermplasmSearchResultsDbIdGet, germplasmSearch diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index b730660e3..7fb115a53 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -79,26 +79,37 @@ public List search(Funct Function3, Optional>>> searchGetMethod, U searchBody ) throws ApiException { - return searchInternal(searchMethod, searchGetMethod, null, searchBody); + return searchInternal(searchMethod, searchGetMethod, null, searchBody, true); } public List search(Function, Optional>>> searchMethod, Function4, Optional>>> searchGetMethod, U searchBody ) throws ApiException { - return searchInternal(searchMethod, null, searchGetMethod, searchBody); + return searchInternal(searchMethod, null, searchGetMethod, searchBody, true); + } + + public List searchNoPaging(Function, Optional>>> searchMethod, + Function3, Optional>>> searchGetMethod, + U searchBody + ) throws ApiException { + return searchInternal(searchMethod, searchGetMethod, null, searchBody, false); } private List searchInternal(Function, Optional>>> searchMethod, Function3, Optional>>> searchGetMethod, Function4, Optional>>> searchGetMethodWithMimeType, - U searchBody) throws ApiException { + U searchBody, boolean sendPaging) throws ApiException { try { List listResult = new ArrayList<>(); //NOTE: Because of the way Breedbase implements BrAPI searches, the page size is initially set to an //arbitrary, large value to ensure that in the event that a 202 response is returned, the searchDbId //stored will refer to all records of the BrAPI variable. - searchBody.pageSize(10000000); + + if (sendPaging) { + searchBody.pageSize(65000); + } + ApiResponse, Optional>> response = searchMethod.apply(searchBody); if (response.getBody().getLeft().isPresent()) { BrAPIResponse listResponse = (BrAPIResponse) response.getBody().getLeft().get(); @@ -108,7 +119,7 @@ private List searchInter pagination params are handled for POST search endpoints or the corresponding endpoints in Breedbase are changed or updated */ - if(hasMorePages(listResponse)) { + if(sendPaging && hasMorePages(listResponse)) { int currentPage = listResponse.getMetadata().getPagination().getCurrentPage() + 1; int totalPages = listResponse.getMetadata().getPagination().getTotalPages(); @@ -137,7 +148,7 @@ private List searchInter BrAPIResponse listResponse = (BrAPIResponse) searchGetResponse.getBody().getLeft().get(); listResult = getListResult(searchGetResponse); - if(hasMorePages(listResponse)) { + if(sendPaging && hasMorePages(listResponse)) { currentPage++; int totalPages = listResponse.getMetadata() .getPagination() From 78c699e06cb90de171595a984df97f9fbaaddcb1 Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Fri, 14 Mar 2025 14:21:45 -0400 Subject: [PATCH 02/59] [BI-2579] Forgo cache refresh on response from server during germ import chunks --- .../org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 75226babf..65a8a49c8 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -296,11 +296,9 @@ public List createBrAPIGermplasm(List postBrAPIG var program = programDAO.fetchOneById(programId); try { if (!postBrAPIGermplasmList.isEmpty()) { - Callable> postFunction = () -> { List postResponse = brAPIDAOUtil.post(postBrAPIGermplasmList, upload, api::germplasmPost, importDAO::update); - return processGermplasmForDisplay(postResponse, program.getKey()); - }; - return programGermplasmCache.post(programId, postFunction); + processGermplasmForDisplay(postResponse, program.getKey()); + return postResponse; } return new ArrayList<>(); } catch (Exception e) { From 9c0724471b8d648ddf1029661d0c69060ef27403 Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Mon, 17 Mar 2025 16:25:09 -0400 Subject: [PATCH 03/59] Add comment about configurable variable on brapi test server side --- .../java/org/breedinginsight/utilities/BrAPIDAOUtil.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 7fb115a53..3d4925414 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -102,11 +102,11 @@ private List searchInter U searchBody, boolean sendPaging) throws ApiException { try { List listResult = new ArrayList<>(); - //NOTE: Because of the way Breedbase implements BrAPI searches, the page size is initially set to an - //arbitrary, large value to ensure that in the event that a 202 response is returned, the searchDbId - //stored will refer to all records of the BrAPI variable. if (sendPaging) { + // This should be set to whatever the maximum allowable value is configured in the brapi test server, + // perhaps it should be configurable on bi side as well. + // For reference, that prop name is paging.page-size.max-allowed searchBody.pageSize(65000); } From 14c1ee3baaa293dd0c37cef6c0fde10b2eabbadd Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Wed, 19 Mar 2025 14:24:09 -0400 Subject: [PATCH 04/59] [BI-2578] Add config props for max rq page size and pagination flipper for germs --- .env.template | 4 ++++ .../brapi/v2/dao/BrAPIGermplasmDAO.java | 24 +++++++++++++++---- .../utilities/BrAPIDAOUtil.java | 5 +++- src/main/resources/application.yml | 13 ++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/.env.template b/.env.template index abe859391..1a481aae6 100644 --- a/.env.template +++ b/.env.template @@ -31,6 +31,10 @@ BRAPI_READ_TIMEOUT=60m # Max number of records to POST to the BrAPI service per request POST_CHUNK_SIZE=1000 +# Request cache records paginating through available records per program by CACHE_BRAPI_FETCH_PAGE_SIZE +CACHE_PAGINATE_GERMPLASM=false +CACHE_BRAPI_FETCH_PAGE_SIZE=65000 + # BrAPI Server Variables BRAPI_SERVER_PORT=8083 diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 9e83615a2..a8feaecf6 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -65,6 +65,9 @@ public class BrAPIGermplasmDAO { @Property(name = "micronaut.bi.api.run-scheduled-tasks") private boolean runScheduledTasks; + @Property(name = "brapi.paginate.germplasm") + private boolean paginateGermplasm; + private final ProgramCache programGermplasmCache; private final BrAPIEndpointProvider brAPIEndpointProvider; @@ -141,11 +144,22 @@ private Map fetchProgramGermplasm(UUID programId) throws BrAPIGermplasmSearchRequest germplasmSearch = new BrAPIGermplasmSearchRequest(); germplasmSearch.externalReferenceIDs(List.of(programId.toString())); germplasmSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); - return processGermplasmForDisplay(brAPIDAOUtil.searchNoPaging( - api::searchGermplasmPost, - api::searchGermplasmSearchResultsDbIdGet, - germplasmSearch - ), program.getKey()); + + if (paginateGermplasm) { + log.debug("Fetching germplasm with pagination to BrAPI"); + return processGermplasmForDisplay(brAPIDAOUtil.search( + api::searchGermplasmPost, + api::searchGermplasmSearchResultsDbIdGet, + germplasmSearch), + program.getKey()); + } else { + log.debug("Fetching germplasm without pagination to BrAPI"); + return processGermplasmForDisplay(brAPIDAOUtil.searchNoPaging( + api::searchGermplasmPost, + api::searchGermplasmSearchResultsDbIdGet, + germplasmSearch), + program.getKey()); + } } /** diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 3d4925414..15dc57236 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -62,6 +62,9 @@ public class BrAPIDAOUtil { private final int postGroupSize; private final ProgramService programService; + @Property(name = "brapi.cache.fetch-page-size") + private int brapiFetchPageSize; + @Inject public BrAPIDAOUtil(@Property(name = "brapi.search.wait-time") int searchWaitTime, @Property(name = "brapi.read-timeout") Duration searchTimeout, @@ -107,7 +110,7 @@ private List searchInter // This should be set to whatever the maximum allowable value is configured in the brapi test server, // perhaps it should be configurable on bi side as well. // For reference, that prop name is paging.page-size.max-allowed - searchBody.pageSize(65000); + searchBody.pageSize(brapiFetchPageSize); } ApiResponse, Optional>> response = searchMethod.apply(searchBody); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 3a98c3fb6..820d0cbac 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -173,6 +173,19 @@ brapi: search: wait-time: 1000 post-group-size: ${POST_CHUNK_SIZE:1000} + paginate: + # These props are used as flippers for specific cache entities to be fetched all at once or paginated. + # If the props are set to true, there's a possibility the BrAPI server could run out of memory grabbing all the + # data and assigning it to entities in memory given a program that has a large amount of those types of entities. + # At that point these flippers should be set to true. + germplasm: ${CACHE_PAGINATE_GERMPLASM:false} + cache: + # This prop sets how many program cache entity records can be fetched at a time from BrAPI. Today, if over 65000 is used, + # a SQL error can happen for programs that have more than this page size, because to fetch these certain entities + # the same amount of IDs must be passed for any collections that need to be fetched in the entity, and SQL has a max of ~65355 params. + # The ProgramCache will iterate through all pages of an entity to retrieve all the data from BrAPI. + fetch-page-size: ${CACHE_BRAPI_FETCH_PAGE_SIZE:65000} + email: relay-server: From fa32de4b89ced730771a1e5f682c1565c66ad9fe Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Thu, 20 Mar 2025 14:20:57 -0400 Subject: [PATCH 05/59] Fully populate program germplasm cache when upload completes --- .../org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java | 4 ++++ .../importer/services/processors/GermplasmProcessor.java | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 2f756ba26..c4090ef8e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -162,6 +162,10 @@ private Map fetchProgramGermplasm(UUID programId) throws } } + public void repopulateGermplasmCacheForProgram(UUID programId) { + programGermplasmCache.populate(programId); + } + /** * Process germplasm into a format for display * @param programGermplasm diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index 0d514ee72..74d6a6066 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -630,6 +630,8 @@ public void postBrapiData(Map mappedBrAPIImport, Program try { // Create germplasm list brAPIListDAO.createBrAPILists(List.of(importList), program.getId(), upload); + // Now that we have finished uploading, fetch all the data posted to BrAPI to the cache so it is up-to-date. + brAPIGermplasmDAO.repopulateGermplasmCacheForProgram(program.getId()); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); } From 51c9f7f8d295be6275dea44ed0f3d898b5f2e02d Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Mon, 31 Mar 2025 12:25:35 -0400 Subject: [PATCH 06/59] Add test server template props to bi-api --- .../resources/brapi/properties/application.properties | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/resources/brapi/properties/application.properties b/src/main/resources/brapi/properties/application.properties index 28a097f3a..3733fcd37 100644 --- a/src/main/resources/brapi/properties/application.properties +++ b/src/main/resources/brapi/properties/application.properties @@ -36,3 +36,13 @@ spring.mvc.dispatch-options-request=true security.oidc_discovery_url=https://example.com/auth/.well-known/openid-configuration security.enabled=false +security.issuer_url=http://example.com/issuerurl + +# This should either be set in accordance with a maximum number of SQL parameters (on JOIN FETCHES of collections, +# if there is more than one collection the IDs of each entity need to be passed through as parameters, and there is a SQL +# maximum of 65535. See GermplasmService.findGermplasmEntities()), +# whatever returns in a reasonable amount of time, +# or if you want to limit for the sake of server efficiency. +paging.page-size.max-allowed=65000 + +paging.page-size.default=1000 From dcfd786532103b4a8a624af81e1aa26c7764349c Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Mon, 31 Mar 2025 12:30:36 -0400 Subject: [PATCH 07/59] Add properties added to test server properties file --- .../brapi/properties/application.properties | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/resources/brapi/properties/application.properties b/src/main/resources/brapi/properties/application.properties index 3733fcd37..5ee3656c5 100644 --- a/src/main/resources/brapi/properties/application.properties +++ b/src/main/resources/brapi/properties/application.properties @@ -25,13 +25,25 @@ spring.datasource.password=${BRAPI_DB_PASSWORD} spring.datasource.driver-class-name=org.postgresql.Driver +# This property when set to true makes it so that a DB transaction is open through the body of a request, nullifying the use of @Transactional. +# It is generally recommended that this be set to false, and methods are properly annotated and release the transactions when complete. +# However, many of the endpoints already rely on this kind of connection infrastructure and changing is more trouble than it's worth. +spring.jpa.open-in-view=true + +spring.jpa.properties.hibernate.jdbc.batch_size=50 +spring.jpa.properties.hibernate.order_inserts=true +spring.jpa.properties.hibernate.order_updates=true +spring.jpa.hibernate.ddl-auto=validate + +# Use these to help debug queries. +# The stats will tell you how long hibernate transactions are taking, how many queries occur, how many entities are being flushed/accessed, etc. +spring.jpa.show-sql=false +spring.jpa.properties.hibernate.generate_statistics=false + spring.flyway.locations=classpath:db/migration,classpath:db/sql,classpath:org/brapi/test/BrAPITestServer/db/migration spring.flyway.schemas=public spring.flyway.baselineOnMigrate=true -spring.jpa.hibernate.ddl-auto=validate -spring.jpa.show-sql=false - spring.mvc.dispatch-options-request=true security.oidc_discovery_url=https://example.com/auth/.well-known/openid-configuration From 5fe02f2359a9e68ce147941563f8bf34822f34be Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Mon, 31 Mar 2025 16:24:03 -0400 Subject: [PATCH 08/59] Restore functionality of using post return values --- .../org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index c4090ef8e..478c99bcf 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -315,8 +315,7 @@ public List createBrAPIGermplasm(List postBrAPIG try { if (!postBrAPIGermplasmList.isEmpty()) { List postResponse = brAPIDAOUtil.post(postBrAPIGermplasmList, upload, api::germplasmPost, importDAO::update); - processGermplasmForDisplay(postResponse, program.getKey()); - return postResponse; + return new ArrayList<>(processGermplasmForDisplay(postResponse, program.getKey()).values()); } return new ArrayList<>(); } catch (Exception e) { From bb359fdeef3993f915672355b65e050add7562e2 Mon Sep 17 00:00:00 2001 From: jloux-brapi Date: Wed, 2 Apr 2025 16:40:43 -0400 Subject: [PATCH 09/59] Fix unit test errors related to new property not loading --- .../java/org/breedinginsight/utilities/BrAPIDAOUtil.java | 6 +++--- .../java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java | 2 +- .../geno/impl/GigwaGenotypeServiceImplIntegrationTest.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 15dc57236..cab44d951 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -60,21 +60,21 @@ public class BrAPIDAOUtil { private final Duration searchTimeout; private final int pageSize; private final int postGroupSize; + private final int brapiFetchPageSize; private final ProgramService programService; - @Property(name = "brapi.cache.fetch-page-size") - private int brapiFetchPageSize; - @Inject public BrAPIDAOUtil(@Property(name = "brapi.search.wait-time") int searchWaitTime, @Property(name = "brapi.read-timeout") Duration searchTimeout, @Property(name = "brapi.page-size") int pageSize, @Property(name = "brapi.post-group-size") int postGroupSize, + @Property(name = "brapi.cache.fetch-page-size") int brapiFetchPageSize, ProgramService programService) { this.searchWaitTime = searchWaitTime; this.searchTimeout = searchTimeout; this.pageSize = pageSize; this.postGroupSize = postGroupSize; + this.brapiFetchPageSize = brapiFetchPageSize; this.programService = programService; } diff --git a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java index 427ed6d88..d77971817 100644 --- a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java +++ b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java @@ -70,7 +70,7 @@ public ApiResponse, Optional Date: Wed, 2 Apr 2025 16:59:49 -0400 Subject: [PATCH 10/59] Fix unit test failure related to searchNoPaging function --- .../services/BrAPIGermplasmServiceUnitTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java index 8f906c169..9d5f7b792 100644 --- a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java +++ b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java @@ -141,9 +141,9 @@ public void getGermplasmListExport() { when(programDAO.getAll()).thenReturn(Arrays.asList(Program.builder().id(testProgramId).name("Test Program").active(true).build())); when(programDAO.fetchOneById(any(UUID.class))).thenReturn(testProgram); when(programDAO.get(any(UUID.class))).thenReturn(Arrays.asList(testProgram)); - when(brAPIDAOUtil.search(any(Function.class), - any(Function3.class), - any(BrAPIGermplasmSearchRequest.class))).thenReturn(germplasm); + when(brAPIDAOUtil.searchNoPaging(any(Function.class), + any(Function3.class), + any(BrAPIGermplasmSearchRequest.class))).thenReturn(germplasm); //Create germplasm cache of stub data Method setupMethod = BrAPIGermplasmDAO.class.getDeclaredMethod("setup"); From 0b51b1f7fdfbd632ed8e1a6e5490a76a3b10bb03 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 9 Apr 2025 15:35:55 +0000 Subject: [PATCH 11/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index ec1e82717..5b40e8aee 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+920 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/83f392e18d60bba157751d18ae72699b68824f3a +version=v1.1.0+926 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/8610b9a5849e0ce306e07d30abc3fbc1ff0aeaac From 32ac1b2c0c35d34606919b3a11fd78b84958ba5e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 9 Apr 2025 20:42:45 +0000 Subject: [PATCH 12/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 80262f90f..40f2de21e 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,6 +14,6 @@ # limitations under the License. # -version=v1.1.0+926 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/8610b9a5849e0ce306e07d30abc3fbc1ff0aeaac +version=v1.1.0+930 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/aeb24ccc78479a5ee665c50b7c4531a7698e1603 From 9b5479442f6026641ed9c8161cb0d7e09a90e3ac Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 16 Apr 2025 19:12:11 +0000 Subject: [PATCH 13/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 40f2de21e..c09bf4250 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,6 +14,6 @@ # limitations under the License. # -version=v1.1.0+930 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/aeb24ccc78479a5ee665c50b7c4531a7698e1603 +version=v1.1.0+936 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5159c9d5cdaf3adb87a8e24fe32702e4cee36cf7 From 5c969e3efd7b474811211568434ba88349902d43 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 24 Apr 2025 19:49:25 +0000 Subject: [PATCH 14/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index c09bf4250..25283e3c6 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,6 +14,6 @@ # limitations under the License. # -version=v1.1.0+936 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5159c9d5cdaf3adb87a8e24fe32702e4cee36cf7 +version=v1.1.0+946 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f4ff8b366966fd4ccdc98f744709b32cb3176d26 From c6d599d8ea6cdd3f75f60f329715f82cbe08b6f5 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 30 Apr 2025 16:07:32 -0400 Subject: [PATCH 15/59] Update src/test/java/org/breedinginsight/BrAPITest.java --- src/test/java/org/breedinginsight/BrAPITest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index e6e6072d6..8c5c35bad 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -49,7 +49,7 @@ public class BrAPITest extends DatabaseTest { public BrAPITest() { super(); - brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server:rc") + brapiContainer = new GenericContainer<>("breedinginsight/brapi-java-server:develop") .withNetwork(super.getNetwork()) .withImagePullPolicy(PullPolicy.ageBased(Duration.ofMinutes(60))) .withExposedPorts(8080) From 74201d78bc1d0670fc47d37a9a3b810e5e4489d1 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 30 Apr 2025 20:07:46 +0000 Subject: [PATCH 16/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 3e26e3f83..c67efc6c8 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+948 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d50dc656df8d551465d34fed891b89709dcb365c +version=v1.1.0+955 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c6d599d8ea6cdd3f75f60f329715f82cbe08b6f5 From b9b0cfe507a150476d7783e8cf7267f00dd7c416 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 30 Apr 2025 20:10:35 +0000 Subject: [PATCH 17/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index c67efc6c8..8267797c6 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+955 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c6d599d8ea6cdd3f75f60f329715f82cbe08b6f5 +version=v1.1.0+957 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6904ba563f374f7791fe93689eb324d1d55e9a5f From c66193ae96e86b4edafa4b117b101e1b2e1a7ed8 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 30 Apr 2025 20:11:31 +0000 Subject: [PATCH 18/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 8267797c6..5dda72802 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+957 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6904ba563f374f7791fe93689eb324d1d55e9a5f +version=v1.1.0+959 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/738dee1bea47d07b5e24a37304dd83e6d3f29b65 From 1b451787ee6997cae36d110ae2a0dd74125b5620 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 19 May 2025 18:38:13 +0000 Subject: [PATCH 19/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5dda72802..8ec9f3000 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+959 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/738dee1bea47d07b5e24a37304dd83e6d3f29b65 +version=v1.1.0+965 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c66193ae96e86b4edafa4b117b101e1b2e1a7ed8 From bcd7f16800bb6fcdf51a893f41222313478a3c7c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 19 May 2025 15:32:46 -0400 Subject: [PATCH 20/59] Update version number --- src/main/resources/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 8ec9f3000..f9fc4d945 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+965 +version=v1.1.1+965 versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c66193ae96e86b4edafa4b117b101e1b2e1a7ed8 From 0a9f8baaceb8572fec2329c97d01f63a2e7e7a76 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 19 May 2025 19:33:04 +0000 Subject: [PATCH 21/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index f9fc4d945..9eebddfde 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+965 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c66193ae96e86b4edafa4b117b101e1b2e1a7ed8 +version=v1.1.1+969 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/bcd7f16800bb6fcdf51a893f41222313478a3c7c From da9891911b9b75a1b2f13340036180ae342ad976 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 29 May 2025 12:10:18 -0400 Subject: [PATCH 22/59] [BI-2654] - replaced reserved keyword with allowed value --- src/test/resources/sql/UserControllerIntegrationTest.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/resources/sql/UserControllerIntegrationTest.sql b/src/test/resources/sql/UserControllerIntegrationTest.sql index 55cafed8d..5dce6cca0 100644 --- a/src/test/resources/sql/UserControllerIntegrationTest.sql +++ b/src/test/resources/sql/UserControllerIntegrationTest.sql @@ -27,19 +27,19 @@ join bi_user on bi_user.name = 'system' limit 1; -- name: InsertUserProgramAssociations insert into program_user_role (program_id, user_id, role_id, created_by, updated_by) -select program.id, bi_user.id, role.id, system_user.id, system_user.id +select program.id, bi_user.id, role.id, bi_system_user.id, bi_system_user.id from program join bi_user on bi_user.name = 'Test User' or bi_user.name = 'Other Test User' join role on role.domain = 'Read Only' -join bi_user as system_user on system_user.name = 'system' +join bi_user as bi_system_user on bi_system_user.name = 'system' where program.name = 'Test Program'; insert into program_user_role (program_id, user_id, role_id, active, created_by, updated_by) -select program.id, bi_user.id, role.id, false, system_user.id, system_user.id +select program.id, bi_user.id, role.id, false, bi_system_user.id, bi_system_user.id from program join bi_user on bi_user.name = 'Test User' or bi_user.name = 'Other Test User' join role on role.domain = 'Read Only' -join bi_user as system_user on system_user.name = 'system' +join bi_user as bi_system_user on bi_system_user.name = 'system' where program.name = 'Test Program1'; -- name: DeactivateProgram From fc3612964f293fe2340be60f926ee9dfa883108d Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 29 May 2025 16:45:05 -0400 Subject: [PATCH 23/59] Increased signup token expiration from 1hr to 24hr --- src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 820d0cbac..313540dbd 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -145,7 +145,7 @@ web: logout: url: ${web.base-url} signup: - url-timeout: 60m + url-timeout: 24h signup: url: ${web.base-url}/signup success: From a3b80c749222b1f8da3fe8a5ef9b4a9e7562f23a Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 30 May 2025 14:39:54 +0000 Subject: [PATCH 24/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 9eebddfde..d450b6379 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+969 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/bcd7f16800bb6fcdf51a893f41222313478a3c7c +version=v1.1.1+979 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a290899ba43d9e0c42730a3c0e6170b416ee70e8 From 1842f026418d4e8fcfa3f2ce6fa905c5f6b22529 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 3 Jun 2025 17:10:40 -0400 Subject: [PATCH 25/59] Update species migration to work with new uuids --- src/main/resources/brapi/sql/R__species.sql | 64 +++++++++++---------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index 00c1b7042..261a18a06 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -13,32 +13,38 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '4', 'Blueberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '5', 'Salmon') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '6', 'Grape') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '7', 'Alfalfa') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '8', 'Sweet Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '9', 'Trout') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '10', 'Soybean') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '11', 'Cranberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '12', 'Cucumber') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'Oat') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; --- for the Honey Bee case, want to overwrite name, not preserve existing -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '21', 'Sorghum') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '22', 'Hemp') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '23', 'Hop') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '24', 'Hydrangea') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '25', 'Red Clover') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '26', 'Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '27', 'Blackberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '28', 'Raspberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '29', 'Sugar Beet') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '30', 'Strawberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '31', 'Coffee') ON CONFLICT DO NOTHING; \ No newline at end of file +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +DO $$ +DECLARE +v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; +BEGIN + /* ------------------------------------------------------------------------------------------ + • uuid_generate_v5(namespace, crop_name) → deterministic UUID can be used for idempotency + • Do it this way so no schema changes are required + • Removed the Honey Bee special case because all systems will be starting fresh + ------------------------------------------------------------------------------------------ */ +INSERT INTO crop (id, auth_user_id, crop_name) +SELECT + uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, + -- 1) lower‑case + -- 2) trim leading/trailing space + -- 3) REMOVE every space or tab + regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), + v_auth_id, + crop_name +FROM (VALUES + ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), + ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), + ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), + ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), + ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), + ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), + ('Sugar Beet'), ('Coffee') + ) AS src(crop_name) + ON CONFLICT (id) DO +-- want case changes or space changes to overwrite existing +-- Only rewrite the row if name changed +UPDATE SET crop_name = EXCLUDED.crop_name +WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; +END $$; \ No newline at end of file From f97f6d41c705685fb13bae41ce60d5666805f4da Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 3 Jun 2025 21:10:59 +0000 Subject: [PATCH 26/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index d450b6379..dc9397197 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+979 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a290899ba43d9e0c42730a3c0e6170b416ee70e8 +version=v1.1.1+985 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1842f026418d4e8fcfa3f2ce6fa905c5f6b22529 From 7a6be40f95e7e534883fb6a4083795b03b013283 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 4 Jun 2025 08:44:58 -0400 Subject: [PATCH 27/59] Update species sql for tests --- src/test/resources/sql/brapi/species.sql | 44 ++++++++++++++---------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index 817a1c726..f62e69156 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -17,22 +17,28 @@ */ -- name: InsertSpecies -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '4', 'Blueberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '5', 'Salmon') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '6', 'Grape') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '7', 'Alfalfa') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '8', 'Sweet Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '9', 'Trout') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '10', 'Soybean') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '11', 'Cranberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '12', 'Cucumber') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'Oat') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '21', 'Sorghum') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '22', 'Hemp') ON CONFLICT DO NOTHING; +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +DO $$ +DECLARE +v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; +BEGIN +INSERT INTO crop (id, auth_user_id, crop_name) +SELECT + uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, + regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), + v_auth_id, + crop_name +FROM (VALUES + ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), + ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), + ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), + ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), + ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), + ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), + ('Sugar Beet'), ('Coffee') + ) AS src(crop_name) + ON CONFLICT (id) DO +UPDATE SET crop_name = EXCLUDED.crop_name +WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; +END $$; \ No newline at end of file From dbec72a2e250b0f1887e7ad0f52314dfd1d9639c Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 4 Jun 2025 12:46:02 +0000 Subject: [PATCH 28/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index dc9397197..cc086f667 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+985 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1842f026418d4e8fcfa3f2ce6fa905c5f6b22529 +version=v1.1.1+987 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/e916fc3980e50062760e8af977e885ef58c3a93e From 7129705ffa33984ae862366f0b00b3cb81b73843 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 4 Jun 2025 09:18:24 -0400 Subject: [PATCH 29/59] Add security issuer env for tests --- src/test/java/org/breedinginsight/BrAPITest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index 8c5c35bad..bdf362979 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -60,6 +60,7 @@ public BrAPITest() { .withEnv("BRAPI_DB", "postgres") .withEnv("BRAPI_DB_USER", "postgres") .withEnv("BRAPI_DB_PASSWORD", "postgres") + .withEnv("SECURITY_ISSUER_URL", "http://example.com/issuerurl") .withClasspathResourceMapping("sql/brapi/mount", "/home/brapi/db/sql", BindMode.READ_WRITE) // HACK - READ_WRITE forces testcontainers to use a bind mount (which overwrites) instead of copying files. .withClasspathResourceMapping("brapi/properties/application.properties", "/home/brapi/properties/application.properties", BindMode.READ_ONLY) .waitingFor(Wait.forLogMessage(".*Started BrapiTestServer in \\d*.\\d* seconds.*", 1).withStartupTimeout(Duration.ofMinutes(1))); From e05bad527b5c4b727925203abc064139ef2857c9 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 4 Jun 2025 13:19:01 +0000 Subject: [PATCH 30/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index cc086f667..69ace79a8 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+987 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/e916fc3980e50062760e8af977e885ef58c3a93e +version=v1.1.1+989 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fab1f5277ad84bdba3c4973efacbb174518d05ec From c0e1e99ee2d7e36d54a07f870fceea05bf81af92 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 4 Jun 2025 09:38:50 -0400 Subject: [PATCH 31/59] Change dbid to uuid --- .../breedinginsight/brapi/v2/ListControllerIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index 2d421d306..108eb2fea 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -231,7 +231,7 @@ public void deleteListSuccess() { // A DELETE request to the brapi/v2/lists/ endpoint with invalid dbId. Flowable> invalidDeleteCall = client.exchange( - DELETE(String.format("/programs/%s/brapi/v2/lists/%s", program.getId().toString(), "NOT-VALID-DBID")) + DELETE(String.format("/programs/%s/brapi/v2/lists/%s", program.getId().toString(), "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); From 4dd096432872405a3fb692802f393579e43878de Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 4 Jun 2025 13:39:10 +0000 Subject: [PATCH 32/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 69ace79a8..472749168 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+989 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fab1f5277ad84bdba3c4973efacbb174518d05ec +version=v1.1.1+991 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c0e1e99ee2d7e36d54a07f870fceea05bf81af92 From 1492b126f6cbdabe5c7fc5f225095762c3265c23 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 4 Jun 2025 13:30:30 -0400 Subject: [PATCH 33/59] Update postgres to 17.5 --- docker-compose.yml | 2 +- src/test/java/org/breedinginsight/DatabaseTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bb30e858e..efebf5fb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,7 @@ services: networks: - backend bidb: - image: postgres:11.4 + image: postgres:17.5 container_name: bidb environment: - POSTGRES_DB=${DB_NAME} diff --git a/src/test/java/org/breedinginsight/DatabaseTest.java b/src/test/java/org/breedinginsight/DatabaseTest.java index b1213ee33..5a1279e7c 100644 --- a/src/test/java/org/breedinginsight/DatabaseTest.java +++ b/src/test/java/org/breedinginsight/DatabaseTest.java @@ -62,7 +62,7 @@ public DatabaseTest() { network = Network.newNetwork(); } if(dbContainer == null) { - dbContainer = new GenericContainer<>("postgres:11.4") + dbContainer = new GenericContainer<>("postgres:17.5") .withNetwork(network) .withNetworkAliases("testdb") .withImagePullPolicy(PullPolicy.defaultPolicy()) From d6242a6ce1afdf466e748e9762a1f3861dd86ca2 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 4 Jun 2025 17:30:49 +0000 Subject: [PATCH 34/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 472749168..6035bb36a 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+991 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c0e1e99ee2d7e36d54a07f870fceea05bf81af92 +version=v1.1.1+993 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1492b126f6cbdabe5c7fc5f225095762c3265c23 From aac64d81d8e623a4aedbb3e53b9dada770efba88 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 5 Jun 2025 15:22:54 +0000 Subject: [PATCH 35/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 6035bb36a..db6f4db79 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+993 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1492b126f6cbdabe5c7fc5f225095762c3265c23 +version=v1.1.1+995 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/942ebf1c00e358fe5ea37f6d91f0c42924c4db39 From a9fd111b436cd209bc21ebf11d3107b7d756f0d5 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 11 Jun 2025 14:49:08 -0400 Subject: [PATCH 36/59] [BI-2632] added back germplasmListExport() --- .../brapi/v2/BrAPIGermplasmController.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java index 84a6a17f0..850ed8138 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java @@ -222,6 +222,26 @@ public HttpResponse>>> getGermplasm( } } + @Get("/programs/{programId}/germplasm/lists/{listDbId}/export{?fileExtension}") + @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) + public HttpResponse germplasmListExport( + @PathVariable("programId") UUID programId, @PathVariable("listDbId") String listDbId, @QueryValue(defaultValue = "XLSX") String fileExtension) { + String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; + try { + FileType extension = Enum.valueOf(FileType.class, fileExtension); + DownloadFile germplasmListFile = germplasmService.exportGermplasmList(programId, listDbId, extension); + HttpResponse germplasmListExport = HttpResponse.ok(germplasmListFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+germplasmListFile.getFileName()+extension.getExtension()); + return germplasmListExport; + } + catch (Exception e) { + log.info(e.getMessage(), e); + e.printStackTrace(); + HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); + return response; + } + } + @Get("/programs/{programId}/germplasm/export{?fileExtension,list}") @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) From b424fbfd298435acb5422f8e4cf31bb5b12cb8c9 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 11 Jun 2025 15:23:03 -0400 Subject: [PATCH 37/59] [BI-2632] added test for germplasmListExport() --- .../GermplasmControllerIntegrationTest.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java index 32781dd8c..760974150 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java @@ -25,16 +25,22 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.Species; import org.breedinginsight.services.SpeciesService; +import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.response.mappers.GermplasmQueryMapper; import org.jooq.DSLContext; import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import javax.inject.Inject; +import java.io.ByteArrayInputStream; import java.io.File; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; +import tech.tablesaw.api.Table; import static io.micronaut.http.HttpRequest.GET; import static io.micronaut.http.HttpRequest.POST; @@ -238,6 +244,38 @@ public void getAllGermplasmListsSuccess() { } } + @ParameterizedTest + @CsvSource(value = {"CSV", "XLSX", "XLS"}) + @SneakyThrows + public void germplasmListExport(String extension) { + String programId = validProgram.getId().toString(); + String germplasmListDbId = fetchGermplasmListDbId(programId); + + // Build the endpoint to get germplasm by germplasm list. + String endpoint = String.format("/programs/%s/germplasm/lists/%s/export?fileExtension=%s", programId, germplasmListDbId, extension); + + // Get germplasm by list. + Flowable> call = client.exchange( + GET(endpoint).cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + + Table download = Table.create(); + if (extension.equals("CSV")) { + download = FileUtil.parseTableFromCsv(bodyStream); + } + if (extension.equals("XLS") || extension.equals("XLSX")) { + download = FileUtil.parseTableFromExcel(bodyStream, 0); + } + int dataSize = download.rowCount(); + assertEquals(3, dataSize, "Wrong number of germplasm were returned"); + } @Test @SneakyThrows public void getAllGermplasmByListSuccess() { From 75dca96076a3ce06a3a6beaac899629df54dc5a2 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 11 Jun 2025 19:55:39 +0000 Subject: [PATCH 38/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index db6f4db79..5d67ea37f 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+995 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/942ebf1c00e358fe5ea37f6d91f0c42924c4db39 +version=v1.1.1+999 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/3a374ed52351c73fcc58359ddc5b87c45b37ea21 From 522a1913e59f1ae52f0445a19c857d1872b3ed5f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 12 Jun 2025 10:13:30 -0400 Subject: [PATCH 39/59] Merge pull request #470 from Breeding-Insight/bug/BI-2664 BI-2664 - Add missing Honey Bee entry to species migration --- src/main/resources/brapi/sql/R__species.sql | 20 ++++++++++---------- src/test/resources/sql/brapi/species.sql | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index 261a18a06..ec6ae4c6c 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -30,17 +30,17 @@ SELECT -- 1) lower‑case -- 2) trim leading/trailing space -- 3) REMOVE every space or tab - regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), - v_auth_id, - crop_name + regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), + v_auth_id, + crop_name FROM (VALUES - ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), - ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), - ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), - ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), - ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), - ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), - ('Sugar Beet'), ('Coffee') + ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), + ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), + ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), + ('Strawberry'), ('Honey Bee'), ('Pecan'), ('Lettuce'), + ('Cotton'), ('Sorghum'), ('Hemp'), ('Hop'), + ('Hydrangea'), ('Red Clover'), ('Potato'), ('Blackberry'), + ('Raspberry'), ('Sugar Beet'), ('Coffee') ) AS src(crop_name) ON CONFLICT (id) DO -- want case changes or space changes to overwrite existing diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index f62e69156..2b8fcf26d 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -33,10 +33,10 @@ FROM (VALUES ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), - ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), - ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), - ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), - ('Sugar Beet'), ('Coffee') + ('Strawberry'), ('Honey Bee'), ('Pecan'), ('Lettuce'), + ('Cotton'), ('Sorghum'), ('Hemp'), ('Hop'), + ('Hydrangea'), ('Red Clover'), ('Potato'), ('Blackberry'), + ('Raspberry'), ('Sugar Beet'), ('Coffee') ) AS src(crop_name) ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name From 9ed32ebd400479f127ff90b9455c411cb248589e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 12 Jun 2025 14:35:13 +0000 Subject: [PATCH 40/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5d67ea37f..f97af9604 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+999 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/3a374ed52351c73fcc58359ddc5b87c45b37ea21 +version=v1.1.1+1003 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/522a1913e59f1ae52f0445a19c857d1872b3ed5f From 44114ed258a1d68247888af725fb7a83a6a96f3e Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 12 Jun 2025 15:55:17 -0400 Subject: [PATCH 41/59] First pass fix, new list only case looks good --- .../brapps/importer/model/base/Germplasm.java | 18 +++++++++++++++- .../germplasm/GermplasmProcessor.java | 21 +++++++++++++------ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java index 00d89e4db..de1cf19f1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java @@ -155,7 +155,19 @@ public static String constructGermplasmListName(String listName, Program program return String.format("%s [%s-germplasm]", listName, program.getKey()); } - public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID listId, boolean commit, boolean updatePedigree) { + /** + * Will mutate synonym and pedigree fields if changed and meet change criteria + * + * @param germplasm germplasm object + * @param program program + * @param listId list id + * @param commit flag indicating if commit changes should be made + * @param updatePedigree flag indicating if pedigree should be updated + * @return mutated indicator + */ + public boolean updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID listId, boolean commit, boolean updatePedigree) { + + boolean mutated = false; if (updatePedigree) { if (!StringUtils.isBlank(getFemaleParentAccessionNumber())) { @@ -170,6 +182,7 @@ public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID if (!StringUtils.isBlank(getMaleParentEntryNo())) { germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO, getMaleParentEntryNo()); } + mutated = true; } // Append synonyms to germplasm that don't already exist @@ -181,6 +194,7 @@ public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID brapiSynonym.setSynonym(synonym); if (!existingSynonyms.contains(brapiSynonym)) { germplasm.addSynonymsItem(brapiSynonym); + mutated = true; } } } @@ -193,6 +207,8 @@ public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID if (commit) { setUpdateCommitFields(germplasm, program.getKey()); } + + return mutated; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index d7e871128..b50c7474a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -383,6 +383,9 @@ private Germplasm removeBreedingMethodBlanks(Germplasm germplasm) { private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors validationErrors, List importRows, Program program, UUID importListId, boolean commit, PendingImport mappedImportRow, int rowIndex) { BrAPIGermplasm existingGermplasm; String gid = germplasm.getAccessionNumber(); + boolean mutated = false; + boolean updatePedigree = false; + if (germplasmByAccessionNumber.containsKey(gid)) { existingGermplasm = germplasmByAccessionNumber.get(gid).getBrAPIObject(); // Serialize and deserialize to deep copy @@ -410,15 +413,20 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v if(germplasm.pedigreeExists()) { validatePedigree(germplasm, rowIndex + 2, validationErrors); + updatePedigree = true; } - germplasm.updateBrAPIGermplasm(existingGermplasm, program, importListId, commit, true); - - updatedGermplasmList.add(existingGermplasm); - mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.MUTATED, existingGermplasm)); - importList.addDataItem(existingGermplasm.getGermplasmName()); + mutated = germplasm.updateBrAPIGermplasm(existingGermplasm, program, importListId, commit, updatePedigree); + if (mutated) { + updatedGermplasmList.add(existingGermplasm); + mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.MUTATED, existingGermplasm)); + } else { + mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm)); + } + // add to list regardless of mutated or not + importList.addDataItem(existingGermplasm.getGermplasmName()); return true; } @@ -631,7 +639,8 @@ public void postBrapiData(Map mappedBrAPIImport, Program } // Create list - if (!newGermplasmList.isEmpty() || !updatedGermplasmList.isEmpty()) { + // create & update flows both unconditionally add germplasm names to importList so use that for check + if (!importList.getData().isEmpty()) { try { // Create germplasm list brAPIListDAO.createBrAPILists(List.of(importList), program.getId(), upload); From eb98f66c4458973998c4468a9a637b0d7da97a4c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 13 Jun 2025 09:40:57 -0400 Subject: [PATCH 42/59] Update pedigree count to handle update case --- .../germplasm/GermplasmProcessor.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index b50c7474a..99baba3be 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -87,6 +87,8 @@ public class GermplasmProcessor implements Processor { List> postOrder = new ArrayList<>(); BrAPIListNewRequest importList = new BrAPIListNewRequest(); + private int numNewPedigreeConnections = 0; + public static String missingGIDsMsg = "The following GIDs were not found in the database: %s"; public static String missingParentalGIDsMsg = "The following parental GIDs were not found in the database: %s"; public static String missingParentalEntryNoMsg = "The following parental entry numbers were not found in the database: %s"; @@ -332,7 +334,7 @@ public Map process(ImportUpload upload, List
breedingMethods, @@ -359,6 +361,10 @@ private void processNewGermplasm(Germplasm germplasm, ValidationErrors validatio validatePedigree(germplasm, i + 2, validationErrors); + if (germplasm.pedigreeExists()) { + numNewPedigreeConnections++; + } + BrAPIGermplasm newGermplasm = germplasm.constructBrAPIGermplasm(program, breedingMethod, user, commit, BRAPI_REFERENCE_SOURCE, nextVal, importListId); newGermplasmList.add(newGermplasm); @@ -421,6 +427,9 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v if (mutated) { updatedGermplasmList.add(existingGermplasm); mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.MUTATED, existingGermplasm)); + if (updatePedigree) { + numNewPedigreeConnections++; + } } else { mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm)); } @@ -529,20 +538,17 @@ private boolean canUpdatePedigreeNoEqualsCheck(BrAPIGermplasm existingGermplasm, germplasm.pedigreeExists(); } - private Map getStatisticsMap(List importRows) { + private Map getStatisticsMap() { ImportPreviewStatistics germplasmStats = ImportPreviewStatistics.builder() .newObjectCount(newGermplasmList.size()) .ignoredObjectCount(germplasmByAccessionNumber.size()) .build(); - //Modified logic here to check for female parent accession number or entry no, removed check for male due to assumption that shouldn't have only male parent - int newObjectCount = newGermplasmList.stream().filter(newGermplasm -> newGermplasm != null).collect(Collectors.toList()).size(); + // TODO: numNewPedigreeConnections is global modified in existing and new flows, refactor at some point ImportPreviewStatistics pedigreeConnectStats = ImportPreviewStatistics.builder() - .newObjectCount(importRows.stream().filter(germplasmImport -> - germplasmImport.getGermplasm() != null && - (germplasmImport.getGermplasm().getFemaleParentAccessionNumber() != null || germplasmImport.getGermplasm().getFemaleParentEntryNo() != null) - ).collect(Collectors.toList()).size()).build(); + .newObjectCount(numNewPedigreeConnections) + .build(); return Map.of( "Germplasm", germplasmStats, From 2b10e0217b9c41cdcbf4b52ba1c2c943fc882250 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 13 Jun 2025 11:33:25 -0400 Subject: [PATCH 43/59] Only update pedigree if no existing and file has pedigree --- .../services/processors/germplasm/GermplasmProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index 99baba3be..491706493 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -417,7 +417,8 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v } } - if(germplasm.pedigreeExists()) { + // if no existing pedigree and file has pedigree then validate and update + if(germplasm.pedigreeExists() && !hasPedigree(existingGermplasm)) { validatePedigree(germplasm, rowIndex + 2, validationErrors); updatePedigree = true; } From 4ff31d35650afa865198fc1ebf90a1f855dbf512 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 17 Jun 2025 18:45:31 +0000 Subject: [PATCH 44/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index f97af9604..5f024da16 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+1003 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/522a1913e59f1ae52f0445a19c857d1872b3ed5f +version=v1.1.1+1005 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1ad31761718d63ac9fa31afbf35c5b7cc2a6c5b4 From f48c2a0cc65120884182f186d288b1a33a9421ce Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 18 Jun 2025 16:30:09 -0400 Subject: [PATCH 45/59] Update version --- src/main/resources/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5f024da16..30657dcaf 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+1005 +version=v1.2+1005 versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1ad31761718d63ac9fa31afbf35c5b7cc2a6c5b4 From ba5f9634b5de6b77206bf05ef8c80f3d92525f9e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 18 Jun 2025 20:30:25 +0000 Subject: [PATCH 46/59] [autocommit] bumping build number --- src/main/resources/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 30657dcaf..8aaa98f5e 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -16,4 +16,4 @@ version=v1.2+1005 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1ad31761718d63ac9fa31afbf35c5b7cc2a6c5b4 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f48c2a0cc65120884182f186d288b1a33a9421ce From d9e1ca087a495659c3f9c2d1ddb2b8daf5c434e4 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 18 Jun 2025 23:09:54 -0400 Subject: [PATCH 47/59] Fix version number --- src/main/resources/version.properties | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 8aaa98f5e..55d102c92 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,6 +14,5 @@ # limitations under the License. # - -version=v1.2+1005 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f48c2a0cc65120884182f186d288b1a33a9421ce +version=v1.2.0+1005 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f48c2a0cc65120884182f186d288b1a33a9421ce \ No newline at end of file From 3c86f265fdeabe2b0fb901af1eaaf80f89c05d82 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 19 Jun 2025 03:10:12 +0000 Subject: [PATCH 48/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 55d102c92..c89a627a2 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1005 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f48c2a0cc65120884182f186d288b1a33a9421ce \ No newline at end of file +version=v1.2.0+1009 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d9e1ca087a495659c3f9c2d1ddb2b8daf5c434e4 \ No newline at end of file From 62f8b1862a2328c50369d88fe635d70af8f9325c Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 9 Jul 2025 15:35:30 -0400 Subject: [PATCH 49/59] [BI-2656] - added null check --- .../appendoverwrite/factory/data/OverwrittenData.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index 54a1afe06..ced100205 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -137,9 +137,13 @@ public PendingImportObject constructPendingObservation() { if (!isTimestampMatched()) { // Update the timestamp - DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); - update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + if (timestamp == null) { + update.setObservationTimeStamp(null); + } else { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); + update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + } // Add original timestamp to changelog entry original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); From 5d25aab5338b8ba6bfbcefb9da531b41c3dfa758 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 10 Jul 2025 15:47:26 -0400 Subject: [PATCH 50/59] [BI-2656] - prevent setting timestamp to null --- .../factory/data/OverwrittenData.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index ced100205..9354d638c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -135,15 +135,11 @@ public PendingImportObject constructPendingObservation() { original = observation.getValue(); } - if (!isTimestampMatched()) { + if (!isTimestampMatched() && timestamp != null) { // Update the timestamp - if (timestamp == null) { - update.setObservationTimeStamp(null); - } else { - DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); - update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); - } + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); + update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); // Add original timestamp to changelog entry original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); From 14eeca3a6aa86e6bd95b046886c82be248a0a9e4 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 16 Jul 2025 17:57:41 -0400 Subject: [PATCH 51/59] [BI-2656] - added integration test --- .../importer/ExperimentFileImportTest.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 975c2b083..881768cbf 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1172,6 +1172,82 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { } } + /* + Scenario: + - an experiment was created with observations and timestamps + - do a second upload with additional observations for the experiment, but without the timestamp column + - verify the second set of observations get uploaded successfully + */ + @Test + @SneakyThrows + public void importNewObsAfterFirstExpWithObsAndTimestamps() { + log.debug("importNewObsAfterFirstExpWithObsAndTimestamps"); + List traits = importTestUtils.createTraits(2); + Program program = createProgram("Exp with TS and additional Uploads ", "EXTSAU", "EXTSAU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, Integer.valueOf("1")); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, Integer.valueOf("2025")); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, Integer.valueOf("1")); + newExp.put(Columns.BLOCK_NUM, Integer.valueOf("1")); + newExp.put(Columns.ROW, Integer.valueOf("1")); + newExp.put(Columns.COLUMN, Integer.valueOf("1")); + newExp.put(traits.get(0).getObservationVariableName(), "1"); + newExp.put("TS:" + traits.get(0).getObservationVariableName(), "2019-12-19T12:14:50Z"); + + // In the first upload, only 1 trait should be present. + List initialTraits = List.of(traits.get(0)); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), initialTraits), null, true, client, program, mappingId, newExperimentWorkflowId); + + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + Map newObservation = new HashMap<>(); + newObservation.put(Columns.GERMPLASM_GID, Integer.valueOf("1")); + newObservation.put(Columns.TEST_CHECK, "T"); + newObservation.put(Columns.EXP_TITLE, "Test Exp"); + newObservation.put(Columns.EXP_UNIT, "Plot"); + newObservation.put(Columns.EXP_TYPE, "Phenotyping"); + newObservation.put(Columns.ENV, "New Env"); + newObservation.put(Columns.ENV_LOCATION, "Location A"); + newObservation.put(Columns.ENV_YEAR, Integer.valueOf("2025")); + newObservation.put(Columns.EXP_UNIT_ID, "a-1"); + newObservation.put(Columns.REP_NUM, Integer.valueOf("1")); + newObservation.put(Columns.BLOCK_NUM, Integer.valueOf("1")); + newObservation.put(Columns.ROW, Integer.valueOf("1")); + newObservation.put(Columns.COLUMN, Integer.valueOf("1")); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put(traits.get(0).getObservationVariableName(), "1"); + newObservation.put(traits.get(1).getObservationVariableName(), "1"); + + // Pass overwrite in request body to allow append workflow to work normally. + Map userData = Map.of("overwrite", "true", "overwriteReason", "testing"); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), userData, true, client, program, mappingId, appendOverwriteWorkflowId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newObservation, program, traits); + + } + /* Scenario: - Create an experiment with valid observations. From 35f6303c660076803dc9fc9a7ac9276691264d65 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 16 Jul 2025 18:08:06 -0400 Subject: [PATCH 52/59] [BI-2656] - cleaned up test --- .../importer/ExperimentFileImportTest.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 881768cbf..7a984072c 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1185,19 +1185,19 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { List traits = importTestUtils.createTraits(2); Program program = createProgram("Exp with TS and additional Uploads ", "EXTSAU", "EXTSAU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); - newExp.put(Columns.GERMPLASM_GID, Integer.valueOf("1")); + newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); newExp.put(Columns.EXP_TITLE, "Test Exp"); newExp.put(Columns.EXP_UNIT, "Plot"); newExp.put(Columns.EXP_TYPE, "Phenotyping"); newExp.put(Columns.ENV, "New Env"); newExp.put(Columns.ENV_LOCATION, "Location A"); - newExp.put(Columns.ENV_YEAR, Integer.valueOf("2025")); + newExp.put(Columns.ENV_YEAR, "2025"); newExp.put(Columns.EXP_UNIT_ID, "a-1"); - newExp.put(Columns.REP_NUM, Integer.valueOf("1")); - newExp.put(Columns.BLOCK_NUM, Integer.valueOf("1")); - newExp.put(Columns.ROW, Integer.valueOf("1")); - newExp.put(Columns.COLUMN, Integer.valueOf("1")); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); newExp.put("TS:" + traits.get(0).getObservationVariableName(), "2019-12-19T12:14:50Z"); @@ -1215,24 +1215,24 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { assertTrue(ouIdXref.isPresent()); Map newObservation = new HashMap<>(); - newObservation.put(Columns.GERMPLASM_GID, Integer.valueOf("1")); + newObservation.put(Columns.GERMPLASM_GID, "1"); newObservation.put(Columns.TEST_CHECK, "T"); newObservation.put(Columns.EXP_TITLE, "Test Exp"); newObservation.put(Columns.EXP_UNIT, "Plot"); newObservation.put(Columns.EXP_TYPE, "Phenotyping"); newObservation.put(Columns.ENV, "New Env"); newObservation.put(Columns.ENV_LOCATION, "Location A"); - newObservation.put(Columns.ENV_YEAR, Integer.valueOf("2025")); + newObservation.put(Columns.ENV_YEAR, "2025"); newObservation.put(Columns.EXP_UNIT_ID, "a-1"); - newObservation.put(Columns.REP_NUM, Integer.valueOf("1")); - newObservation.put(Columns.BLOCK_NUM, Integer.valueOf("1")); - newObservation.put(Columns.ROW, Integer.valueOf("1")); - newObservation.put(Columns.COLUMN, Integer.valueOf("1")); + newObservation.put(Columns.REP_NUM, "1"); + newObservation.put(Columns.BLOCK_NUM, "1"); + newObservation.put(Columns.ROW, "1"); + newObservation.put(Columns.COLUMN, "1"); newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "1"); - // Pass overwrite in request body to allow append workflow to work normally. + // Send overwrite parameters in request body to allow the append workflow to work normally. Map userData = Map.of("overwrite", "true", "overwriteReason", "testing"); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), userData, true, client, program, mappingId, appendOverwriteWorkflowId); From 42c9c2ffb6a4021de57bc812ad9902f188862a5f Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 29 Jul 2025 14:47:14 -0400 Subject: [PATCH 53/59] [BI-2656] - handled timestamp edge case prevents erroneous mutated data indication on upload preview --- .../middleware/process/ImportTableProcess.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 770576b67..4a2846fcd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -284,7 +284,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); // Is there a change to the prior data? - if (isChanged(cellData, observation, cell.timestamp)) { + if (isChanged(cellData, observation, cell.timestamp, tsColByPheno.containsKey(phenoColumnName))) { // Is prior data protected? /** @@ -394,14 +394,18 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext } } - private boolean isChanged(String cellData, BrAPIObservation observation, String newTimestamp) { + private boolean isChanged(String cellData, BrAPIObservation observation, String newTimestamp, boolean timestampColumnPresent) { if (!cellData.isBlank() && !cellData.equals(observation.getValue())){ return true; } - if (StringUtils.isBlank(newTimestamp)) { - return (observation.getObservationTimeStamp()!=null); + // Only check timestamp if the TS: column was present in the uploaded file. + if (timestampColumnPresent) { + if (StringUtils.isBlank(newTimestamp)) { + return (observation.getObservationTimeStamp()!=null); + } + return !observationService.parseDateTime(newTimestamp).equals(observation.getObservationTimeStamp()); } - return !observationService.parseDateTime(newTimestamp).equals(observation.getObservationTimeStamp()); + return false; } /** From b8ba6f91a1bd3805a027b4678846dcb1abc1fc02 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 30 Jul 2025 14:23:25 +0000 Subject: [PATCH 54/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index c89a627a2..7597e75d0 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1009 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d9e1ca087a495659c3f9c2d1ddb2b8daf5c434e4 \ No newline at end of file +version=v1.2.0+1019 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/35238b11390a08c68a77f6f08778a7178257fd51 \ No newline at end of file From 365c3b26689eb8ff7ce0eb1c15b585aeaf96fb0d Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Fri, 8 Aug 2025 13:01:53 -0400 Subject: [PATCH 55/59] [BI-2656] - added error logging --- .../processors/experiment/service/ObservationService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index a6ecac2c0..de98202fa 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.core.BrAPISeason; @@ -42,6 +43,7 @@ import java.util.*; import java.util.stream.Collectors; +@Slf4j @Singleton public class ObservationService { private final ExperimentUtilities experimentUtilities; @@ -115,6 +117,7 @@ public OffsetDateTime parseDateTime(String dateString) { LocalDate localDate = LocalDate.parse(dateString, formatter); return localDate.atStartOfDay().atOffset(ZoneOffset.UTC); } catch (DateTimeParseException ex) { + log.error("Failed to parse timestamp: \"{}\".", dateString); // If both parsing attempts fail, return null return null; } From 77f5a1f8ecf195ea8a98807feda09aa0832e3edd Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 8 Aug 2025 17:31:11 +0000 Subject: [PATCH 56/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 7597e75d0..7cc40ef49 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1019 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/35238b11390a08c68a77f6f08778a7178257fd51 \ No newline at end of file +version=v1.2.0+1025 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2ddc8c9ad16047631e87f5b7f1ab28c7765353c1 \ No newline at end of file From d4d3873f32cd58a9544fcd7756877c95cec2f179 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 12 Aug 2025 12:25:02 -0400 Subject: [PATCH 57/59] [BI-2656] - updated isChanged logic handles append use case when prior data is left blank --- .../appendoverwrite/factory/data/OverwrittenData.java | 2 +- .../middleware/process/ImportTableProcess.java | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index 9354d638c..f3ce7e530 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -135,7 +135,7 @@ public PendingImportObject constructPendingObservation() { original = observation.getValue(); } - if (!isTimestampMatched() && timestamp != null) { + if (!isTimestampMatched() && StringUtils.isNotBlank(timestamp)) { // Update the timestamp DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 4a2846fcd..f2e74982e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -398,11 +398,8 @@ private boolean isChanged(String cellData, BrAPIObservation observation, String if (!cellData.isBlank() && !cellData.equals(observation.getValue())){ return true; } - // Only check timestamp if the TS: column was present in the uploaded file. - if (timestampColumnPresent) { - if (StringUtils.isBlank(newTimestamp)) { - return (observation.getObservationTimeStamp()!=null); - } + // Only check timestamp if the TS: column was present in the uploaded file and there's a valid timestamp. + if (timestampColumnPresent && !StringUtils.isBlank(newTimestamp)) { return !observationService.parseDateTime(newTimestamp).equals(observation.getObservationTimeStamp()); } return false; From 49d7ef65f83856883b40cd3182c14d388eed7859 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 13 Aug 2025 14:31:55 +0000 Subject: [PATCH 58/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 7cc40ef49..a018e5553 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1025 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2ddc8c9ad16047631e87f5b7f1ab28c7765353c1 \ No newline at end of file +version=v1.2.0+1027 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/64c0cf856256620b0f2e9cabf962a18257730bca \ No newline at end of file From 7112d7e1597680a5bea880ab408f24d3896ffe9f Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 13 Aug 2025 21:07:45 +0000 Subject: [PATCH 59/59] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 05d3816fe..bb93e957a 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.2.0 -versionInfo=https://github.com/Breeding-Insight/bi-api/releases/tag/v1.2.0 +version=v1.2.0+1031 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/4497ad61d94168c8c9e44e23698c01f91002599f