diff --git a/.env.template b/.env.template index abe859391..1a481aae6 100644 --- a/.env.template +++ b/.env.template @@ -31,6 +31,10 @@ BRAPI_READ_TIMEOUT=60m # Max number of records to POST to the BrAPI service per request POST_CHUNK_SIZE=1000 +# Request cache records paginating through available records per program by CACHE_BRAPI_FETCH_PAGE_SIZE +CACHE_PAGINATE_GERMPLASM=false +CACHE_BRAPI_FETCH_PAGE_SIZE=65000 + # BrAPI Server Variables BRAPI_SERVER_PORT=8083 diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 75226babf..a8feaecf6 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -65,6 +65,9 @@ public class BrAPIGermplasmDAO { @Property(name = "micronaut.bi.api.run-scheduled-tasks") private boolean runScheduledTasks; + @Property(name = "brapi.paginate.germplasm") + private boolean paginateGermplasm; + private final ProgramCache programGermplasmCache; private final BrAPIEndpointProvider brAPIEndpointProvider; @@ -141,11 +144,22 @@ private Map fetchProgramGermplasm(UUID programId) throws BrAPIGermplasmSearchRequest germplasmSearch = new BrAPIGermplasmSearchRequest(); germplasmSearch.externalReferenceIDs(List.of(programId.toString())); germplasmSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); - return processGermplasmForDisplay(brAPIDAOUtil.search( - api::searchGermplasmPost, - api::searchGermplasmSearchResultsDbIdGet, - germplasmSearch - ), program.getKey()); + + if (paginateGermplasm) { + log.debug("Fetching germplasm with pagination to BrAPI"); + return processGermplasmForDisplay(brAPIDAOUtil.search( + api::searchGermplasmPost, + api::searchGermplasmSearchResultsDbIdGet, + germplasmSearch), + program.getKey()); + } else { + log.debug("Fetching germplasm without pagination to BrAPI"); + return processGermplasmForDisplay(brAPIDAOUtil.searchNoPaging( + api::searchGermplasmPost, + api::searchGermplasmSearchResultsDbIdGet, + germplasmSearch), + program.getKey()); + } } /** diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index b730660e3..cab44d951 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -60,6 +60,7 @@ public class BrAPIDAOUtil { private final Duration searchTimeout; private final int pageSize; private final int postGroupSize; + private final int brapiFetchPageSize; private final ProgramService programService; @Inject @@ -67,11 +68,13 @@ public BrAPIDAOUtil(@Property(name = "brapi.search.wait-time") int searchWaitTim @Property(name = "brapi.read-timeout") Duration searchTimeout, @Property(name = "brapi.page-size") int pageSize, @Property(name = "brapi.post-group-size") int postGroupSize, + @Property(name = "brapi.cache.fetch-page-size") int brapiFetchPageSize, ProgramService programService) { this.searchWaitTime = searchWaitTime; this.searchTimeout = searchTimeout; this.pageSize = pageSize; this.postGroupSize = postGroupSize; + this.brapiFetchPageSize = brapiFetchPageSize; this.programService = programService; } @@ -79,26 +82,37 @@ public List search(Funct Function3, Optional>>> searchGetMethod, U searchBody ) throws ApiException { - return searchInternal(searchMethod, searchGetMethod, null, searchBody); + return searchInternal(searchMethod, searchGetMethod, null, searchBody, true); } public List search(Function, Optional>>> searchMethod, Function4, Optional>>> searchGetMethod, U searchBody ) throws ApiException { - return searchInternal(searchMethod, null, searchGetMethod, searchBody); + return searchInternal(searchMethod, null, searchGetMethod, searchBody, true); + } + + public List searchNoPaging(Function, Optional>>> searchMethod, + Function3, Optional>>> searchGetMethod, + U searchBody + ) throws ApiException { + return searchInternal(searchMethod, searchGetMethod, null, searchBody, false); } private List searchInternal(Function, Optional>>> searchMethod, Function3, Optional>>> searchGetMethod, Function4, Optional>>> searchGetMethodWithMimeType, - U searchBody) throws ApiException { + U searchBody, boolean sendPaging) throws ApiException { try { List listResult = new ArrayList<>(); - //NOTE: Because of the way Breedbase implements BrAPI searches, the page size is initially set to an - //arbitrary, large value to ensure that in the event that a 202 response is returned, the searchDbId - //stored will refer to all records of the BrAPI variable. - searchBody.pageSize(10000000); + + if (sendPaging) { + // This should be set to whatever the maximum allowable value is configured in the brapi test server, + // perhaps it should be configurable on bi side as well. + // For reference, that prop name is paging.page-size.max-allowed + searchBody.pageSize(brapiFetchPageSize); + } + ApiResponse, Optional>> response = searchMethod.apply(searchBody); if (response.getBody().getLeft().isPresent()) { BrAPIResponse listResponse = (BrAPIResponse) response.getBody().getLeft().get(); @@ -108,7 +122,7 @@ private List searchInter pagination params are handled for POST search endpoints or the corresponding endpoints in Breedbase are changed or updated */ - if(hasMorePages(listResponse)) { + if(sendPaging && hasMorePages(listResponse)) { int currentPage = listResponse.getMetadata().getPagination().getCurrentPage() + 1; int totalPages = listResponse.getMetadata().getPagination().getTotalPages(); @@ -137,7 +151,7 @@ private List searchInter BrAPIResponse listResponse = (BrAPIResponse) searchGetResponse.getBody().getLeft().get(); listResult = getListResult(searchGetResponse); - if(hasMorePages(listResponse)) { + if(sendPaging && hasMorePages(listResponse)) { currentPage++; int totalPages = listResponse.getMetadata() .getPagination() diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 3a98c3fb6..820d0cbac 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -173,6 +173,19 @@ brapi: search: wait-time: 1000 post-group-size: ${POST_CHUNK_SIZE:1000} + paginate: + # These props are used as flippers for specific cache entities to be fetched all at once or paginated. + # If the props are set to true, there's a possibility the BrAPI server could run out of memory grabbing all the + # data and assigning it to entities in memory given a program that has a large amount of those types of entities. + # At that point these flippers should be set to true. + germplasm: ${CACHE_PAGINATE_GERMPLASM:false} + cache: + # This prop sets how many program cache entity records can be fetched at a time from BrAPI. Today, if over 65000 is used, + # a SQL error can happen for programs that have more than this page size, because to fetch these certain entities + # the same amount of IDs must be passed for any collections that need to be fetched in the entity, and SQL has a max of ~65355 params. + # The ProgramCache will iterate through all pages of an entity to retrieve all the data from BrAPI. + fetch-page-size: ${CACHE_BRAPI_FETCH_PAGE_SIZE:65000} + email: relay-server: diff --git a/src/main/resources/brapi/properties/application.properties b/src/main/resources/brapi/properties/application.properties index 28a097f3a..3733fcd37 100644 --- a/src/main/resources/brapi/properties/application.properties +++ b/src/main/resources/brapi/properties/application.properties @@ -36,3 +36,13 @@ spring.mvc.dispatch-options-request=true security.oidc_discovery_url=https://example.com/auth/.well-known/openid-configuration security.enabled=false +security.issuer_url=http://example.com/issuerurl + +# This should either be set in accordance with a maximum number of SQL parameters (on JOIN FETCHES of collections, +# if there is more than one collection the IDs of each entity need to be passed through as parameters, and there is a SQL +# maximum of 65535. See GermplasmService.findGermplasmEntities()), +# whatever returns in a reasonable amount of time, +# or if you want to limit for the sake of server efficiency. +paging.page-size.max-allowed=65000 + +paging.page-size.default=1000 diff --git a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java index 427ed6d88..d77971817 100644 --- a/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java +++ b/src/test/java/org/breedinginsight/daos/BrAPIDAOUtilUnitTest.java @@ -70,7 +70,7 @@ public ApiResponse, Optional