From 1b451787ee6997cae36d110ae2a0dd74125b5620 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 19 May 2025 18:38:13 +0000 Subject: [PATCH 01/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5dda72802..8ec9f3000 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+959 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/738dee1bea47d07b5e24a37304dd83e6d3f29b65 +version=v1.1.0+965 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c66193ae96e86b4edafa4b117b101e1b2e1a7ed8 From bcd7f16800bb6fcdf51a893f41222313478a3c7c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Mon, 19 May 2025 15:32:46 -0400 Subject: [PATCH 02/46] Update version number --- src/main/resources/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 8ec9f3000..f9fc4d945 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.0+965 +version=v1.1.1+965 versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c66193ae96e86b4edafa4b117b101e1b2e1a7ed8 From 0a9f8baaceb8572fec2329c97d01f63a2e7e7a76 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 19 May 2025 19:33:04 +0000 Subject: [PATCH 03/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index f9fc4d945..9eebddfde 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+965 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c66193ae96e86b4edafa4b117b101e1b2e1a7ed8 +version=v1.1.1+969 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/bcd7f16800bb6fcdf51a893f41222313478a3c7c From da9891911b9b75a1b2f13340036180ae342ad976 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 29 May 2025 12:10:18 -0400 Subject: [PATCH 04/46] [BI-2654] - replaced reserved keyword with allowed value --- src/test/resources/sql/UserControllerIntegrationTest.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/resources/sql/UserControllerIntegrationTest.sql b/src/test/resources/sql/UserControllerIntegrationTest.sql index 55cafed8d..5dce6cca0 100644 --- a/src/test/resources/sql/UserControllerIntegrationTest.sql +++ b/src/test/resources/sql/UserControllerIntegrationTest.sql @@ -27,19 +27,19 @@ join bi_user on bi_user.name = 'system' limit 1; -- name: InsertUserProgramAssociations insert into program_user_role (program_id, user_id, role_id, created_by, updated_by) -select program.id, bi_user.id, role.id, system_user.id, system_user.id +select program.id, bi_user.id, role.id, bi_system_user.id, bi_system_user.id from program join bi_user on bi_user.name = 'Test User' or bi_user.name = 'Other Test User' join role on role.domain = 'Read Only' -join bi_user as system_user on system_user.name = 'system' +join bi_user as bi_system_user on bi_system_user.name = 'system' where program.name = 'Test Program'; insert into program_user_role (program_id, user_id, role_id, active, created_by, updated_by) -select program.id, bi_user.id, role.id, false, system_user.id, system_user.id +select program.id, bi_user.id, role.id, false, bi_system_user.id, bi_system_user.id from program join bi_user on bi_user.name = 'Test User' or bi_user.name = 'Other Test User' join role on role.domain = 'Read Only' -join bi_user as system_user on system_user.name = 'system' +join bi_user as bi_system_user on bi_system_user.name = 'system' where program.name = 'Test Program1'; -- name: DeactivateProgram From fc3612964f293fe2340be60f926ee9dfa883108d Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 29 May 2025 16:45:05 -0400 Subject: [PATCH 05/46] Increased signup token expiration from 1hr to 24hr --- src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 820d0cbac..313540dbd 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -145,7 +145,7 @@ web: logout: url: ${web.base-url} signup: - url-timeout: 60m + url-timeout: 24h signup: url: ${web.base-url}/signup success: From a3b80c749222b1f8da3fe8a5ef9b4a9e7562f23a Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 30 May 2025 14:39:54 +0000 Subject: [PATCH 06/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 9eebddfde..d450b6379 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+969 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/bcd7f16800bb6fcdf51a893f41222313478a3c7c +version=v1.1.1+979 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a290899ba43d9e0c42730a3c0e6170b416ee70e8 From 1842f026418d4e8fcfa3f2ce6fa905c5f6b22529 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Tue, 3 Jun 2025 17:10:40 -0400 Subject: [PATCH 07/46] Update species migration to work with new uuids --- src/main/resources/brapi/sql/R__species.sql | 64 +++++++++++---------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index 00c1b7042..261a18a06 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -13,32 +13,38 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '4', 'Blueberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '5', 'Salmon') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '6', 'Grape') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '7', 'Alfalfa') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '8', 'Sweet Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '9', 'Trout') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '10', 'Soybean') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '11', 'Cranberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '12', 'Cucumber') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'Oat') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; --- for the Honey Bee case, want to overwrite name, not preserve existing -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey Bee') ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '21', 'Sorghum') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '22', 'Hemp') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '23', 'Hop') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '24', 'Hydrangea') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '25', 'Red Clover') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '26', 'Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '27', 'Blackberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '28', 'Raspberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '29', 'Sugar Beet') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '30', 'Strawberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '31', 'Coffee') ON CONFLICT DO NOTHING; \ No newline at end of file +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +DO $$ +DECLARE +v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; +BEGIN + /* ------------------------------------------------------------------------------------------ + • uuid_generate_v5(namespace, crop_name) → deterministic UUID can be used for idempotency + • Do it this way so no schema changes are required + • Removed the Honey Bee special case because all systems will be starting fresh + ------------------------------------------------------------------------------------------ */ +INSERT INTO crop (id, auth_user_id, crop_name) +SELECT + uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, + -- 1) lower‑case + -- 2) trim leading/trailing space + -- 3) REMOVE every space or tab + regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), + v_auth_id, + crop_name +FROM (VALUES + ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), + ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), + ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), + ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), + ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), + ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), + ('Sugar Beet'), ('Coffee') + ) AS src(crop_name) + ON CONFLICT (id) DO +-- want case changes or space changes to overwrite existing +-- Only rewrite the row if name changed +UPDATE SET crop_name = EXCLUDED.crop_name +WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; +END $$; \ No newline at end of file From f97f6d41c705685fb13bae41ce60d5666805f4da Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 3 Jun 2025 21:10:59 +0000 Subject: [PATCH 08/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index d450b6379..dc9397197 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+979 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a290899ba43d9e0c42730a3c0e6170b416ee70e8 +version=v1.1.1+985 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1842f026418d4e8fcfa3f2ce6fa905c5f6b22529 From 7a6be40f95e7e534883fb6a4083795b03b013283 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 4 Jun 2025 08:44:58 -0400 Subject: [PATCH 09/46] Update species sql for tests --- src/test/resources/sql/brapi/species.sql | 44 ++++++++++++++---------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index 817a1c726..f62e69156 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -17,22 +17,28 @@ */ -- name: InsertSpecies -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '4', 'Blueberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '5', 'Salmon') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '6', 'Grape') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '7', 'Alfalfa') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '8', 'Sweet Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '9', 'Trout') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '10', 'Soybean') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '11', 'Cranberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '12', 'Cucumber') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'Oat') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '21', 'Sorghum') ON CONFLICT DO NOTHING; -INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '22', 'Hemp') ON CONFLICT DO NOTHING; +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +DO $$ +DECLARE +v_auth_id CONSTANT uuid := 'AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA'; +BEGIN +INSERT INTO crop (id, auth_user_id, crop_name) +SELECT + uuid_generate_v5('9a4deca9-4068-46a3-9efe-db0c181f491a'::uuid, + regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), + v_auth_id, + crop_name +FROM (VALUES + ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), + ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), + ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), + ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), + ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), + ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), + ('Sugar Beet'), ('Coffee') + ) AS src(crop_name) + ON CONFLICT (id) DO +UPDATE SET crop_name = EXCLUDED.crop_name +WHERE crop.crop_name IS DISTINCT FROM EXCLUDED.crop_name; +END $$; \ No newline at end of file From dbec72a2e250b0f1887e7ad0f52314dfd1d9639c Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 4 Jun 2025 12:46:02 +0000 Subject: [PATCH 10/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index dc9397197..cc086f667 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+985 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1842f026418d4e8fcfa3f2ce6fa905c5f6b22529 +version=v1.1.1+987 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/e916fc3980e50062760e8af977e885ef58c3a93e From 7129705ffa33984ae862366f0b00b3cb81b73843 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 4 Jun 2025 09:18:24 -0400 Subject: [PATCH 11/46] Add security issuer env for tests --- src/test/java/org/breedinginsight/BrAPITest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index 8c5c35bad..bdf362979 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -60,6 +60,7 @@ public BrAPITest() { .withEnv("BRAPI_DB", "postgres") .withEnv("BRAPI_DB_USER", "postgres") .withEnv("BRAPI_DB_PASSWORD", "postgres") + .withEnv("SECURITY_ISSUER_URL", "http://example.com/issuerurl") .withClasspathResourceMapping("sql/brapi/mount", "/home/brapi/db/sql", BindMode.READ_WRITE) // HACK - READ_WRITE forces testcontainers to use a bind mount (which overwrites) instead of copying files. .withClasspathResourceMapping("brapi/properties/application.properties", "/home/brapi/properties/application.properties", BindMode.READ_ONLY) .waitingFor(Wait.forLogMessage(".*Started BrapiTestServer in \\d*.\\d* seconds.*", 1).withStartupTimeout(Duration.ofMinutes(1))); From e05bad527b5c4b727925203abc064139ef2857c9 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 4 Jun 2025 13:19:01 +0000 Subject: [PATCH 12/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index cc086f667..69ace79a8 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+987 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/e916fc3980e50062760e8af977e885ef58c3a93e +version=v1.1.1+989 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fab1f5277ad84bdba3c4973efacbb174518d05ec From c0e1e99ee2d7e36d54a07f870fceea05bf81af92 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 4 Jun 2025 09:38:50 -0400 Subject: [PATCH 13/46] Change dbid to uuid --- .../breedinginsight/brapi/v2/ListControllerIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index 2d421d306..108eb2fea 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -231,7 +231,7 @@ public void deleteListSuccess() { // A DELETE request to the brapi/v2/lists/ endpoint with invalid dbId. Flowable> invalidDeleteCall = client.exchange( - DELETE(String.format("/programs/%s/brapi/v2/lists/%s", program.getId().toString(), "NOT-VALID-DBID")) + DELETE(String.format("/programs/%s/brapi/v2/lists/%s", program.getId().toString(), "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa")) .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class ); From 4dd096432872405a3fb692802f393579e43878de Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 4 Jun 2025 13:39:10 +0000 Subject: [PATCH 14/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 69ace79a8..472749168 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+989 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fab1f5277ad84bdba3c4973efacbb174518d05ec +version=v1.1.1+991 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c0e1e99ee2d7e36d54a07f870fceea05bf81af92 From 1492b126f6cbdabe5c7fc5f225095762c3265c23 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 4 Jun 2025 13:30:30 -0400 Subject: [PATCH 15/46] Update postgres to 17.5 --- docker-compose.yml | 2 +- src/test/java/org/breedinginsight/DatabaseTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index bb30e858e..efebf5fb6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,7 +71,7 @@ services: networks: - backend bidb: - image: postgres:11.4 + image: postgres:17.5 container_name: bidb environment: - POSTGRES_DB=${DB_NAME} diff --git a/src/test/java/org/breedinginsight/DatabaseTest.java b/src/test/java/org/breedinginsight/DatabaseTest.java index b1213ee33..5a1279e7c 100644 --- a/src/test/java/org/breedinginsight/DatabaseTest.java +++ b/src/test/java/org/breedinginsight/DatabaseTest.java @@ -62,7 +62,7 @@ public DatabaseTest() { network = Network.newNetwork(); } if(dbContainer == null) { - dbContainer = new GenericContainer<>("postgres:11.4") + dbContainer = new GenericContainer<>("postgres:17.5") .withNetwork(network) .withNetworkAliases("testdb") .withImagePullPolicy(PullPolicy.defaultPolicy()) From d6242a6ce1afdf466e748e9762a1f3861dd86ca2 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 4 Jun 2025 17:30:49 +0000 Subject: [PATCH 16/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 472749168..6035bb36a 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+991 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c0e1e99ee2d7e36d54a07f870fceea05bf81af92 +version=v1.1.1+993 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1492b126f6cbdabe5c7fc5f225095762c3265c23 From aac64d81d8e623a4aedbb3e53b9dada770efba88 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 5 Jun 2025 15:22:54 +0000 Subject: [PATCH 17/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 6035bb36a..db6f4db79 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+993 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1492b126f6cbdabe5c7fc5f225095762c3265c23 +version=v1.1.1+995 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/942ebf1c00e358fe5ea37f6d91f0c42924c4db39 From a9fd111b436cd209bc21ebf11d3107b7d756f0d5 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 11 Jun 2025 14:49:08 -0400 Subject: [PATCH 18/46] [BI-2632] added back germplasmListExport() --- .../brapi/v2/BrAPIGermplasmController.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java index 84a6a17f0..850ed8138 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java @@ -222,6 +222,26 @@ public HttpResponse>>> getGermplasm( } } + @Get("/programs/{programId}/germplasm/lists/{listDbId}/export{?fileExtension}") + @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) + public HttpResponse germplasmListExport( + @PathVariable("programId") UUID programId, @PathVariable("listDbId") String listDbId, @QueryValue(defaultValue = "XLSX") String fileExtension) { + String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; + try { + FileType extension = Enum.valueOf(FileType.class, fileExtension); + DownloadFile germplasmListFile = germplasmService.exportGermplasmList(programId, listDbId, extension); + HttpResponse germplasmListExport = HttpResponse.ok(germplasmListFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+germplasmListFile.getFileName()+extension.getExtension()); + return germplasmListExport; + } + catch (Exception e) { + log.info(e.getMessage(), e); + e.printStackTrace(); + HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); + return response; + } + } + @Get("/programs/{programId}/germplasm/export{?fileExtension,list}") @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) From b424fbfd298435acb5422f8e4cf31bb5b12cb8c9 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 11 Jun 2025 15:23:03 -0400 Subject: [PATCH 19/46] [BI-2632] added test for germplasmListExport() --- .../GermplasmControllerIntegrationTest.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java index 32781dd8c..760974150 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java @@ -25,16 +25,22 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.Species; import org.breedinginsight.services.SpeciesService; +import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.response.mappers.GermplasmQueryMapper; import org.jooq.DSLContext; import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import javax.inject.Inject; +import java.io.ByteArrayInputStream; import java.io.File; import java.time.OffsetDateTime; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; +import tech.tablesaw.api.Table; import static io.micronaut.http.HttpRequest.GET; import static io.micronaut.http.HttpRequest.POST; @@ -238,6 +244,38 @@ public void getAllGermplasmListsSuccess() { } } + @ParameterizedTest + @CsvSource(value = {"CSV", "XLSX", "XLS"}) + @SneakyThrows + public void germplasmListExport(String extension) { + String programId = validProgram.getId().toString(); + String germplasmListDbId = fetchGermplasmListDbId(programId); + + // Build the endpoint to get germplasm by germplasm list. + String endpoint = String.format("/programs/%s/germplasm/lists/%s/export?fileExtension=%s", programId, germplasmListDbId, extension); + + // Get germplasm by list. + Flowable> call = client.exchange( + GET(endpoint).cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + + Table download = Table.create(); + if (extension.equals("CSV")) { + download = FileUtil.parseTableFromCsv(bodyStream); + } + if (extension.equals("XLS") || extension.equals("XLSX")) { + download = FileUtil.parseTableFromExcel(bodyStream, 0); + } + int dataSize = download.rowCount(); + assertEquals(3, dataSize, "Wrong number of germplasm were returned"); + } @Test @SneakyThrows public void getAllGermplasmByListSuccess() { From 75dca96076a3ce06a3a6beaac899629df54dc5a2 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 11 Jun 2025 19:55:39 +0000 Subject: [PATCH 20/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index db6f4db79..5d67ea37f 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+995 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/942ebf1c00e358fe5ea37f6d91f0c42924c4db39 +version=v1.1.1+999 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/3a374ed52351c73fcc58359ddc5b87c45b37ea21 From 522a1913e59f1ae52f0445a19c857d1872b3ed5f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 12 Jun 2025 10:13:30 -0400 Subject: [PATCH 21/46] Merge pull request #470 from Breeding-Insight/bug/BI-2664 BI-2664 - Add missing Honey Bee entry to species migration --- src/main/resources/brapi/sql/R__species.sql | 20 ++++++++++---------- src/test/resources/sql/brapi/species.sql | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/resources/brapi/sql/R__species.sql b/src/main/resources/brapi/sql/R__species.sql index 261a18a06..ec6ae4c6c 100644 --- a/src/main/resources/brapi/sql/R__species.sql +++ b/src/main/resources/brapi/sql/R__species.sql @@ -30,17 +30,17 @@ SELECT -- 1) lower‑case -- 2) trim leading/trailing space -- 3) REMOVE every space or tab - regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), - v_auth_id, - crop_name + regexp_replace(lower(trim(crop_name)), '\s', '', 'g')), + v_auth_id, + crop_name FROM (VALUES - ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), - ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), - ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), - ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), - ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), - ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), - ('Sugar Beet'), ('Coffee') + ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), + ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), + ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), + ('Strawberry'), ('Honey Bee'), ('Pecan'), ('Lettuce'), + ('Cotton'), ('Sorghum'), ('Hemp'), ('Hop'), + ('Hydrangea'), ('Red Clover'), ('Potato'), ('Blackberry'), + ('Raspberry'), ('Sugar Beet'), ('Coffee') ) AS src(crop_name) ON CONFLICT (id) DO -- want case changes or space changes to overwrite existing diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index f62e69156..2b8fcf26d 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -33,10 +33,10 @@ FROM (VALUES ('Blueberry'), ('Salmon'), ('Grape'), ('Alfalfa'), ('Sweet Potato'), ('Trout'), ('Soybean'), ('Cranberry'), ('Cucumber'), ('Oat'), ('Citrus'), ('Sugar Cane'), - ('Strawberry'), ('Pecan'), ('Lettuce'), ('Cotton'), - ('Sorghum'), ('Hemp'), ('Hop'), ('Hydrangea'), - ('Red Clover'), ('Potato'), ('Blackberry'), ('Raspberry'), - ('Sugar Beet'), ('Coffee') + ('Strawberry'), ('Honey Bee'), ('Pecan'), ('Lettuce'), + ('Cotton'), ('Sorghum'), ('Hemp'), ('Hop'), + ('Hydrangea'), ('Red Clover'), ('Potato'), ('Blackberry'), + ('Raspberry'), ('Sugar Beet'), ('Coffee') ) AS src(crop_name) ON CONFLICT (id) DO UPDATE SET crop_name = EXCLUDED.crop_name From 9ed32ebd400479f127ff90b9455c411cb248589e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 12 Jun 2025 14:35:13 +0000 Subject: [PATCH 22/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5d67ea37f..f97af9604 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+999 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/3a374ed52351c73fcc58359ddc5b87c45b37ea21 +version=v1.1.1+1003 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/522a1913e59f1ae52f0445a19c857d1872b3ed5f From 44114ed258a1d68247888af725fb7a83a6a96f3e Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 12 Jun 2025 15:55:17 -0400 Subject: [PATCH 23/46] First pass fix, new list only case looks good --- .../brapps/importer/model/base/Germplasm.java | 18 +++++++++++++++- .../germplasm/GermplasmProcessor.java | 21 +++++++++++++------ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java index 00d89e4db..de1cf19f1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java @@ -155,7 +155,19 @@ public static String constructGermplasmListName(String listName, Program program return String.format("%s [%s-germplasm]", listName, program.getKey()); } - public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID listId, boolean commit, boolean updatePedigree) { + /** + * Will mutate synonym and pedigree fields if changed and meet change criteria + * + * @param germplasm germplasm object + * @param program program + * @param listId list id + * @param commit flag indicating if commit changes should be made + * @param updatePedigree flag indicating if pedigree should be updated + * @return mutated indicator + */ + public boolean updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID listId, boolean commit, boolean updatePedigree) { + + boolean mutated = false; if (updatePedigree) { if (!StringUtils.isBlank(getFemaleParentAccessionNumber())) { @@ -170,6 +182,7 @@ public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID if (!StringUtils.isBlank(getMaleParentEntryNo())) { germplasm.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_ENTRY_NO, getMaleParentEntryNo()); } + mutated = true; } // Append synonyms to germplasm that don't already exist @@ -181,6 +194,7 @@ public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID brapiSynonym.setSynonym(synonym); if (!existingSynonyms.contains(brapiSynonym)) { germplasm.addSynonymsItem(brapiSynonym); + mutated = true; } } } @@ -193,6 +207,8 @@ public void updateBrAPIGermplasm(BrAPIGermplasm germplasm, Program program, UUID if (commit) { setUpdateCommitFields(germplasm, program.getKey()); } + + return mutated; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index d7e871128..b50c7474a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -383,6 +383,9 @@ private Germplasm removeBreedingMethodBlanks(Germplasm germplasm) { private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors validationErrors, List importRows, Program program, UUID importListId, boolean commit, PendingImport mappedImportRow, int rowIndex) { BrAPIGermplasm existingGermplasm; String gid = germplasm.getAccessionNumber(); + boolean mutated = false; + boolean updatePedigree = false; + if (germplasmByAccessionNumber.containsKey(gid)) { existingGermplasm = germplasmByAccessionNumber.get(gid).getBrAPIObject(); // Serialize and deserialize to deep copy @@ -410,15 +413,20 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v if(germplasm.pedigreeExists()) { validatePedigree(germplasm, rowIndex + 2, validationErrors); + updatePedigree = true; } - germplasm.updateBrAPIGermplasm(existingGermplasm, program, importListId, commit, true); - - updatedGermplasmList.add(existingGermplasm); - mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.MUTATED, existingGermplasm)); - importList.addDataItem(existingGermplasm.getGermplasmName()); + mutated = germplasm.updateBrAPIGermplasm(existingGermplasm, program, importListId, commit, updatePedigree); + if (mutated) { + updatedGermplasmList.add(existingGermplasm); + mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.MUTATED, existingGermplasm)); + } else { + mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm)); + } + // add to list regardless of mutated or not + importList.addDataItem(existingGermplasm.getGermplasmName()); return true; } @@ -631,7 +639,8 @@ public void postBrapiData(Map mappedBrAPIImport, Program } // Create list - if (!newGermplasmList.isEmpty() || !updatedGermplasmList.isEmpty()) { + // create & update flows both unconditionally add germplasm names to importList so use that for check + if (!importList.getData().isEmpty()) { try { // Create germplasm list brAPIListDAO.createBrAPILists(List.of(importList), program.getId(), upload); From eb98f66c4458973998c4468a9a637b0d7da97a4c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 13 Jun 2025 09:40:57 -0400 Subject: [PATCH 24/46] Update pedigree count to handle update case --- .../germplasm/GermplasmProcessor.java | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index b50c7474a..99baba3be 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -87,6 +87,8 @@ public class GermplasmProcessor implements Processor { List> postOrder = new ArrayList<>(); BrAPIListNewRequest importList = new BrAPIListNewRequest(); + private int numNewPedigreeConnections = 0; + public static String missingGIDsMsg = "The following GIDs were not found in the database: %s"; public static String missingParentalGIDsMsg = "The following parental GIDs were not found in the database: %s"; public static String missingParentalEntryNoMsg = "The following parental entry numbers were not found in the database: %s"; @@ -332,7 +334,7 @@ public Map process(ImportUpload upload, List
breedingMethods, @@ -359,6 +361,10 @@ private void processNewGermplasm(Germplasm germplasm, ValidationErrors validatio validatePedigree(germplasm, i + 2, validationErrors); + if (germplasm.pedigreeExists()) { + numNewPedigreeConnections++; + } + BrAPIGermplasm newGermplasm = germplasm.constructBrAPIGermplasm(program, breedingMethod, user, commit, BRAPI_REFERENCE_SOURCE, nextVal, importListId); newGermplasmList.add(newGermplasm); @@ -421,6 +427,9 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v if (mutated) { updatedGermplasmList.add(existingGermplasm); mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.MUTATED, existingGermplasm)); + if (updatePedigree) { + numNewPedigreeConnections++; + } } else { mappedImportRow.setGermplasm(new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm)); } @@ -529,20 +538,17 @@ private boolean canUpdatePedigreeNoEqualsCheck(BrAPIGermplasm existingGermplasm, germplasm.pedigreeExists(); } - private Map getStatisticsMap(List importRows) { + private Map getStatisticsMap() { ImportPreviewStatistics germplasmStats = ImportPreviewStatistics.builder() .newObjectCount(newGermplasmList.size()) .ignoredObjectCount(germplasmByAccessionNumber.size()) .build(); - //Modified logic here to check for female parent accession number or entry no, removed check for male due to assumption that shouldn't have only male parent - int newObjectCount = newGermplasmList.stream().filter(newGermplasm -> newGermplasm != null).collect(Collectors.toList()).size(); + // TODO: numNewPedigreeConnections is global modified in existing and new flows, refactor at some point ImportPreviewStatistics pedigreeConnectStats = ImportPreviewStatistics.builder() - .newObjectCount(importRows.stream().filter(germplasmImport -> - germplasmImport.getGermplasm() != null && - (germplasmImport.getGermplasm().getFemaleParentAccessionNumber() != null || germplasmImport.getGermplasm().getFemaleParentEntryNo() != null) - ).collect(Collectors.toList()).size()).build(); + .newObjectCount(numNewPedigreeConnections) + .build(); return Map.of( "Germplasm", germplasmStats, From 2b10e0217b9c41cdcbf4b52ba1c2c943fc882250 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Fri, 13 Jun 2025 11:33:25 -0400 Subject: [PATCH 25/46] Only update pedigree if no existing and file has pedigree --- .../services/processors/germplasm/GermplasmProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index 99baba3be..491706493 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -417,7 +417,8 @@ private boolean processExistingGermplasm(Germplasm germplasm, ValidationErrors v } } - if(germplasm.pedigreeExists()) { + // if no existing pedigree and file has pedigree then validate and update + if(germplasm.pedigreeExists() && !hasPedigree(existingGermplasm)) { validatePedigree(germplasm, rowIndex + 2, validationErrors); updatePedigree = true; } From 4ff31d35650afa865198fc1ebf90a1f855dbf512 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 17 Jun 2025 18:45:31 +0000 Subject: [PATCH 26/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index f97af9604..5f024da16 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+1003 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/522a1913e59f1ae52f0445a19c857d1872b3ed5f +version=v1.1.1+1005 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1ad31761718d63ac9fa31afbf35c5b7cc2a6c5b4 From f48c2a0cc65120884182f186d288b1a33a9421ce Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 18 Jun 2025 16:30:09 -0400 Subject: [PATCH 27/46] Update version --- src/main/resources/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5f024da16..30657dcaf 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -15,5 +15,5 @@ # -version=v1.1.1+1005 +version=v1.2+1005 versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1ad31761718d63ac9fa31afbf35c5b7cc2a6c5b4 From ba5f9634b5de6b77206bf05ef8c80f3d92525f9e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 18 Jun 2025 20:30:25 +0000 Subject: [PATCH 28/46] [autocommit] bumping build number --- src/main/resources/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 30657dcaf..8aaa98f5e 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -16,4 +16,4 @@ version=v1.2+1005 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1ad31761718d63ac9fa31afbf35c5b7cc2a6c5b4 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f48c2a0cc65120884182f186d288b1a33a9421ce From d9e1ca087a495659c3f9c2d1ddb2b8daf5c434e4 Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Wed, 18 Jun 2025 23:09:54 -0400 Subject: [PATCH 29/46] Fix version number --- src/main/resources/version.properties | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 8aaa98f5e..55d102c92 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,6 +14,5 @@ # limitations under the License. # - -version=v1.2+1005 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f48c2a0cc65120884182f186d288b1a33a9421ce +version=v1.2.0+1005 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f48c2a0cc65120884182f186d288b1a33a9421ce \ No newline at end of file From 3c86f265fdeabe2b0fb901af1eaaf80f89c05d82 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 19 Jun 2025 03:10:12 +0000 Subject: [PATCH 30/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 55d102c92..c89a627a2 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1005 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f48c2a0cc65120884182f186d288b1a33a9421ce \ No newline at end of file +version=v1.2.0+1009 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d9e1ca087a495659c3f9c2d1ddb2b8daf5c434e4 \ No newline at end of file From 62f8b1862a2328c50369d88fe635d70af8f9325c Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 9 Jul 2025 15:35:30 -0400 Subject: [PATCH 31/46] [BI-2656] - added null check --- .../appendoverwrite/factory/data/OverwrittenData.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index 54a1afe06..ced100205 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -137,9 +137,13 @@ public PendingImportObject constructPendingObservation() { if (!isTimestampMatched()) { // Update the timestamp - DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); - update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + if (timestamp == null) { + update.setObservationTimeStamp(null); + } else { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); + update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + } // Add original timestamp to changelog entry original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); From 5d25aab5338b8ba6bfbcefb9da531b41c3dfa758 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 10 Jul 2025 15:47:26 -0400 Subject: [PATCH 32/46] [BI-2656] - prevent setting timestamp to null --- .../factory/data/OverwrittenData.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index ced100205..9354d638c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -135,15 +135,11 @@ public PendingImportObject constructPendingObservation() { original = observation.getValue(); } - if (!isTimestampMatched()) { + if (!isTimestampMatched() && timestamp != null) { // Update the timestamp - if (timestamp == null) { - update.setObservationTimeStamp(null); - } else { - DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); - update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); - } + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); + update.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); // Add original timestamp to changelog entry original = Optional.ofNullable(original).map(o -> o + " " + observation.getObservationTimeStamp()).orElse(String.valueOf(observation.getObservationTimeStamp())); From 14eeca3a6aa86e6bd95b046886c82be248a0a9e4 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 16 Jul 2025 17:57:41 -0400 Subject: [PATCH 33/46] [BI-2656] - added integration test --- .../importer/ExperimentFileImportTest.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 975c2b083..881768cbf 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1172,6 +1172,82 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { } } + /* + Scenario: + - an experiment was created with observations and timestamps + - do a second upload with additional observations for the experiment, but without the timestamp column + - verify the second set of observations get uploaded successfully + */ + @Test + @SneakyThrows + public void importNewObsAfterFirstExpWithObsAndTimestamps() { + log.debug("importNewObsAfterFirstExpWithObsAndTimestamps"); + List traits = importTestUtils.createTraits(2); + Program program = createProgram("Exp with TS and additional Uploads ", "EXTSAU", "EXTSAU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, Integer.valueOf("1")); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, Integer.valueOf("2025")); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, Integer.valueOf("1")); + newExp.put(Columns.BLOCK_NUM, Integer.valueOf("1")); + newExp.put(Columns.ROW, Integer.valueOf("1")); + newExp.put(Columns.COLUMN, Integer.valueOf("1")); + newExp.put(traits.get(0).getObservationVariableName(), "1"); + newExp.put("TS:" + traits.get(0).getObservationVariableName(), "2019-12-19T12:14:50Z"); + + // In the first upload, only 1 trait should be present. + List initialTraits = List.of(traits.get(0)); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), initialTraits), null, true, client, program, mappingId, newExperimentWorkflowId); + + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + Map newObservation = new HashMap<>(); + newObservation.put(Columns.GERMPLASM_GID, Integer.valueOf("1")); + newObservation.put(Columns.TEST_CHECK, "T"); + newObservation.put(Columns.EXP_TITLE, "Test Exp"); + newObservation.put(Columns.EXP_UNIT, "Plot"); + newObservation.put(Columns.EXP_TYPE, "Phenotyping"); + newObservation.put(Columns.ENV, "New Env"); + newObservation.put(Columns.ENV_LOCATION, "Location A"); + newObservation.put(Columns.ENV_YEAR, Integer.valueOf("2025")); + newObservation.put(Columns.EXP_UNIT_ID, "a-1"); + newObservation.put(Columns.REP_NUM, Integer.valueOf("1")); + newObservation.put(Columns.BLOCK_NUM, Integer.valueOf("1")); + newObservation.put(Columns.ROW, Integer.valueOf("1")); + newObservation.put(Columns.COLUMN, Integer.valueOf("1")); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put(traits.get(0).getObservationVariableName(), "1"); + newObservation.put(traits.get(1).getObservationVariableName(), "1"); + + // Pass overwrite in request body to allow append workflow to work normally. + Map userData = Map.of("overwrite", "true", "overwriteReason", "testing"); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), userData, true, client, program, mappingId, appendOverwriteWorkflowId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newObservation, program, traits); + + } + /* Scenario: - Create an experiment with valid observations. From 35f6303c660076803dc9fc9a7ac9276691264d65 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 16 Jul 2025 18:08:06 -0400 Subject: [PATCH 34/46] [BI-2656] - cleaned up test --- .../importer/ExperimentFileImportTest.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 881768cbf..7a984072c 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1185,19 +1185,19 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { List traits = importTestUtils.createTraits(2); Program program = createProgram("Exp with TS and additional Uploads ", "EXTSAU", "EXTSAU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); - newExp.put(Columns.GERMPLASM_GID, Integer.valueOf("1")); + newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); newExp.put(Columns.EXP_TITLE, "Test Exp"); newExp.put(Columns.EXP_UNIT, "Plot"); newExp.put(Columns.EXP_TYPE, "Phenotyping"); newExp.put(Columns.ENV, "New Env"); newExp.put(Columns.ENV_LOCATION, "Location A"); - newExp.put(Columns.ENV_YEAR, Integer.valueOf("2025")); + newExp.put(Columns.ENV_YEAR, "2025"); newExp.put(Columns.EXP_UNIT_ID, "a-1"); - newExp.put(Columns.REP_NUM, Integer.valueOf("1")); - newExp.put(Columns.BLOCK_NUM, Integer.valueOf("1")); - newExp.put(Columns.ROW, Integer.valueOf("1")); - newExp.put(Columns.COLUMN, Integer.valueOf("1")); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); newExp.put("TS:" + traits.get(0).getObservationVariableName(), "2019-12-19T12:14:50Z"); @@ -1215,24 +1215,24 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { assertTrue(ouIdXref.isPresent()); Map newObservation = new HashMap<>(); - newObservation.put(Columns.GERMPLASM_GID, Integer.valueOf("1")); + newObservation.put(Columns.GERMPLASM_GID, "1"); newObservation.put(Columns.TEST_CHECK, "T"); newObservation.put(Columns.EXP_TITLE, "Test Exp"); newObservation.put(Columns.EXP_UNIT, "Plot"); newObservation.put(Columns.EXP_TYPE, "Phenotyping"); newObservation.put(Columns.ENV, "New Env"); newObservation.put(Columns.ENV_LOCATION, "Location A"); - newObservation.put(Columns.ENV_YEAR, Integer.valueOf("2025")); + newObservation.put(Columns.ENV_YEAR, "2025"); newObservation.put(Columns.EXP_UNIT_ID, "a-1"); - newObservation.put(Columns.REP_NUM, Integer.valueOf("1")); - newObservation.put(Columns.BLOCK_NUM, Integer.valueOf("1")); - newObservation.put(Columns.ROW, Integer.valueOf("1")); - newObservation.put(Columns.COLUMN, Integer.valueOf("1")); + newObservation.put(Columns.REP_NUM, "1"); + newObservation.put(Columns.BLOCK_NUM, "1"); + newObservation.put(Columns.ROW, "1"); + newObservation.put(Columns.COLUMN, "1"); newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "1"); - // Pass overwrite in request body to allow append workflow to work normally. + // Send overwrite parameters in request body to allow the append workflow to work normally. Map userData = Map.of("overwrite", "true", "overwriteReason", "testing"); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), userData, true, client, program, mappingId, appendOverwriteWorkflowId); From 42c9c2ffb6a4021de57bc812ad9902f188862a5f Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 29 Jul 2025 14:47:14 -0400 Subject: [PATCH 35/46] [BI-2656] - handled timestamp edge case prevents erroneous mutated data indication on upload preview --- .../middleware/process/ImportTableProcess.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 770576b67..4a2846fcd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -284,7 +284,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext BrAPIObservation observation = gson.fromJson(gson.toJson(observationByObsHash.get(observationHash)), BrAPIObservation.class); // Is there a change to the prior data? - if (isChanged(cellData, observation, cell.timestamp)) { + if (isChanged(cellData, observation, cell.timestamp, tsColByPheno.containsKey(phenoColumnName))) { // Is prior data protected? /** @@ -394,14 +394,18 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext } } - private boolean isChanged(String cellData, BrAPIObservation observation, String newTimestamp) { + private boolean isChanged(String cellData, BrAPIObservation observation, String newTimestamp, boolean timestampColumnPresent) { if (!cellData.isBlank() && !cellData.equals(observation.getValue())){ return true; } - if (StringUtils.isBlank(newTimestamp)) { - return (observation.getObservationTimeStamp()!=null); + // Only check timestamp if the TS: column was present in the uploaded file. + if (timestampColumnPresent) { + if (StringUtils.isBlank(newTimestamp)) { + return (observation.getObservationTimeStamp()!=null); + } + return !observationService.parseDateTime(newTimestamp).equals(observation.getObservationTimeStamp()); } - return !observationService.parseDateTime(newTimestamp).equals(observation.getObservationTimeStamp()); + return false; } /** From b8ba6f91a1bd3805a027b4678846dcb1abc1fc02 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 30 Jul 2025 14:23:25 +0000 Subject: [PATCH 36/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index c89a627a2..7597e75d0 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1009 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d9e1ca087a495659c3f9c2d1ddb2b8daf5c434e4 \ No newline at end of file +version=v1.2.0+1019 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/35238b11390a08c68a77f6f08778a7178257fd51 \ No newline at end of file From 365c3b26689eb8ff7ce0eb1c15b585aeaf96fb0d Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Fri, 8 Aug 2025 13:01:53 -0400 Subject: [PATCH 37/46] [BI-2656] - added error logging --- .../processors/experiment/service/ObservationService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java index a6ecac2c0..de98202fa 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationService.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapps.importer.services.processors.experiment.service; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.core.BrAPISeason; @@ -42,6 +43,7 @@ import java.util.*; import java.util.stream.Collectors; +@Slf4j @Singleton public class ObservationService { private final ExperimentUtilities experimentUtilities; @@ -115,6 +117,7 @@ public OffsetDateTime parseDateTime(String dateString) { LocalDate localDate = LocalDate.parse(dateString, formatter); return localDate.atStartOfDay().atOffset(ZoneOffset.UTC); } catch (DateTimeParseException ex) { + log.error("Failed to parse timestamp: \"{}\".", dateString); // If both parsing attempts fail, return null return null; } From 77f5a1f8ecf195ea8a98807feda09aa0832e3edd Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 8 Aug 2025 17:31:11 +0000 Subject: [PATCH 38/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 7597e75d0..7cc40ef49 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1019 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/35238b11390a08c68a77f6f08778a7178257fd51 \ No newline at end of file +version=v1.2.0+1025 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2ddc8c9ad16047631e87f5b7f1ab28c7765353c1 \ No newline at end of file From d4d3873f32cd58a9544fcd7756877c95cec2f179 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 12 Aug 2025 12:25:02 -0400 Subject: [PATCH 39/46] [BI-2656] - updated isChanged logic handles append use case when prior data is left blank --- .../appendoverwrite/factory/data/OverwrittenData.java | 2 +- .../middleware/process/ImportTableProcess.java | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java index 9354d638c..f3ce7e530 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/data/OverwrittenData.java @@ -135,7 +135,7 @@ public PendingImportObject constructPendingObservation() { original = observation.getValue(); } - if (!isTimestampMatched() && timestamp != null) { + if (!isTimestampMatched() && StringUtils.isNotBlank(timestamp)) { // Update the timestamp DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; String formattedTimeStampValue = formatter.format(observationService.parseDateTime(timestamp)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 4a2846fcd..f2e74982e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -398,11 +398,8 @@ private boolean isChanged(String cellData, BrAPIObservation observation, String if (!cellData.isBlank() && !cellData.equals(observation.getValue())){ return true; } - // Only check timestamp if the TS: column was present in the uploaded file. - if (timestampColumnPresent) { - if (StringUtils.isBlank(newTimestamp)) { - return (observation.getObservationTimeStamp()!=null); - } + // Only check timestamp if the TS: column was present in the uploaded file and there's a valid timestamp. + if (timestampColumnPresent && !StringUtils.isBlank(newTimestamp)) { return !observationService.parseDateTime(newTimestamp).equals(observation.getObservationTimeStamp()); } return false; From 49d7ef65f83856883b40cd3182c14d388eed7859 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 13 Aug 2025 14:31:55 +0000 Subject: [PATCH 40/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 7cc40ef49..a018e5553 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1025 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2ddc8c9ad16047631e87f5b7f1ab28c7765353c1 \ No newline at end of file +version=v1.2.0+1027 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/64c0cf856256620b0f2e9cabf962a18257730bca \ No newline at end of file From 57aebce2e1feffbed558bbb7b79a7dab5fee04f8 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 28 Aug 2025 15:39:18 +0000 Subject: [PATCH 41/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index a018e5553..5e95f8de2 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1027 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/64c0cf856256620b0f2e9cabf962a18257730bca \ No newline at end of file +version=v1.2.0+1049 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/49d7ef65f83856883b40cd3182c14d388eed7859 \ No newline at end of file From c1ea5ac9187396731b0a79d53d1369f526022a26 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 28 Aug 2025 16:20:08 +0000 Subject: [PATCH 42/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 5e95f8de2..40e236110 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1049 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/49d7ef65f83856883b40cd3182c14d388eed7859 \ No newline at end of file +version=v1.2.0+1051 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a137a7c034e4309f1e0c235161831be78141fe52 \ No newline at end of file From 91d08483b2fdb844937e991145e88a7dfebd08f7 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 28 Aug 2025 15:52:41 -0400 Subject: [PATCH 43/46] fix merge issue --- .../brapi/v2/BrAPIGermplasmController.java | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java index 85646f3d8..850ed8138 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java @@ -221,25 +221,6 @@ public HttpResponse>>> getGermplasm( return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, "Error parsing requested date format"); } } - @Get("/programs/{programId}/germplasm/lists/{listDbId}/export{?fileExtension}") - @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") - @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) - public HttpResponse germplasmListExport( - @PathVariable("programId") UUID programId, @PathVariable("listDbId") String listDbId, @QueryValue(defaultValue = "XLSX") String fileExtension) { - String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; - try { - FileType extension = Enum.valueOf(FileType.class, fileExtension); - DownloadFile germplasmListFile = germplasmService.exportGermplasmList(programId, listDbId, extension); - HttpResponse germplasmListExport = HttpResponse.ok(germplasmListFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+germplasmListFile.getFileName()+extension.getExtension()); - return germplasmListExport; - } - catch (Exception e) { - log.info(e.getMessage(), e); - e.printStackTrace(); - HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); - return response; - } - } @Get("/programs/{programId}/germplasm/lists/{listDbId}/export{?fileExtension}") @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") From b769afce5fe9dae801c0469bf410d1d7c494420c Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 28 Aug 2025 19:53:33 +0000 Subject: [PATCH 44/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 40e236110..1770a8fce 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1051 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/a137a7c034e4309f1e0c235161831be78141fe52 \ No newline at end of file +version=v1.2.0+1053 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/91d08483b2fdb844937e991145e88a7dfebd08f7 \ No newline at end of file From 0a030718b0b626689e4e19a7acf17be481ac6add Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 28 Aug 2025 16:36:16 -0400 Subject: [PATCH 45/46] fix test --- .../brapps/importer/ExperimentFileImportTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 86437bc5e..8d31d62b7 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -1204,7 +1204,7 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { // In the first upload, only 1 trait should be present. List initialTraits = List.of(traits.get(0)); - importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), initialTraits), null, true, client, program, mappingId, newExperimentWorkflowId); + importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), initialTraits, false), null, true, client, program, mappingId, newExperimentWorkflowId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -1229,13 +1229,13 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "1"); // Send overwrite parameters in request body to allow the append workflow to work normally. Map userData = Map.of("overwrite", "true", "overwriteReason", "testing"); - JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), userData, true, client, program, mappingId, appendOverwriteWorkflowId); + JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true), userData, true, client, program, mappingId, appendOverwriteWorkflowId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); From 0c250e557f83b92914e9042e88bc4c7242c13426 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 28 Aug 2025 20:36:36 +0000 Subject: [PATCH 46/46] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 1770a8fce..56ce1b70c 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v1.2.0+1053 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/91d08483b2fdb844937e991145e88a7dfebd08f7 \ No newline at end of file +version=v1.2.0+1055 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0a030718b0b626689e4e19a7acf17be481ac6add \ No newline at end of file