From 0533601642c1ab3a07261468b254630b4a0aeba6 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 30 Dec 2025 12:25:01 +0100 Subject: [PATCH 1/5] fix: optimizations for query members --- .../data-access-layer/src/members/base.ts | 2 +- .../src/members/queryBuilder.ts | 25 ++++++++++++++----- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/services/libs/data-access-layer/src/members/base.ts b/services/libs/data-access-layer/src/members/base.ts index 88d609e22d..9fc459e84b 100644 --- a/services/libs/data-access-layer/src/members/base.ts +++ b/services/libs/data-access-layer/src/members/base.ts @@ -120,7 +120,7 @@ const QUERY_FILTER_COLUMN_MAP: Map 'isBot' ->> 'default')::BOOLEAN, FALSE)` }], diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 7a8a9cfbdb..1059a0b32e 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -309,7 +309,7 @@ const buildActivityCountOptimizedQuery = ({ const ctes: string[] = [] if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) - const searchJoinForTopMembers = searchConfig.cte +const searchJoinForFiltering = searchConfig.cte ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` : '' @@ -317,28 +317,41 @@ const buildActivityCountOptimizedQuery = ({ const oversampleMultiplier = hasNonIdMemberFields ? 10 : 1 // 10x oversampling for m.* filters const totalNeeded = Math.min(baseNeeded * oversampleMultiplier, 50000) // Cap at 50k + const prefetchLimit = Math.min(totalNeeded * 10, 50000) + ctes.push( ` - top_members AS ( + top_msa AS ( SELECT msa."memberId", msa."activityCount" FROM "memberSegmentsAgg" msa - INNER JOIN members m ON m.id = msa."memberId" - ${searchJoinForTopMembers} WHERE msa."segmentId" = $(segmentId) - AND (${filterString}) ORDER BY msa."activityCount" ${direction} NULLS LAST + LIMIT ${prefetchLimit} + ), + top_members AS ( + SELECT + t."memberId", + t."activityCount" + FROM top_msa t + INNER JOIN members m ON m.id = t."memberId" + ${searchJoinForFiltering} + WHERE + (${filterString}) + ORDER BY + t."activityCount" ${direction} NULLS LAST LIMIT ${totalNeeded} ) `.trim(), ) + + const withClause = `WITH ${ctes.join(',\n')}` - // Outer query is much simpler now - no more filtering needed return ` ${withClause} SELECT ${fields} From b16178002367a9d4baed6266c2991bdc15a146dc Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 30 Dec 2025 12:28:05 +0100 Subject: [PATCH 2/5] fix: lint --- services/libs/data-access-layer/src/members/queryBuilder.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index 1059a0b32e..adbb9f9d5b 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -309,7 +309,7 @@ const buildActivityCountOptimizedQuery = ({ const ctes: string[] = [] if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) -const searchJoinForFiltering = searchConfig.cte + const searchJoinForFiltering = searchConfig.cte ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` : '' @@ -317,7 +317,7 @@ const searchJoinForFiltering = searchConfig.cte const oversampleMultiplier = hasNonIdMemberFields ? 10 : 1 // 10x oversampling for m.* filters const totalNeeded = Math.min(baseNeeded * oversampleMultiplier, 50000) // Cap at 50k - const prefetchLimit = Math.min(totalNeeded * 10, 50000) + const prefetchLimit = Math.min(totalNeeded * 10, 50000) ctes.push( ` @@ -348,8 +348,6 @@ const searchJoinForFiltering = searchConfig.cte `.trim(), ) - - const withClause = `WITH ${ctes.join(',\n')}` return ` From 3fd0b543acc2491e6ef2dc4811140b7cb8daf6e0 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Tue, 30 Dec 2025 14:51:11 +0100 Subject: [PATCH 3/5] fix: avoid double scan of the same table --- services/libs/data-access-layer/src/members/queryBuilder.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index adbb9f9d5b..a1486fae46 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -356,13 +356,10 @@ const buildActivityCountOptimizedQuery = ({ FROM top_members tm JOIN members m ON m.id = tm."memberId" - INNER JOIN "memberSegmentsAgg" msa - ON msa."memberId" = m.id - AND msa."segmentId" = $(segmentId) LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id ORDER BY - msa."activityCount" ${direction} NULLS LAST + tm."activityCount" ${direction} NULLS LAST LIMIT ${limit} OFFSET ${offset} `.trim() From d0e87c34fbf284da9b0c8d3f33a172a301000610 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Wed, 31 Dec 2025 11:15:59 +0100 Subject: [PATCH 4/5] fix: add inner join --- .../data-access-layer/src/members/queryBuilder.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index a1486fae46..c8b4255909 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -309,6 +309,9 @@ const buildActivityCountOptimizedQuery = ({ const ctes: string[] = [] if (searchConfig.cte) ctes.push(searchConfig.cte.trim()) + // We must keep msa available in the outer SELECT if "fields" references msa.* + const needsMsaInOuterSelect = /\bmsa\./.test(fields) + const searchJoinForFiltering = searchConfig.cte ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` : '' @@ -350,12 +353,21 @@ const buildActivityCountOptimizedQuery = ({ const withClause = `WITH ${ctes.join(',\n')}` + const msaOuterJoin = needsMsaInOuterSelect + ? ` + INNER JOIN "memberSegmentsAgg" msa + ON msa."memberId" = m.id + AND msa."segmentId" = $(segmentId) + ` + : '' + return ` ${withClause} SELECT ${fields} FROM top_members tm JOIN members m ON m.id = tm."memberId" + ${msaOuterJoin} LEFT JOIN "memberEnrichments" me ON me."memberId" = m.id ORDER BY From a31edb575ddaf5fe3d5957c573adfe6f91a61f67 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Wed, 31 Dec 2025 11:42:02 +0100 Subject: [PATCH 5/5] fix: for review --- services/libs/data-access-layer/src/members/queryBuilder.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/libs/data-access-layer/src/members/queryBuilder.ts b/services/libs/data-access-layer/src/members/queryBuilder.ts index c8b4255909..e434f4a2bb 100644 --- a/services/libs/data-access-layer/src/members/queryBuilder.ts +++ b/services/libs/data-access-layer/src/members/queryBuilder.ts @@ -311,6 +311,7 @@ const buildActivityCountOptimizedQuery = ({ // We must keep msa available in the outer SELECT if "fields" references msa.* const needsMsaInOuterSelect = /\bmsa\./.test(fields) + const filterNeedsMsa = /\bmsa\./.test(filterString) const searchJoinForFiltering = searchConfig.cte ? `\n INNER JOIN member_search ms ON ms."memberId" = msa."memberId"` @@ -322,6 +323,10 @@ const buildActivityCountOptimizedQuery = ({ const prefetchLimit = Math.min(totalNeeded * 10, 50000) + const msaJoinForFiltering = filterNeedsMsa + ? `\n INNER JOIN "memberSegmentsAgg" msa ON msa."memberId" = m.id AND msa."segmentId" = $(segmentId)` + : '' + ctes.push( ` top_msa AS ( @@ -341,6 +346,7 @@ const buildActivityCountOptimizedQuery = ({ t."activityCount" FROM top_msa t INNER JOIN members m ON m.id = t."memberId" + ${msaJoinForFiltering} ${searchJoinForFiltering} WHERE (${filterString})