Skip to content

Commit e1d5e38

Browse files
waleedlatif1waleedlatif1
authored
fix(chunks): instantaneous search + server side searching instead of client-side (#940)
* fix(chunks): instantaneous search + server side searching instead of client-side * add knowledge tags component to sidebar, replace old knowledge tags UI * add types, remove extraneous comments * added knowledge-base level tag definitions viewer, ability to create/delete slots in sidebar and respective routes * ui * fix stale tag issue * use logger
1 parent 3c7b3e1 commit e1d5e38

File tree

11 files changed

+2185
-497
lines changed

11 files changed

+2185
-497
lines changed

‎apps/sim/app/api/knowledge/[id]/tag-definitions/[tagId]/route.ts‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
import { randomUUID } from 'crypto'
2+
import { and, eq, isNotNull } from 'drizzle-orm'
3+
import { type NextRequest, NextResponse } from 'next/server'
4+
import { getSession } from '@/lib/auth'
5+
import { createLogger } from '@/lib/logs/console/logger'
6+
import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
7+
import { db } from '@/db'
8+
import { document, embedding, knowledgeBaseTagDefinitions } from '@/db/schema'
9+
10+
export const dynamic = 'force-dynamic'
11+
12+
const logger = createLogger('TagDefinitionAPI')
13+
14+
// DELETE /api/knowledge/[id]/tag-definitions/[tagId] - Delete a tag definition
15+
export async function DELETE(
16+
req: NextRequest,
17+
{ params }: { params: Promise<{ id: string; tagId: string }> }
18+
) {
19+
const requestId = randomUUID().slice(0, 8)
20+
const { id: knowledgeBaseId, tagId } = await params
21+
22+
try {
23+
logger.info(
24+
`[${requestId}] Deleting tag definition ${tagId} from knowledge base ${knowledgeBaseId}`
25+
)
26+
27+
const session = await getSession()
28+
if (!session?.user?.id) {
29+
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
30+
}
31+
32+
// Check if user has access to the knowledge base
33+
const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, session.user.id)
34+
if (!accessCheck.hasAccess) {
35+
return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
36+
}
37+
38+
// Get the tag definition to find which slot it uses
39+
const tagDefinition = await db
40+
.select({
41+
id: knowledgeBaseTagDefinitions.id,
42+
tagSlot: knowledgeBaseTagDefinitions.tagSlot,
43+
displayName: knowledgeBaseTagDefinitions.displayName,
44+
})
45+
.from(knowledgeBaseTagDefinitions)
46+
.where(
47+
and(
48+
eq(knowledgeBaseTagDefinitions.id, tagId),
49+
eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId)
50+
)
51+
)
52+
.limit(1)
53+
54+
if (tagDefinition.length === 0) {
55+
return NextResponse.json({ error: 'Tag definition not found' }, { status: 404 })
56+
}
57+
58+
const tagDef = tagDefinition[0]
59+
60+
// Delete the tag definition and clear all document tags in a transaction
61+
await db.transaction(async (tx) => {
62+
logger.info(`[${requestId}] Starting transaction to delete ${tagDef.tagSlot}`)
63+
64+
try {
65+
// Clear the tag from documents that actually have this tag set
66+
logger.info(`[${requestId}] Clearing tag from documents...`)
67+
await tx
68+
.update(document)
69+
.set({ [tagDef.tagSlot]: null })
70+
.where(
71+
and(
72+
eq(document.knowledgeBaseId, knowledgeBaseId),
73+
isNotNull(document[tagDef.tagSlot as keyof typeof document.$inferSelect])
74+
)
75+
)
76+
77+
logger.info(`[${requestId}] Documents updated successfully`)
78+
79+
// Clear the tag from embeddings that actually have this tag set
80+
logger.info(`[${requestId}] Clearing tag from embeddings...`)
81+
await tx
82+
.update(embedding)
83+
.set({ [tagDef.tagSlot]: null })
84+
.where(
85+
and(
86+
eq(embedding.knowledgeBaseId, knowledgeBaseId),
87+
isNotNull(embedding[tagDef.tagSlot as keyof typeof embedding.$inferSelect])
88+
)
89+
)
90+
91+
logger.info(`[${requestId}] Embeddings updated successfully`)
92+
93+
// Delete the tag definition
94+
logger.info(`[${requestId}] Deleting tag definition...`)
95+
await tx
96+
.delete(knowledgeBaseTagDefinitions)
97+
.where(eq(knowledgeBaseTagDefinitions.id, tagId))
98+
99+
logger.info(`[${requestId}] Tag definition deleted successfully`)
100+
} catch (error) {
101+
logger.error(`[${requestId}] Error in transaction:`, error)
102+
throw error
103+
}
104+
})
105+
106+
logger.info(
107+
`[${requestId}] Successfully deleted tag definition ${tagDef.displayName} (${tagDef.tagSlot})`
108+
)
109+
110+
return NextResponse.json({
111+
success: true,
112+
message: `Tag definition "${tagDef.displayName}" deleted successfully`,
113+
})
114+
} catch (error) {
115+
logger.error(`[${requestId}] Error deleting tag definition`, error)
116+
return NextResponse.json({ error: 'Failed to delete tag definition' }, { status: 500 })
117+
}
118+
}

‎apps/sim/app/api/knowledge/[id]/tag-definitions/route.ts‎

Lines changed: 87 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { randomUUID } from 'crypto'
2-
import { eq } from 'drizzle-orm'
2+
import { and, eq } from 'drizzle-orm'
33
import { type NextRequest, NextResponse } from 'next/server'
44
import { getSession } from '@/lib/auth'
55
import { createLogger } from '@/lib/logs/console/logger'
@@ -55,3 +55,89 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
5555
return NextResponse.json({ error: 'Failed to get tag definitions' }, { status: 500 })
5656
}
5757
}
58+
59+
// POST /api/knowledge/[id]/tag-definitions - Create a new tag definition
60+
export async function POST(req: NextRequest, { params }: { params: Promise<{ id: string }> }) {
61+
const requestId = randomUUID().slice(0, 8)
62+
const { id: knowledgeBaseId } = await params
63+
64+
try {
65+
logger.info(`[${requestId}] Creating tag definition for knowledge base ${knowledgeBaseId}`)
66+
67+
const session = await getSession()
68+
if (!session?.user?.id) {
69+
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
70+
}
71+
72+
// Check if user has access to the knowledge base
73+
const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, session.user.id)
74+
if (!accessCheck.hasAccess) {
75+
return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
76+
}
77+
78+
const body = await req.json()
79+
const { tagSlot, displayName, fieldType } = body
80+
81+
if (!tagSlot || !displayName || !fieldType) {
82+
return NextResponse.json(
83+
{ error: 'tagSlot, displayName, and fieldType are required' },
84+
{ status: 400 }
85+
)
86+
}
87+
88+
// Check if tag slot is already used
89+
const existingTag = await db
90+
.select()
91+
.from(knowledgeBaseTagDefinitions)
92+
.where(
93+
and(
94+
eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId),
95+
eq(knowledgeBaseTagDefinitions.tagSlot, tagSlot)
96+
)
97+
)
98+
.limit(1)
99+
100+
if (existingTag.length > 0) {
101+
return NextResponse.json({ error: 'Tag slot is already in use' }, { status: 409 })
102+
}
103+
104+
// Check if display name is already used
105+
const existingName = await db
106+
.select()
107+
.from(knowledgeBaseTagDefinitions)
108+
.where(
109+
and(
110+
eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId),
111+
eq(knowledgeBaseTagDefinitions.displayName, displayName)
112+
)
113+
)
114+
.limit(1)
115+
116+
if (existingName.length > 0) {
117+
return NextResponse.json({ error: 'Tag name is already in use' }, { status: 409 })
118+
}
119+
120+
// Create the new tag definition
121+
const newTagDefinition = {
122+
id: randomUUID(),
123+
knowledgeBaseId,
124+
tagSlot,
125+
displayName,
126+
fieldType,
127+
createdAt: new Date(),
128+
updatedAt: new Date(),
129+
}
130+
131+
await db.insert(knowledgeBaseTagDefinitions).values(newTagDefinition)
132+
133+
logger.info(`[${requestId}] Successfully created tag definition ${displayName} (${tagSlot})`)
134+
135+
return NextResponse.json({
136+
success: true,
137+
data: newTagDefinition,
138+
})
139+
} catch (error) {
140+
logger.error(`[${requestId}] Error creating tag definition`, error)
141+
return NextResponse.json({ error: 'Failed to create tag definition' }, { status: 500 })
142+
}
143+
}

‎apps/sim/app/api/knowledge/[id]/tag-usage/route.ts‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
import { randomUUID } from 'crypto'
2+
import { and, eq, isNotNull } from 'drizzle-orm'
3+
import { type NextRequest, NextResponse } from 'next/server'
4+
import { getSession } from '@/lib/auth'
5+
import { createLogger } from '@/lib/logs/console/logger'
6+
import { checkKnowledgeBaseAccess } from '@/app/api/knowledge/utils'
7+
import { db } from '@/db'
8+
import { document, knowledgeBaseTagDefinitions } from '@/db/schema'
9+
10+
export const dynamic = 'force-dynamic'
11+
12+
const logger = createLogger('TagUsageAPI')
13+
14+
// GET /api/knowledge/[id]/tag-usage - Get usage statistics for all tag definitions
15+
export async function GET(req: NextRequest, { params }: { params: Promise<{ id: string }> }) {
16+
const requestId = randomUUID().slice(0, 8)
17+
const { id: knowledgeBaseId } = await params
18+
19+
try {
20+
logger.info(`[${requestId}] Getting tag usage statistics for knowledge base ${knowledgeBaseId}`)
21+
22+
const session = await getSession()
23+
if (!session?.user?.id) {
24+
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
25+
}
26+
27+
// Check if user has access to the knowledge base
28+
const accessCheck = await checkKnowledgeBaseAccess(knowledgeBaseId, session.user.id)
29+
if (!accessCheck.hasAccess) {
30+
return NextResponse.json({ error: 'Forbidden' }, { status: 403 })
31+
}
32+
33+
// Get all tag definitions for the knowledge base
34+
const tagDefinitions = await db
35+
.select({
36+
id: knowledgeBaseTagDefinitions.id,
37+
tagSlot: knowledgeBaseTagDefinitions.tagSlot,
38+
displayName: knowledgeBaseTagDefinitions.displayName,
39+
})
40+
.from(knowledgeBaseTagDefinitions)
41+
.where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
42+
43+
// Get usage statistics for each tag definition
44+
const usageStats = await Promise.all(
45+
tagDefinitions.map(async (tagDef) => {
46+
// Count documents using this tag slot
47+
const tagSlotColumn = tagDef.tagSlot as keyof typeof document.$inferSelect
48+
49+
const documentsWithTag = await db
50+
.select({
51+
id: document.id,
52+
filename: document.filename,
53+
[tagDef.tagSlot]: document[tagSlotColumn as keyof typeof document.$inferSelect] as any,
54+
})
55+
.from(document)
56+
.where(
57+
and(
58+
eq(document.knowledgeBaseId, knowledgeBaseId),
59+
isNotNull(document[tagSlotColumn as keyof typeof document.$inferSelect])
60+
)
61+
)
62+
63+
return {
64+
tagName: tagDef.displayName,
65+
tagSlot: tagDef.tagSlot,
66+
documentCount: documentsWithTag.length,
67+
documents: documentsWithTag.map((doc) => ({
68+
id: doc.id,
69+
name: doc.filename,
70+
tagValue: doc[tagDef.tagSlot],
71+
})),
72+
}
73+
})
74+
)
75+
76+
logger.info(
77+
`[${requestId}] Retrieved usage statistics for ${tagDefinitions.length} tag definitions`
78+
)
79+
80+
return NextResponse.json({
81+
success: true,
82+
data: usageStats,
83+
})
84+
} catch (error) {
85+
logger.error(`[${requestId}] Error getting tag usage statistics`, error)
86+
return NextResponse.json({ error: 'Failed to get tag usage statistics' }, { status: 500 })
87+
}
88+
}

0 commit comments

Comments
 (0)