From b453d11b58c6da6a91469a21cfc12049186dbe1f Mon Sep 17 00:00:00 2001 From: waleed Date: Tue, 30 Dec 2025 18:42:51 -0800 Subject: [PATCH 1/9] improvement(kb): removed zustand cache syncing in kb, added chunk text tokenizer --- .../create-chunk-modal/create-chunk-modal.tsx | 14 +- .../delete-chunk-modal/delete-chunk-modal.tsx | 16 +- .../document-tags-modal.tsx | 15 +- .../edit-chunk-modal/edit-chunk-modal.tsx | 154 ++- .../knowledge/[id]/[documentId]/document.tsx | 101 +- .../[workspaceId]/knowledge/[id]/base.tsx | 26 +- .../add-documents-modal.tsx | 9 +- .../create-base-modal/create-base-modal.tsx | 29 +- .../knowledge-header/knowledge-header.tsx | 13 +- .../knowledge/hooks/use-knowledge-upload.ts | 8 +- .../[workspaceId]/knowledge/knowledge.tsx | 32 +- .../[workspaceId]/knowledge/utils/sort.ts | 2 +- .../knowledge-base-selector.tsx | 4 +- .../components/team-members/team-members.tsx | 32 +- .../components/user-avatar/user-avatar.tsx | 66 -- apps/sim/hooks/queries/knowledge.ts | 4 +- apps/sim/hooks/use-knowledge-base-name.ts | 25 +- apps/sim/hooks/use-knowledge.ts | 713 ++++++-------- apps/sim/lib/knowledge/types.ts | 112 +++ apps/sim/lib/tokenization/estimators.ts | 34 + apps/sim/stores/knowledge/store.ts | 923 ------------------ 21 files changed, 690 insertions(+), 1642 deletions(-) delete mode 100644 apps/sim/components/user-avatar/user-avatar.tsx delete mode 100644 apps/sim/stores/knowledge/store.ts diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/create-chunk-modal/create-chunk-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/create-chunk-modal/create-chunk-modal.tsx index 876da5f081..c8c49998b2 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/create-chunk-modal/create-chunk-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/create-chunk-modal/create-chunk-modal.tsx @@ -2,6 +2,7 @@ import { useRef, useState } from 'react' import { createLogger } from '@sim/logger' +import { useQueryClient } from '@tanstack/react-query' import { AlertCircle } from 'lucide-react' import { Button, @@ -13,7 +14,8 @@ import { ModalHeader, Textarea, } from '@/components/emcn' -import type { ChunkData, DocumentData } from '@/stores/knowledge/store' +import type { DocumentData } from '@/lib/knowledge/types' +import { knowledgeKeys } from '@/hooks/queries/knowledge' const logger = createLogger('CreateChunkModal') @@ -22,7 +24,6 @@ interface CreateChunkModalProps { onOpenChange: (open: boolean) => void document: DocumentData | null knowledgeBaseId: string - onChunkCreated?: (chunk: ChunkData) => void } export function CreateChunkModal({ @@ -30,8 +31,8 @@ export function CreateChunkModal({ onOpenChange, document, knowledgeBaseId, - onChunkCreated, }: CreateChunkModalProps) { + const queryClient = useQueryClient() const [content, setContent] = useState('') const [isCreating, setIsCreating] = useState(false) const [error, setError] = useState(null) @@ -77,9 +78,9 @@ export function CreateChunkModal({ if (result.success && result.data) { logger.info('Chunk created successfully:', result.data.id) - if (onChunkCreated) { - onChunkCreated(result.data) - } + await queryClient.invalidateQueries({ + queryKey: knowledgeKeys.detail(knowledgeBaseId), + }) onClose() } else { @@ -96,7 +97,6 @@ export function CreateChunkModal({ const onClose = () => { onOpenChange(false) - // Reset form state when modal closes setContent('') setError(null) setShowUnsavedChangesAlert(false) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/delete-chunk-modal/delete-chunk-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/delete-chunk-modal/delete-chunk-modal.tsx index c9560493ed..7d69f5e146 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/delete-chunk-modal/delete-chunk-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/delete-chunk-modal/delete-chunk-modal.tsx @@ -2,8 +2,10 @@ import { useState } from 'react' import { createLogger } from '@sim/logger' +import { useQueryClient } from '@tanstack/react-query' import { Button, Modal, ModalBody, ModalContent, ModalFooter, ModalHeader } from '@/components/emcn' -import type { ChunkData } from '@/stores/knowledge/store' +import type { ChunkData } from '@/lib/knowledge/types' +import { knowledgeKeys } from '@/hooks/queries/knowledge' const logger = createLogger('DeleteChunkModal') @@ -13,7 +15,6 @@ interface DeleteChunkModalProps { documentId: string isOpen: boolean onClose: () => void - onChunkDeleted?: () => void } export function DeleteChunkModal({ @@ -22,8 +23,8 @@ export function DeleteChunkModal({ documentId, isOpen, onClose, - onChunkDeleted, }: DeleteChunkModalProps) { + const queryClient = useQueryClient() const [isDeleting, setIsDeleting] = useState(false) const handleDeleteChunk = async () => { @@ -47,16 +48,17 @@ export function DeleteChunkModal({ if (result.success) { logger.info('Chunk deleted successfully:', chunk.id) - if (onChunkDeleted) { - onChunkDeleted() - } + + await queryClient.invalidateQueries({ + queryKey: knowledgeKeys.detail(knowledgeBaseId), + }) + onClose() } else { throw new Error(result.error || 'Failed to delete chunk') } } catch (err) { logger.error('Error deleting chunk:', err) - // You might want to show an error state here } finally { setIsDeleting(false) } diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx index 9bbd328bd5..fda11582ae 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/document-tags-modal/document-tags-modal.tsx @@ -18,13 +18,13 @@ import { import { cn } from '@/lib/core/utils/cn' import { ALL_TAG_SLOTS, type AllTagSlot, MAX_TAG_SLOTS } from '@/lib/knowledge/constants' import type { DocumentTag } from '@/lib/knowledge/tags/types' +import type { DocumentData } from '@/lib/knowledge/types' import { type TagDefinition, useKnowledgeBaseTagDefinitions, } from '@/hooks/use-knowledge-base-tag-definitions' import { useNextAvailableSlot } from '@/hooks/use-next-available-slot' import { type TagDefinitionInput, useTagDefinitions } from '@/hooks/use-tag-definitions' -import { type DocumentData, useKnowledgeStore } from '@/stores/knowledge/store' const logger = createLogger('DocumentTagsModal') @@ -93,8 +93,6 @@ export function DocumentTagsModal({ documentData, onDocumentUpdate, }: DocumentTagsModalProps) { - const { updateDocument: updateDocumentInStore } = useKnowledgeStore() - const documentTagHook = useTagDefinitions(knowledgeBaseId, documentId) const kbTagHook = useKnowledgeBaseTagDefinitions(knowledgeBaseId) const { getNextAvailableSlot: getServerNextSlot } = useNextAvailableSlot(knowledgeBaseId) @@ -171,23 +169,14 @@ export function DocumentTagsModal({ throw new Error('Failed to update document tags') } - updateDocumentInStore(knowledgeBaseId, documentId, tagData as Record) onDocumentUpdate?.(tagData as Record) - await fetchTagDefinitions() } catch (error) { logger.error('Error updating document tags:', error) throw error } }, - [ - documentData, - knowledgeBaseId, - documentId, - updateDocumentInStore, - fetchTagDefinitions, - onDocumentUpdate, - ] + [documentData, knowledgeBaseId, documentId, fetchTagDefinitions, onDocumentUpdate] ) const handleRemoveTag = async (index: number) => { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/edit-chunk-modal/edit-chunk-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/edit-chunk-modal/edit-chunk-modal.tsx index 389dedb36e..f7edfbce7f 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/edit-chunk-modal/edit-chunk-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/edit-chunk-modal/edit-chunk-modal.tsx @@ -1,9 +1,9 @@ 'use client' -import { useEffect, useState } from 'react' -import * as DialogPrimitive from '@radix-ui/react-dialog' +import { useEffect, useMemo, useRef, useState } from 'react' import { createLogger } from '@sim/logger' -import { AlertCircle, ChevronDown, ChevronUp, X } from 'lucide-react' +import { useQueryClient } from '@tanstack/react-query' +import { AlertCircle, ChevronDown, ChevronUp } from 'lucide-react' import { Button, Label, @@ -12,11 +12,14 @@ import { ModalContent, ModalFooter, ModalHeader, + Switch, Textarea, Tooltip, } from '@/components/emcn' +import type { ChunkData, DocumentData } from '@/lib/knowledge/types' +import { getAccurateTokenCount, getTokenStrings } from '@/lib/tokenization/estimators' import { useUserPermissionsContext } from '@/app/workspace/[workspaceId]/providers/workspace-permissions-provider' -import type { ChunkData, DocumentData } from '@/stores/knowledge/store' +import { knowledgeKeys } from '@/hooks/queries/knowledge' const logger = createLogger('EditChunkModal') @@ -26,13 +29,14 @@ interface EditChunkModalProps { knowledgeBaseId: string isOpen: boolean onClose: () => void - onChunkUpdate?: (updatedChunk: ChunkData) => void - // New props for navigation + // Props for navigation allChunks?: ChunkData[] currentPage?: number totalPages?: number onNavigateToChunk?: (chunk: ChunkData) => void onNavigateToPage?: (page: number, selectChunk: 'first' | 'last') => Promise + /** Max chunk size in tokens from knowledge base config */ + maxChunkSize?: number } export function EditChunkModal({ @@ -41,13 +45,14 @@ export function EditChunkModal({ knowledgeBaseId, isOpen, onClose, - onChunkUpdate, allChunks = [], currentPage = 1, totalPages = 1, onNavigateToChunk, onNavigateToPage, + maxChunkSize = 1024, }: EditChunkModalProps) { + const queryClient = useQueryClient() const userPermissions = useUserPermissionsContext() const [editedContent, setEditedContent] = useState(chunk?.content || '') const [isSaving, setIsSaving] = useState(false) @@ -55,9 +60,56 @@ export function EditChunkModal({ const [error, setError] = useState(null) const [showUnsavedChangesAlert, setShowUnsavedChangesAlert] = useState(false) const [pendingNavigation, setPendingNavigation] = useState<(() => void) | null>(null) + const [tokenizerOn, setTokenizerOn] = useState(false) + const [scrollTop, setScrollTop] = useState(0) + const textareaRef = useRef(null) const hasUnsavedChanges = editedContent !== (chunk?.content || '') + const tokenStrings = useMemo(() => { + if (!tokenizerOn || !editedContent) return [] + return getTokenStrings(editedContent) + }, [editedContent, tokenizerOn]) + + const tokenCount = useMemo(() => { + if (!editedContent) return 0 + if (tokenizerOn) return tokenStrings.length + return getAccurateTokenCount(editedContent) + }, [editedContent, tokenizerOn, tokenStrings]) + + const TOKEN_BG_COLORS = [ + 'rgba(185, 28, 28, 0.5)', // Red + 'rgba(194, 65, 12, 0.5)', // Orange + 'rgba(161, 98, 7, 0.5)', // Amber + 'rgba(77, 124, 15, 0.5)', // Lime + 'rgba(21, 128, 61, 0.5)', // Green + 'rgba(15, 118, 110, 0.5)', // Teal + 'rgba(3, 105, 161, 0.5)', // Sky + 'rgba(29, 78, 216, 0.5)', // Blue + 'rgba(109, 40, 217, 0.5)', // Violet + 'rgba(162, 28, 175, 0.5)', // Fuchsia + ] + + const getTokenBgColor = (index: number): string => { + return TOKEN_BG_COLORS[index % TOKEN_BG_COLORS.length] + } + + const handleScroll = () => { + if (textareaRef.current) { + setScrollTop(textareaRef.current.scrollTop) + } + } + + useEffect(() => { + if (tokenizerOn && textareaRef.current) { + requestAnimationFrame(() => { + if (textareaRef.current) { + setScrollTop(textareaRef.current.scrollTop) + } + }) + } + }, [editedContent, tokenizerOn]) + useEffect(() => { if (chunk?.content) { setEditedContent(chunk.content) @@ -96,8 +148,10 @@ export function EditChunkModal({ const result = await response.json() - if (result.success && onChunkUpdate) { - onChunkUpdate(result.data) + if (result.success) { + await queryClient.invalidateQueries({ + queryKey: knowledgeKeys.detail(knowledgeBaseId), + }) } } catch (err) { logger.error('Error updating chunk:', err) @@ -125,7 +179,6 @@ export function EditChunkModal({ const nextChunk = allChunks[currentChunkIndex + 1] onNavigateToChunk?.(nextChunk) } else if (currentPage < totalPages) { - // Load next page and navigate to first chunk await onNavigateToPage?.(currentPage + 1, 'first') } } @@ -173,12 +226,9 @@ export function EditChunkModal({ <> -
- - Edit Chunk #{chunk.chunkIndex} - - -
+ +
+ Edit Chunk #{chunk.chunkIndex} {/* Navigation Controls */}
@@ -225,17 +275,8 @@ export function EditChunkModal({
- -
-
+
@@ -250,17 +291,56 @@ export function EditChunkModal({ {/* Content Input Section */} -