diff --git a/.idea/jsLibraryMappings.xml b/.idea/jsLibraryMappings.xml new file mode 100644 index 00000000..d23208fb --- /dev/null +++ b/.idea/jsLibraryMappings.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Resources/examples_v3.json b/Resources/examples_v3.json deleted file mode 100755 index e670e4d7..00000000 --- a/Resources/examples_v3.json +++ /dev/null @@ -1,283 +0,0 @@ -[ - { - "title": "Knowledge", - "accessLink": "https://github.com/KnowledgeCanvas/knowledge", - "topics": [ - "2nd Brain", - "Learning", - "Research", - "Chat" - ] - }, - { - "title": "Knowledge - Chrome Extension", - "accessLink": "https://github.com/KnowledgeCanvas/knowledge-canvas-extensions", - "topics": [ - "Browser", - "Learning", - "Research", - "Chrome Extension" - ] - }, - { - "title": "Knowledge - Wiki", - "accessLink": "https://github.com/KnowledgeCanvas/knowledge/wiki", - "topics": [ - "Wiki", - "Learning", - "Research", - "Knowledge" - ] - }, - { - "title": "Twitter's Recommendation Algorithm", - "accessLink": "https://blog.twitter.com/engineering/en_us/topics/open-source/2023/twitter-recommendation-algorithm", - "topics": [ - "Recommendation System", - "Models", - "Algorithms", - "Pipeline" - ] - }, - { - "title": "The Italian farmers saving an ancient fruit with solar power", - "accessLink": "https://www.bbc.com/future/article/20230424-how-agrivoltaics-helped-save-italys-citron-and-lemon-trees", - "topics": [ - "Futurism", - "History", - "Renewable Energy", - "Climate Change" - ] - }, - { - "title": "Employees Are Feeding Sensitive Business Data to ChatGPT", - "accessLink": "https://www.darkreading.com/risk/employees-feeding-sensitive-business-data-chatgpt-raising-security-fears", - "topics": [ - "Security", - "AI Chatbots", - "ChatGPT", - "Privacy" - ] - }, - { - "title": "A Short 100-Question Diligence Checklist", - "accessLink": "https://www.thediff.co/archive/100-due-diligence-questions-checklist/", - "topics": [ - "Investments", - "Finance", - "Equities", - "Heuristics" - ] - }, - { - "title": "How Helping the Environment Can Make You Feel Better", - "accessLink": "https://www.verywellmind.com/how-helping-the-environment-can-make-you-feel-better-5296131", - "topics": [ - "Psychology", - "Mental Health", - "Environment", - "Wellness" - ] - }, - { - "title": "Are We Ready for Artificial Empathy? | Psychology Today", - "accessLink": "https://www.psychologytoday.com/us/blog/202304/are-we-ready-for-artificial-empathy", - "topics": [ - "Artificial Intelligence", - "Psychology", - "Technology", - "Futurism" - ] - }, - { - "title": "State-by-State Guide to Taxes on Middle-Class Families", - "accessLink": "https://www.kiplinger.com/kiplinger-tools/taxes/t055-s001-kiplinger-tax-map/index.php?map=&state_id=5&state=California", - "topics": [ - "Finance", - "Taxes", - "Money", - "Economics" - ] - }, - { - "title": "The Open Graph Protocol", - "accessLink": "https://ogp.me/", - "topics": [ - "Web3", - "Social Graph", - "Metadata", - "RDFs" - ] - }, - { - "title": "What We Still Don’t Know About How A.I. Is Trained | The New Yorker", - "accessLink": "https://www.newyorker.com/news/daily-comment/what-we-still-dont-know-about-how-ai-is-trained?", - "topics": [ - "Artificial Intelligence", - "GPT-4", - "Futurism", - "Ethics" - ] - }, - { - "title": "Leading a Midsize Business Through Change - Harvard Business Review", - "accessLink": "https://hbr.org/2023/04/leading-a-midsize-business-through-change", - "topics": [ - "Business", - "Management", - "Leadership", - "Change" - ] - }, - { - "title": "JPLs Open Source Rover", - "accessLink": "https://github.com/nasa-jpl/open-source-rover", - "topics": [ - "Robotics", - "Space Exploration", - "Engineering", - "Software" - ] - }, - { - "title": "GPTs are GPTs: An early look at the labor market impact potential of large language models", - "accessLink": "https://openai.com/research/gpts-are-gpts", - "topics": [ - "Generative Pretrained Transformers", - "Wage Levels", - "Economy", - "Futurism" - ] - }, - { - "title": "Reflexion: an autonomous agent with dynamic memory and self-reflection", - "accessLink": "https://arxiv.org/abs/2303.11366", - "topics": [ - "AI Agents", - "Decision Making", - "Fine-tuning", - "Research" - ] - }, - { - "title": "A Short 100-Question Diligence Checklist", - "accessLink": "https://www.thediff.co/archive/100-due-diligence-questions-checklist/", - "topics": [ - "Investments", - "Heuristics", - "Equities", - "Finance" - ] - }, - { - "title": "GitHub - statelyai/xstate: State machines and statecharts for the modern web.", - "accessLink": "https://github.com/statelyai/xstate", - "topics": [ - "State Machines", - "Statecharts", - "Web3", - "Software" - ] - }, - { - "title": "Scientists finally figure out why the water bear is nearly indestructible - Big Think", - "accessLink": "https://bigthink.com/surprising-science/scientists-finally-figure-out-why-the-water-bear-is-nearly-unstoppable/", - "topics": [ - "Biology", - "Water Bears", - "Tardigrades", - "Research" - ] - }, - { - "title": "Enjoy An Open-Source Espresso | Hackaday", - "accessLink": "https://hackaday.com/2023/03/24/enjoy-an-open-source-espresso/", - "topics": [ - "Open-Source", - "Espresso", - "Code", - "Hardware" - ] - }, - { - "title": "Here is the FBI’s Contract to Buy Mass Internet Data", - "accessLink": "https://www.vice.com/en/article/dy3z9a/fbi-bought-netflow-data-team-cymru-contract", - "topics": [ - "Internet Data", - "Hackers", - "Cybercrime", - "National Security" - ] - }, - { - "title": "Where Did Writing Come From?", - "accessLink": "https://www.getty.edu/news/where-did-writing-come-from/", - "topics": [ - "History Of Writing", - "Cuneiform Writing", - "Writing Tools", - "Ancient Mesopotamia" - ] - }, - { - "title": "Big tech and the pursuit of AI dominance | The Economist", - "accessLink": "https://www.economist.com/business/2023/03/26/big-tech-and-the-pursuit-of-ai-dominance", - "topics": [ - "Artificial Intelligence", - "Tech Giants", - "Competition", - "Innovation" - ] - }, - { - "title": "Automate your Python project with Makefile", - "accessLink": "https://antonz.org/makefile-automation/", - "topics": [ - "Automation", - "Makefile", - "Python", - "Project" - ] - }, - { - "title": "The rise and rise of e-sports | The Economist", - "accessLink": "https://www.economist.com/special-report/2023/03/20/the-rise-and-rise-of-e-sports", - "topics": [ - "E Sports", - "Broadcasting", - "Sponsorship", - "Popularity" - ] - }, - { - "title": "When programming is gone, will we like what's left?", - "accessLink": "https://newsletter.goodtechthings.com/p/when-programming-is-gone-will-we", - "topics": [ - "Programming", - "Automation", - "Software Engineering", - "AI" - ] - }, - { - "title": "Microsoft rebuilt Teams from the ground up, promises 2x faster performance | TechCrunch", - "accessLink": "https://techcrunch.com/2023/03/27/microsoft-rebuilds-teams-promises-2x-faster-performance/?guccounter=1", - "topics": [ - "Microsoft Teams", - "Performance", - "Efficiency", - "Ai Features" - ] - }, - { - "title": "Knowledge + Chat (New Features) - YouTube", - "accessLink": "https://www.youtube.com/watch?v=8n26MazawQk", - "flagged": false, - "topics": [ - "Knowledge", - "Features", - "Open Source Software", - "Free" - ] - } -] diff --git a/package.json b/package.json index 9c82045a..65d90bfd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "knowledge-canvas", - "version": "0.8.5", + "version": "0.8.6", "description": "Knowledge is a tool for saving, searching, and chatting with all of your favorite websites, documents and files.", "author": "Rob Royce (https://robroyce.dev)", "license": "Apache-2.0", @@ -26,7 +26,7 @@ "clean": "rm -rf dist && yarn workspaces foreach run clean", "purge": "yarn workspaces foreach run purge && rm -rf node_modules && yarn cache clean", "pack": "electron-builder --dir", - "dist": "electron-builder --publish never", + "dist": "electron-builder --publish never --x64 --arm64", "dist-all": "electron-builder -mwl --x64 --arm64 --publish never", "interactive-upgrade": "yarn upgrade-interactive", "publish": "electron-builder --publish always", @@ -45,11 +45,11 @@ "dependencies": { "@adobe/css-tools": "^4.3.2", "async": "^3.2.5", - "axios": "^1.6.0", + "axios": "^1.6.5", "canvas": "^2.11.2", "ejs": "^3.1.9", "express": "^4.18.2", - "follow-redirects": "^1.15.4", + "follow-redirects": "^1.15.5", "got": "^13.0.0", "highlight.js": "^11.9.0", "html-to-text": "^9.0.5", @@ -59,8 +59,8 @@ "node-forge": "^1.3.1", "pdfjs-dist": "^3.11.174", "plist": "^3.1.0", - "postcss": "^8.4.31", - "terser": "^5.22.0", + "postcss": "^8.4.33", + "terser": "^5.27.0", "url": "^0.11.3" }, "devDependencies": { @@ -69,16 +69,16 @@ "@types/async": "^3.2.24", "@types/ejs": "^3.1.4", "@types/eslint": "^8", - "@types/express": "^4.17.20", - "@types/follow-redirects": "^1.14.3", - "@types/html-to-text": "^9.0.3", - "@types/minimist": "^1.2.4", - "@types/multer": "^1.4.9", + "@types/express": "^4.17.21", + "@types/follow-redirects": "^1.14.4", + "@types/html-to-text": "^9.0.4", + "@types/minimist": "^1.2.5", + "@types/multer": "^1.4.11", "@types/node": "^17.0.45", - "@types/node-forge": "^1.3.8", - "@types/plist": "^3.0.4", + "@types/node-forge": "^1.3.11", + "@types/plist": "^3.0.5", "@types/prettier": "^2.7.3", - "@types/uuid": "^9.0.6", + "@types/uuid": "^9.0.7", "electron-builder": "^24.6.4", "electron-notarize": "^1.2.2", "eslint": "^8.38.0", diff --git a/src/kc_angular/package.json b/src/kc_angular/package.json index ce1d4abd..989e4cb3 100755 --- a/src/kc_angular/package.json +++ b/src/kc_angular/package.json @@ -38,7 +38,7 @@ "@fullcalendar/timegrid": "^5.11.5", "chart.js": "^3.9.1", "chokidar": "^3.5.3", - "cytoscape": "^3.26.0", + "cytoscape": "^3.28.1", "cytoscape-cola": "^2.5.1", "cytoscape-dagre": "^2.5.0", "cytoscape-fcose": "^2.2.0", @@ -63,10 +63,10 @@ "@angular-eslint/template-parser": "14.4.0", "@angular/cli": "^14.2.13", "@angular/compiler-cli": "^14.3.0", - "@types/cytoscape": "^3.19.13", - "@types/cytoscape-dagre": "^2.3.2", - "@types/cytoscape-fcose": "^2.2.3", - "@types/cytoscape-klay": "^3.1.3", + "@types/cytoscape": "^3.19.16", + "@types/cytoscape-dagre": "^2.3.3", + "@types/cytoscape-fcose": "^2.2.4", + "@types/cytoscape-klay": "^3.1.4", "@types/marked": "^4.3.2", "@typescript-eslint/eslint-plugin": "5.37.0", "@typescript-eslint/parser": "5.37.0", @@ -76,7 +76,7 @@ "eslint-plugin-prettier": "^4.2.1", "ng-packagr": "^14.2.2", "typescript": "<4.9.0", - "web-worker": "^1.2.0" + "web-worker": "^1.3.0" }, "eslintConfig": { "extends": [ diff --git a/src/kc_angular/src/app/app.component.html b/src/kc_angular/src/app/app.component.html index d24aef28..e812e471 100755 --- a/src/kc_angular/src/app/app.component.html +++ b/src/kc_angular/src/app/app.component.html @@ -1,5 +1,5 @@ - -
- - -
-
- - -
- - -
-
-
-
- - -
- - - diff --git a/src/kc_angular/src/app/components/chat-components/chat.view.component.scss b/src/kc_angular/src/app/components/chat-components/chat.view.component.scss deleted file mode 100755 index 0b1a13ce..00000000 --- a/src/kc_angular/src/app/components/chat-components/chat.view.component.scss +++ /dev/null @@ -1,49 +0,0 @@ -/*! - * Copyright (c) 2023 Rob Royce - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -.user-message { - background-color: var(--primary-color) !important; - color: var(--primary-color-text) !important; - border-radius: 10px; - padding: 0 10px 0 10px; - margin: 10px; - text-align: right; - max-width: 80%; -} - -.agent-message { - color: var(--primary-color) !important; - background-color: var(--primary-color-text) !important; - border-radius: 10px; - padding: 0 10px 0 10px; - margin: 10px; - text-align: left; - max-width: 80%; -} - -#chat-toolbar { - height: 2rem; - background-color: var(--surface-ground); -} - -.next-question-suggestion:hover { - opacity: 1 !important; -} - -.html2canvas-container { - width: 3000px !important; - height: 3000px !important; -} diff --git a/src/kc_angular/src/app/components/chat-components/chat.view.component.ts b/src/kc_angular/src/app/components/chat-components/chat.view.component.ts index 05dd68d7..5d556b1b 100755 --- a/src/kc_angular/src/app/components/chat-components/chat.view.component.ts +++ b/src/kc_angular/src/app/components/chat-components/chat.view.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,173 +14,117 @@ * limitations under the License. */ -import { - Component, - ElementRef, - EventEmitter, - Input, - OnChanges, - OnInit, - Output, - SimpleChanges, - ViewChild, -} from '@angular/core'; +import { Component, Input, ViewChild } from '@angular/core'; import { ChatService } from '@app/services/chat-services/chat.service'; -import { debounceTime, finalize, map, take, tap } from 'rxjs/operators'; -import { ConfirmationService, MenuItem, PrimeIcons } from 'primeng/api'; +import { MenuItem } from 'primeng/api'; import { ContextMenu } from 'primeng/contextmenu'; -import { DialogService } from 'primeng/dynamicdialog'; -import { Clipboard } from '@angular/cdk/clipboard'; -import { MarkdownService } from '@app/services/factory-services/markdown.service'; -import { - KcNotification, - NotificationsService, -} from '@app/services/user-services/notifications.service'; -import { ChatContextMenuService } from '@app/services/factory-services/chat-context-menu.service'; import { AgentType, ChatMessage } from '@app/models/chat.model'; -import { ChatPrompts } from '@services/chat-services/prompts'; -import { SettingsService } from '@services/ipc-services/settings.service'; -import { BehaviorSubject, combineLatest, Observable } from 'rxjs'; +import { Observable } from 'rxjs'; +import { ChatContextMenuService } from '@services/factory-services/chat-context-menu.service'; +import { ChatCommandService } from '@services/chat-services/commands.service'; +import { distinctUntilChanged, tap } from 'rxjs/operators'; @Component({ selector: 'app-chat-view', - templateUrl: './chat.view.component.html', - styleUrls: ['./chat.view.component.scss'], + template: ` +
+ + +
+
+
+
+ +
+
+
+
+ No Messages +
+
+
+ + +
+ + + + `, + styles: [ + ` + #chat-toolbar { + height: 2rem; + background-color: var(--surface-ground); + } + `, + ], }) -export class ChatViewComponent implements OnInit, OnChanges { - /* The input element for the chat */ - @ViewChild('chatInput', { static: true }) chatInput!: ElementRef; - +export class ChatViewComponent { /* The context menu for chat messages */ @ViewChild('cm', { static: true }) cm!: ContextMenu; - /* The chat history to display */ - @Input() history: ChatMessage[] = []; - - /* Whether the chat is loading (e.g. waiting for a response from the server) */ - @Input() loading = false; - /* Whether to apply a height restriction to the chat history */ @Input() heightRestricted = false; - /* The event emitted when the user submits a message */ - @Output() submit: EventEmitter = new EventEmitter(); - - /* The event emitted when the user clicks on the Regenerate button */ - @Output() regenerate: EventEmitter = - new EventEmitter(); - - /* The event emitted when the user clicks on the Delete button */ - @Output() delete: EventEmitter = new EventEmitter< - ChatMessage[] - >(); - - /* An array of suggestions for the next question */ - nextQuestionSuggestions: string[] = []; - - /* The context menu used to display the options for a message */ messageMenu: MenuItem[] = []; - suggestions = true; - - focusTrap = true; - - editedMessage?: ChatMessage; - - filtered$: Observable; - - private _history$ = new BehaviorSubject([]); - - history$ = this._history$.asObservable(); - - private filter = new BehaviorSubject(''); - - filter$ = this.filter.asObservable(); - - private messageInput = new BehaviorSubject(''); + messages$: Observable; constructor( - private clipboard: Clipboard, private chat: ChatService, private context: ChatContextMenuService, - private confirm: ConfirmationService, - private dialog: DialogService, - private markdown: MarkdownService, - private notify: NotificationsService, - private prompts: ChatPrompts, - private settings: SettingsService + private command: ChatCommandService ) { - settings.all - .pipe( - map((settings) => settings.app.chat.suggestions), - debounceTime(250), - tap((suggestions) => { - this.suggestions = suggestions.enabled; - this.focusTrap = suggestions.onInput; - }) - ) - .subscribe(); - - this.history$.pipe(debounceTime(100)).subscribe((history) => { - if (history.length === 0) { - // Add an introductory message to the chat history - const msg = this.chat.createMessage( - AgentType.Knowledge, - AgentType.User, - "Welcome to Knowledge Chat! I'm here to help you with any questions or research needs you may have. Whether you need help with a project or just want to learn more about a topic, I'm here to assist you. Please feel free to ask me anything!" - ); - this._history$.next([msg]); - } - }); - - // Filter the chat history based on the filter text - this.filtered$ = combineLatest([this.filter$, this.history$]).pipe( - debounceTime(100), - map(([filter, history]) => { - if (!filter) return history; - - return history.filter((message) => { - return message.text.toLowerCase().includes(filter.toLowerCase()); - }); + this.messages$ = this.chat.messages$.pipe( + distinctUntilChanged((a, b) => { + // If the messages are the same length, check if they are the same messages + if (a.length === b.length) { + for (let i = 0; i < a.length; i++) { + if (a[i].id !== b[i].id) { + return false; + } + } + return true; + } + return false; + }), + tap(() => { + this.scroll(); }) ); - this.messageInput - .asObservable() - .pipe(debounceTime(1000)) - .subscribe((message) => { - this.chat.countTokens(message); - }); - } - - ngOnInit(): void { - // Scroll to the bottom of the chat history after rest of component finishes loading - setTimeout(() => { + this.command.scrollView$.asObservable().subscribe(() => { this.scroll(); - }, 50); - } - - /** - * If the history changes, reset the suggestions and sort the history by timestamp - * If the suggestions are enabled, use the most recent message to generate suggestions - */ - ngOnChanges(changes: SimpleChanges) { - if (changes.history) { - // Reset the suggestions when the history changes - this.nextQuestionSuggestions = []; - this.filter.next(''); - this._history$.next(changes.history.currentValue); - - setTimeout(() => { - if ( - changes.history?.currentValue?.length === - changes.history?.previousValue?.length - 1 - ) { - return; - } - this.scroll(); - }, 1000); - } + }); } /* Scroll to the end of the #chat-history element */ @@ -195,227 +139,19 @@ export class ChatViewComponent implements OnInit, OnChanges { } } - sendMessage(message: string) { - if (!message) { - return; - } - - this.submit.emit(message); - this.chat.countTokens(''); - this.scroll(); - } - - /** - * Generate the next question suggestions for the given message - * - * @param message The message to generate suggestions for - * @description The suggestions are generated using the message and the 14 messages prior to it - */ - generateNextQuestion(message: ChatMessage) { - if (!this.settings.get().app.chat.suggestions.enabled) { - return; - } - - // Get the message and the 14 messages prior to it, then send those to the prompt service - this.chat.getHistory(message, this.history, true); - const index = this.history.indexOf(message); - const messages = this.history.slice(index - 14, index + 1); - - this.prompts - .predictNextQuestion(messages, message.source ? 'source' : 'project') - .pipe( - take(1), - tap((response) => { - this.nextQuestionSuggestions = response; - this.scroll(); - }) - ) - .subscribe(); - } - - chatFocusTrap() { - if (this.suggestions && this.focusTrap) { - this.generateNextQuestion(this.history[this.history.length - 1]); - } - } - - /* Submit the given suggestion as the next question */ - sendSuggestion(suggestion: string) { - if (this.loading) { - return; - } - this.submit.emit(suggestion); - this.nextQuestionSuggestions = this.nextQuestionSuggestions.filter( - (s) => s !== suggestion - ); - - if (this.nextQuestionSuggestions.length === 0) { - this.generateNextQuestion(this.history[this.history.length - 1]); - } - } - - /** - * Regenerate the given message, replacing it with a new message - * @param message The message to regenerate - * @param warn Whether to warn the user that the message will be replaced in the chat history - * @description This will replace the message in the chat history with a new message - */ - regenerateMessage(message: ChatMessage, warn = true) { - if (warn) { - this.warnUser( - [message], - `Regenerate`, - `Are you sure you want to regenerate this message? This will replace the message in the chat history.`, - `Regenerate Message`, - this.regenerate - ); + blurHistory($event: boolean) { + // If the command bar is active, make the chat history slightly opaque and transparent + if ($event) { + document + .getElementsByClassName('chat-history')[0] + .classList.add('chat-history-active'); } else { - this.regenerate.emit(message); + document + .getElementsByClassName('chat-history')[0] + .classList.remove('chat-history-active'); } } - /** - * Delete the given message from the chat history and storage - * @param message The message to delete - * @description This will delete the message from the chat history and storage - */ - deleteMessage(message: ChatMessage) { - this.warnUser( - [message], - `Delete`, - `Are you sure you want to permanently delete this message?`, - `Delete Message`, - this.delete - ); - } - - /** - * Edit the given message in the chat history - * @param message The message to edit - * @param edited (optional) The edited message to replace the original message with - * @description This will replace the message in the chat history with the edited message. - * If no edited message is given, the message will turn into an input field. If an edited message is given, - * the message will be replaced with the edited message and the response will be regenerated. - */ - editMessage(message: ChatMessage, edited?: string) { - if (edited && edited !== message.text) { - message.text = edited; - - // If this is a user message, regenerate the response to the message - if (message.sender === AgentType.User) { - // Get the message that was a response to the message being edited and regenerate it - const index = this.history.indexOf(message); - if (index + 1 <= this.history.length) { - const response = this.history[index + 1]; - - if (response.sender === message.recipient) { - // If the next message is from the same agent, regenerate the message - this.regenerateMessage(response, false); - } else { - // Otherwise, we need to insert a new response into the history immediately after the edited message - const newResponse = this.chat.createMessage( - message.recipient, - AgentType.User, - '', - message.project, - message.source - ); - this.history.splice(index + 1, 0, newResponse); - this.regenerateMessage(newResponse, false); - } - } - } - } - this.editedMessage = edited ? undefined : message; - } - - /** - * Summarize the given message using the chat service - * @param message The message to summarize - * @description This will add a tldr section to the top of the message - */ - tldr(message: ChatMessage) { - // Get most recent messages leading up to the message to summarize - const index = this.history.indexOf(message); - const messages = this.history.slice(index - 14, index + 1); - - this.loading = true; - this.chat - .elaborate(messages, 'TLDR') - .pipe( - take(1), - tap((response) => { - message.text += ` - -**TLDR:** ${response.content}`; - this.chat.saveChat(this.history); - }), - finalize(() => { - this.loading = false; - }) - ) - .subscribe(); - } - - /** - * Explain the given message to a 5-year-old - * @param message The message to explain - * @description This will append the ELI5 explaination to the bottom of the message - */ - explainLikeIm5(message: ChatMessage) { - // Get most recent messages leading up to the message to explain - this.loading = true; - const index = this.history.indexOf(message); - const messages = this.history.slice(index - 14, index + 1); - this.chat - .elaborate(messages, 'ELI5') - .pipe( - take(1), - tap((response) => { - message.text += ` - -**ELI5:** ${response.content}`; - this.chat.saveChat(this.history); - }), - finalize(() => { - this.loading = false; - }) - ) - .subscribe(); - } - - /** - * Continue the response to the given message (useful when the response is too long to fit in one message) - * @param message - */ - continueResponse(message: ChatMessage) { - // Get most recent messages leading up to the message to explain - this.loading = true; - const index = this.history.indexOf(message); - const messages = this.history.slice(index - 14, index + 1); - this.chat - .elaborate(messages, 'Continue') - .pipe( - take(1), - tap((response) => { - message.text += `${response.content}`; - this.chat.saveChat(this.history); - }), - finalize(() => { - this.loading = false; - }) - ) - .subscribe(); - } - - /** - * Filter the chat history by the given string - * @param $event The string to filter by - */ - onFilter($event: string) { - this.filter.next($event); - } - /** * Show the context menu for the given message at the given event location * @param $event The event to show the context menu at @@ -431,24 +167,18 @@ export class ChatViewComponent implements OnInit, OnChanges { const text = selection.toString().trim(); const tokens = text.split(' '); if (text.length > 0 && tokens.length > 0 && tokens.length <= 5) { - this.messageMenu.push( - this.context.topics(message, text, () => { - this.chat.saveChat(this.history); - }) - ); + this.messageMenu.push(this.context.topics(message, text, () => {})); } + // Add the search option - this.messageMenu.push( - this.context.search(message, selection, () => { - this.chat.saveChat(this.history); - }) - ); + this.messageMenu.push(this.context.search(message, selection, () => {})); } // If the user has highlighted text, add a highlight option const highlight = this.context.highlight(message, $event, () => { - this.chat.saveChat(this.history); + this.chat.persistChatHistory(); }); + if (highlight) { this.messageMenu.push(highlight); } @@ -466,120 +196,17 @@ export class ChatViewComponent implements OnInit, OnChanges { this.messageMenu.push(this.context.save(message)); } - this.messageMenu.push(this.context.copy(message)); - this.messageMenu.push( - this.context.edit(() => { - this.editMessage(message); - }) - ); - - // If the message is not a user message, add continue, regenerate, tldr, and eli5 options - if (message.sender !== AgentType.User) { + if (this.messageMenu.length > 0) { this.messageMenu.push(this.context.separator()); - this.messageMenu.push( - this.context.continue(() => { - this.continueResponse(message); - }) - ); - this.messageMenu.push( - this.context.regenerate(() => { - this.regenerateMessage(message); - }) - ); - this.messageMenu.push( - this.context.tldr(() => { - this.tldr(message); - }) - ); - this.messageMenu.push( - this.context.eli5(() => { - this.explainLikeIm5(message); - }) - ); } - this.messageMenu.push(this.context.separator()); this.messageMenu.push( + this.context.copy(message), this.context.delete(() => { - this.deleteMessage(message); + this.chat.deleteMessage(message); }) ); this.cm.show($event); } - - rating(rating: 'thumbs-up' | 'thumbs-down' | 'none', message: ChatMessage) { - message.rating = rating; - - this.notify.debug('Chat View', rating.toLocaleUpperCase(), message); - this.chat.saveChat(this._history$.value); - - if (rating === 'none') { - return; - } - - const msg: KcNotification = { - severity: rating === 'thumbs-up' ? 'success' : 'warn', - icon: - rating === 'thumbs-up' ? PrimeIcons.THUMBS_UP : PrimeIcons.THUMBS_DOWN, - summary: rating === 'thumbs-up' ? 'Context++' : 'Context--', - detail: - rating === 'thumbs-up' - ? `This message will be prioritized for providing context.` - : `This message will not be used to provide context.`, - closable: true, - presentation: 'toast', - }; - this.notify.broadcast(msg); - } - - /** - * Warn the user that the given action will replace the message in the chat history - * @param messages The message(s) to regenerate or delete - * @param acceptLabel The label for the accept button - * @param confirmation The confirmation message to display to the user - * @param header The header to display to the user - * @param emitter The emitter to emit the message to when the user confirms the action - */ - private warnUser( - messages: ChatMessage[], - acceptLabel: string, - confirmation: string, - header: string, - emitter: EventEmitter - ) { - this.confirm.confirm({ - message: confirmation, - icon: `pi pi-exclamation-triangle`, - header: header, - accept: () => { - emitter.emit(messages); - }, - acceptLabel: acceptLabel, - acceptIcon: `pi pi-check`, - rejectIcon: `pi pi-times`, - acceptButtonStyleClass: `p-button-danger`, - rejectButtonStyleClass: `p-button-secondary`, - }); - } - - onClear() { - this.confirm.confirm({ - message: 'Are you sure you want to clear the chat history?', - icon: `pi pi-exclamation-triangle`, - header: 'Clear Chat History', - accept: () => { - this.delete.emit(this.history); - }, - acceptLabel: 'Clear', - acceptIcon: `pi pi-check`, - rejectIcon: `pi pi-times`, - acceptButtonStyleClass: `p-button-danger`, - rejectButtonStyleClass: `p-button-secondary`, - }); - } - - countTokens(value: string) { - this.messageInput.next(value); - } } diff --git a/src/kc_angular/src/app/components/chat-components/message-templates/categorize.message.ts b/src/kc_angular/src/app/components/chat-components/message-templates/categorize.message.ts new file mode 100644 index 00000000..85ac58d6 --- /dev/null +++ b/src/kc_angular/src/app/components/chat-components/message-templates/categorize.message.ts @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; +import { ChatMessage } from '@app/models/chat.model'; +import { + ProjectIdentifiers, + ProjectService, +} from '@services/factory-services/project.service'; +import { take } from 'rxjs/operators'; +import { ProjectTreeNode } from '@app/models/project.tree.model'; +import { KcProject, ProjectCreationRequest } from '@app/models/project.model'; +import { ChatService } from '@services/chat-services/chat.service'; +import { IngestService } from '@services/ingest-services/ingest.service'; +import { NotificationsService } from '@services/user-services/notifications.service'; +import { select } from 'async'; + +interface ProjectSuggestion { + project: KcProject | null | undefined; + lineage: string[]; + details: string; + ancestors: ProjectIdentifiers[]; +} + +@Component({ + template: ` +
+
Suggested Projects
+
+ The following projects were suggested for this source: +
+
+
+
+ + + +
+
+
+ +
+ `, + selector: 'chat-categorize', + styles: [ + ` + :host { + justify-content: center; + align-content: center; + } + `, + ], +}) +export class CategorizeMessage implements OnInit { + @Input() message!: ChatMessage; + + @Output() add = new EventEmitter(); + + @Output() create = new EventEmitter(); + + projects: ProjectSuggestion[] = []; + + lines: string[] = []; + + selectedSuggestion = undefined; + + buttonLabel = 'Add'; + + constructor( + private service: ProjectService, + private chat: ChatService, + private ingest: IngestService, + private notify: NotificationsService + ) {} + + ngOnInit() { + this.parseMessage(this.message); + } + + parseMessage(message: ChatMessage) { + // Suggested projects are formatted as follows: + // - Root Project > Subproject > ... > Final Project + // The message will contain at least one project suggestion + + // Get all lines that start with a dash + const lines = message.text + .split('\n') + .filter((line) => line.startsWith('-')) + .map((line) => line.replace('-', '').trim()); + + this.lines = lines; + + // Find the project by searching for the root project + this.service.projectTree.pipe(take(1)).subscribe((tree) => { + for (const line of lines) { + const parts = line.split(':'); + const lineage = parts[0] + .trim() + .split('>') + .map((project) => project.trim()); + + let details = parts[1]?.trim() ?? null; + details = details?.charAt(0).toUpperCase() + details?.slice(1); + + const project = this.find(tree, lineage); + const ancestors = this.service.getAncestors(project?.id.value ?? ''); + + this.projects.push({ + project: + project?.name === lineage[lineage.length - 1] ? project : null, + lineage, + details: details, + ancestors: ancestors ?? [], + }); + } + + if (this.projects.length === 0) { + this.notify.error( + 'Categorize Message', + 'Error parsing categories.', + 'The suggested proejcts message could not be parsed (see console for details).', + 'toast' + ); + console.log('Error parsing message', message); + } + }); + } + + private find( + root: ProjectTreeNode[], + lineage: string[], + last?: ProjectTreeNode + ): KcProject | null | undefined { + // Recursively search for the projects in the lineage until we find the final project + const project = lineage[0]; + const subprojects = lineage.slice(1); + + // Find the node, ignoring case + const node = root.find( + (node) => node.name.toLowerCase() === project.toLowerCase() + ); + + if (!node) { + if (last) { + return this.service.getProject(last.id); + } else { + return null; + } + } else { + if (subprojects.length === 0) { + return this.service.getProject(node.id); + } else { + return this.find(node.subprojects, subprojects, node); + } + } + } + + addProject() { + // Find the project suggestion that matches the selected suggestion + const suggestion = this.projects.find( + (s) => s.lineage.join(' > ') === this.selectedSuggestion + ); + + if (!suggestion) { + console.warn('No suggestion selected'); + return; + } + + // If the project does not exist, create it + if (!suggestion.project) { + this.createProjects(suggestion.lineage, suggestion.ancestors).then( + (parent) => { + this.addSource(parent); + } + ); + } else { + // Otherwise, add the source to the project + this.addSource({ + id: suggestion.project.id.value, + title: suggestion.project.name, + }); + } + } + + setLabel(project: ProjectSuggestion) { + this.buttonLabel = project.project + ? `Add to ${project.project.name}` + : 'Create and Add'; + } + + private async addSource(project: ProjectIdentifiers) { + this.chat.target$.pipe(take(1)).subscribe((target) => { + if (!target.source) { + console.warn('No source selected'); + return; + } + + const source = target.source; + + if (source.associatedProject.value.trim().length > 0) { + // If the Source is already in a project, we need to move it + this.service.updateProjects([ + { + id: source.associatedProject, + moveKnowledgeSource: { + ks: source, + new: { value: project.id }, + }, + }, + ]); + } else { + // Otherwise it must be in the inbox + this.service + .updateProjects([ + { + id: { value: project.id }, + addKnowledgeSource: [source], + }, + ]) + .then(() => { + setTimeout(() => { + this.ingest.add(source); + this.service.setCurrentProject(project.id); + }, 500); + }); + } + }); + } + + private async createProjects( + lineage: string[], + ancestors: ProjectIdentifiers[] + ) { + /** + * Create the projects in the lineage that do not exist + * @param lineage The lineage of the project to create, starting with the root project + * @param ancestors The ancestors of the project that already exist + */ + let parent = ancestors[ancestors.length - 1] ?? null; + + for (let i = ancestors.length; i < lineage.length; i++) { + const name = lineage[i]; + + const request: ProjectCreationRequest = { + authors: [], + calendar: { events: [], start: null, end: null }, + description: '', + knowledgeSource: [], + name: name, + parentId: { value: parent?.id ?? '' }, + sources: [], + subProjects: [], + topics: [], + type: 'default', + }; + const newProject = await this.service.newProject(request); + parent = { id: newProject.id.value, title: newProject.name }; + + // Wait for 250 ms to allow the project to be created + await new Promise((resolve) => setTimeout(resolve, 250)); + } + + return parent; + } + + protected readonly select = select; +} diff --git a/src/kc_angular/src/app/components/chat-components/message-templates/project.message.ts b/src/kc_angular/src/app/components/chat-components/message-templates/project.message.ts new file mode 100644 index 00000000..e69de29b diff --git a/src/kc_angular/src/app/components/chat-components/message-templates/quiz.message.ts b/src/kc_angular/src/app/components/chat-components/message-templates/quiz.message.ts new file mode 100644 index 00000000..7011f363 --- /dev/null +++ b/src/kc_angular/src/app/components/chat-components/message-templates/quiz.message.ts @@ -0,0 +1,367 @@ +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, Input, OnChanges, SimpleChanges } from '@angular/core'; +import { ChatMessage } from '@app/models/chat.model'; +import { fadeIn, fadeInAndOut, flyInOut } from '@app/animations'; +import { ChatService } from '@services/chat-services/chat.service'; +import { take } from 'rxjs/operators'; + +interface QuizQuestion { + question: string; + choices: string[]; + answer: string; + correct: boolean; + answered: boolean; +} + +@Component({ + selector: 'chat-quiz', + template: ` +
+
+
Nice Work!
+
+
+ +
+
+
+ {{ quip }} +
+
+
+
+
+
Try Again?
+
+ +
+
+ You got {{ numberCorrect }} out of {{ questions.length }} correct +
+
+
+
+
+
Uh oh!
+
+ There was a problem with this quiz. You might need to delete this + message and try again. +
+
+
+
+
+
+
+ Correct! +
+ +
+
Try Again!
+
The correct answer was {{ question.answer }}
+
+
+
+
+
+ {{ questions[questionIndex].question }} +
+
+
+
+ {{ choice }} +
+
+
+
+ {{ questionIndex + 1 }} / + {{ questions.length }} +
+
+
+ Need Help? +
+
+ {{ hint }} +
+
+
+
+
+ `, + styles: [ + ` + :host { + max-width: 100rem !important; + display: flex; + flex-direction: column; + align-items: center; + justify-content: flex-start; + } + .question-choice { + cursor: pointer !important; + display: flex; + flex-direction: column; + align-content: center; + justify-content: center; + align-items: center; + text-align: center !important; + min-width: 50% !important; + height: 5rem; + } + `, + ], + animations: [fadeInAndOut, flyInOut, fadeIn], +}) +export class QuizMessage implements OnChanges { + @Input() message!: ChatMessage; + + /* The questions, choices, and answers */ + questions: QuizQuestion[] = []; + + /* The index of the current question */ + questionIndex = 0; + + complete = false; + + tryAgain = false; + + private quips = [ + 'Look at you go, you little genius', + "It's big brain time", + "You're on fire", + "You're a wizard, Harry", + 'Take it easy, Einstein', + 'How do you know all this stuff?', + 'Next stop, Jeopardy', + 'Can I borrow your brain for a sec?', + 'Your mom would be proud', + "I'd give you a high five if I could", + ]; + + quip: string; + + hint?: string = undefined; + + private audio = { + correct: new Audio(), + incorrect: new Audio(), + complete: new Audio(), + }; + + constructor(private chat: ChatService) { + // Select a random quip to display after the user gets all questions correct + function getRandomInt(max: number) { + return Math.floor(Math.random() * Math.floor(max)); + } + this.quip = this.quips[getRandomInt(this.quips.length)]; + this.audio.correct.src = 'assets/sounds/correct.wav'; + this.audio.incorrect.src = 'assets/sounds/incorrect.wav'; + this.audio.complete.src = 'assets/sounds/complete.wav'; + this.audio.correct.load(); + this.audio.incorrect.load(); + this.audio.complete.load(); + } + + ngOnChanges(changes: SimpleChanges) { + if (changes.message) { + setTimeout(() => { + this.questions = this.parseQuiz(changes.message.currentValue); + }, 100); + } + } + + get numberCorrect() { + return this.questions.filter((question) => question.correct).length; + } + + private parseQuiz(message: ChatMessage): QuizQuestion[] { + const questions: QuizQuestion[] = []; + const qaRegex = /Q.*\n+A.*\n+B.*\n+C.*\n+D.*\n+A.*\n*$/gm; + const matches = message.text.match(qaRegex); + + if (!matches) { + console.warn('No matches found in quiz message: ', message.text); + return []; + } + + for (const match of matches) { + const components = match.split('\n'); + + // Remove any blank elements + for (let i = 0; i < components.length; i++) { + if (components[i] === '') { + components.splice(i, 1); + } + } + + const question: QuizQuestion = { + question: components[0].replace(/Q[0-9]*\. /, ''), + choices: [components[1], components[2], components[3], components[4]], + answer: components[5].replace('Answer: ', '').substring(0, 1), + correct: false, + answered: false, + }; + questions.push(question); + } + + return questions; + } + + answer(question: QuizQuestion, choice: string) { + question.answered = true; + + // Answer index is the index of the choice in the choices array + const answerIndex = question.choices.indexOf(choice); + + // The answer will be either A, B, C, or D + const answer = String.fromCharCode(65 + answerIndex); + + // Check if the answer is correct + question.correct = answer === question.answer; + + // If the answer is correct, play a sound + if (question.correct) { + this.audio.correct.play(); + } else { + this.audio.incorrect.play(); + } + + setTimeout(() => { + this.nextQuestion(); + }, 1650); + } + + nextQuestion() { + this.hint = undefined; + + if (this.questions.every((question) => question.answered)) { + // If all questions were answered correctly, celebrate with a small animation + if (this.questions.every((question) => question.correct)) { + this.complete = true; + this.audio.complete.play(); + return; + } else { + this.tryAgain = true; + this.audio.incorrect.play(); + } + } + + // Otherwise, go to the next question that has not been answered correctly + // Make sure to wrap around to the beginning if necessary + let index = (this.questionIndex + 1) % this.questions.length; + while (index < this.questions.length && this.questions[index].correct) { + index = (index + 1) % this.questions.length; + } + + if (this.questions[index].answered) { + this.questions[index].answered = false; + } + this.questionIndex = index; + } + + help(question: QuizQuestion) { + this.chat.messages$.pipe(take(1)).subscribe((messages) => { + const augmented = [ + ...this.chat.convertToOpenAI(messages), + { + role: 'system', + content: + 'The following is a question from a quiz you gave to the user.', + }, + { + role: 'system', + content: 'Keep in mind it is a multiple choice question.', + }, + { + role: 'system', + content: `Question: ${question.question}`, + }, + { + role: 'system', + content: `The following answer is provided strictly for you to paraphrase and help the user figure it out on their own.`, + }, + { + role: 'system', + content: `${question.answer}`, + }, + { + role: 'system', + content: + 'Do not, under any circumstance, give the user the answer to this question.', + }, + { + role: 'system', + content: 'Please starting your response with "hint: "', + }, + { + role: 'user', + content: + 'I need help trying to figure out this question, but I do not want you to tell me the answer.', + }, + ]; + this.chat + .post('/chat', { messages: augmented }) + .pipe(take(1)) + .subscribe((response) => { + this.hint = response; + }); + }); + } +} diff --git a/src/kc_angular/src/app/components/chat-components/message-templates/topic.message.ts b/src/kc_angular/src/app/components/chat-components/message-templates/topic.message.ts new file mode 100644 index 00000000..ff5757c8 --- /dev/null +++ b/src/kc_angular/src/app/components/chat-components/message-templates/topic.message.ts @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, Input, OnChanges, SimpleChanges } from '@angular/core'; +import { fadeIn } from '@app/animations'; +import { ChatMessage } from '@app/models/chat.model'; +import { ChatService } from '@services/chat-services/chat.service'; +import { KcProject } from '@app/models/project.model'; + +export interface TopicDetails { + topic: string; + description?: string; + added?: boolean; + keywords?: string[]; + passages?: string[]; + projects?: KcProject[]; +} + +@Component({ + selector: 'chat-topic', + template: ` +
+
+
+
Topics
+
+
+
+ +
+
+
+
+ {{ topic.topic }} +
+
+ +
+
+
+
+
+
+
+
+ {{ description }} +
+ +
+ Click on a topic to learn more +
+
+
+
+ `, + styles: [ + ` + :host { + width: 100%; + } + `, + ], + animations: [fadeIn], +}) +export class TopicMessage implements OnChanges { + @Input() message!: ChatMessage; + + topics: TopicDetails[] = []; + + description?: string = ''; + + constructor(private chat: ChatService) {} + + ngOnChanges(changes: SimpleChanges) { + if (changes.message) { + this.topics = this.parseTopics(this.message.text); + + for (const topic of this.topics) { + topic.added = + this.message.source?.topics?.includes(topic.topic) || + this.message.project?.topics?.includes(topic.topic); + } + + setTimeout(() => { + this.onClick(this.topics[0]); + }); + } + } + + private parseTopics(text: string): TopicDetails[] { + const details: TopicDetails[] = []; + let topics: string[] = text.split('\n'); + + // Filter out empty strings + topics = topics.filter( + (topic) => topic.trim() !== '' && topic.startsWith('-') + ); + + for (const topic of topics) { + const split = topic.split(':'); + const topicName = split[0].trim().substring(1).trim(); + const description = split[1]?.trim() ?? ''; + details.push({ topic: topicName, description }); + } + + return details; + } + + addTopic(topic: TopicDetails) { + topic.added = true; + this.chat.addTopic(topic.topic); + } + + onClick(topic: TopicDetails) { + this.description = + this.description === topic?.description ? '' : topic?.description; + } +} diff --git a/src/kc_angular/src/app/components/chat.component.ts b/src/kc_angular/src/app/components/chat.component.ts index 8ed3030b..9f2b5d23 100755 --- a/src/kc_angular/src/app/components/chat.component.ts +++ b/src/kc_angular/src/app/components/chat.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,309 +14,30 @@ * limitations under the License. */ -import { Component, ViewChild } from '@angular/core'; -import { DataService } from '@services/user-services/data.service'; -import { finalize, map, take, tap } from 'rxjs/operators'; -import { KnowledgeSource } from '../models/knowledge.source.model'; -import { KcProject } from '../models/project.model'; -import { ContextMenu } from 'primeng/contextmenu'; -import { MenuItem } from 'primeng/api'; +import { Component, OnDestroy } from '@angular/core'; import { ProjectService } from '@services/factory-services/project.service'; import { ChatService } from '@services/chat-services/chat.service'; -import { ProjectChat } from '@services/chat-services/project'; -import { ChatViewComponent } from './chat-components/chat.view.component'; -import { SourceChat } from '@services/chat-services/source'; -import { AgentType, ChatMessage } from '@app/models/chat.model'; -import { SettingsService } from '@services/ipc-services/settings.service'; -import { combineLatest } from 'rxjs'; -import { ChatCompletionMessage } from 'openai/resources/chat/completions'; +import { Subscription } from 'rxjs'; @Component({ selector: 'app-chat', template: `
- +
- - `, styles: [``], }) -export class ChatComponent { - /* The context menu for the chat view */ - @ViewChild('cm') cm!: ContextMenu; +export class ChatComponent implements OnDestroy { + subscription: Subscription; - /* The chat view component */ - @ViewChild('chatView') chatView!: ChatViewComponent; - - /* The chat history */ - chatHistory: ChatMessage[] = []; - - /* The loading state of the chat, used to show the loading bar */ - loading = false; - - /* The knowledge sources for the current project */ - sources: KnowledgeSource[] = []; - - /* The current project */ - project!: KcProject; - - /* The context menu items */ - menuItems: MenuItem[] = []; - - showSources = true; - - constructor( - private data: DataService, - private chat: ChatService, - private projects: ProjectService, - private projectChat: ProjectChat, - private settings: SettingsService, - private sourceChat: SourceChat - ) { - combineLatest([ - settings.app.pipe(map((app) => app.chat.display.sourceMessages)), - projects.currentProject.pipe( - map((project) => project || new KcProject('', { value: '' })) - ), - ]) - .pipe( - map(([show, project]) => { - this.showSources = show; - this.chatHistory = []; - this.sources = []; - this.project = project; - let sources: KnowledgeSource[] = project.knowledgeSource; - - // Get all the sources for the current project and its subprojects - const subProjects = [...project.subprojects]; - - while (subProjects.length > 0) { - const subProject = subProjects.pop(); - - if (subProject) { - const sub = this.projects.getProject(subProject); - if (sub) { - for (const source of sub.knowledgeSource) { - sources = [...sources, source]; - } - for (const sp of sub.subprojects) { - subProjects.push(sp); - } - } - } - } - - this.sources = sources; - this.loadHistory(); - }) - ) - .subscribe(); - } - - /** - * Save the chat history for the current project and all sources - */ - saveHistory() { - if (this.project) { - const projectHistory = this.chatHistory.filter( - (message) => message.project?.id.value === this.project?.id.value - ); - this.chat.saveChat(projectHistory, this.project?.id); - } - - // Save the individual history for all sources - let sources = this.chatHistory.map((message) => message.source); - - // Get unique sources - sources = sources.filter((source, index) => { - return sources.indexOf(source) === index; + constructor(private chat: ChatService, private projects: ProjectService) { + this.subscription = this.projects.currentProject.subscribe((project) => { + this.chat.setTarget({ project: project }); }); - - for (const source of sources) { - if (source) { - const sourceHistory = this.chatHistory.filter( - (message) => message.source?.id.value === source?.id.value - ); - this.chat.saveChat(sourceHistory, source.id); - } - } } - /** - * Load the chat history for the current project and all sources - */ - loadHistory() { - this.loading = true; - this.chatHistory = []; - - if (this.project?.id) { - const projectChat = this.chat.loadChat( - this.project.id, - this.project, - undefined - ); - for (const message of projectChat) { - this.chatHistory.push(message); - } - } - - // Load the source history - if (this.showSources) { - for (const source of this.sources) { - const sourceChat = this.chat.loadChat(source.id, undefined, source); - for (const message of sourceChat) { - this.chatHistory.push(message); - } - } - } - - // Sort history by timestamp - this.chatHistory = this.chatHistory.sort( - (a, b) => a.timestamp.getTime() - b.timestamp.getTime() - ); - this.loading = false; - } - - /** - * Create a new chat message and send it to the chat service on submit - * @param $event The text of the message - */ - onSubmit($event: string) { - this.loading = true; - - if (this.chatHistory.length === 0) { - this.chatHistory = []; - } - - this.chatHistory.push( - this.chat.createMessage( - AgentType.User, - AgentType.Project, - $event, - this.project - ) - ); - this.chatView.scroll(); - - this.projectChat - .send(this.project, this.chatHistory) - .pipe( - take(1), - tap((response: ChatCompletionMessage) => { - if (response.content) { - this.chatHistory.push( - this.chat.createMessage( - AgentType.Project, - AgentType.User, - response.content, - this.project - ) - ); - } - this.saveHistory(); - this.chatView.scroll(); - }), - finalize(() => { - // If the user message was not responded to, remove it from the history - if ( - this.chatHistory[this.chatHistory.length - 1].sender !== - AgentType.Project - ) { - // Remove first element - this.chatHistory.shift(); - } - this.loading = false; - }) - ) - .subscribe(); - } - - /** - * Delete a message from the history - * @param messages The message(s) to delete - */ - onDeleteMessage(messages: ChatMessage[]) { - // Remove messages from the history and save - this.chatHistory = this.chatHistory.filter( - (message) => !messages.includes(message) - ); - } - - /** - * Regenerate a message by sending it and its history to the chat service - * @param message The message to regenerate - */ - regenerateMessage(message: ChatMessage) { - if (!this.project) { - return; - } - - this.loading = true; - - /* Use the message index to get the history up to that point - * Source history should only include messages to/from the same source - * Project history should include messages from the project and any sources that are part of the project */ - const historySlice = this.chat.getHistory( - message, - this.chatHistory, - false, - !message.project - ); - - /* Create a new message with the same text as the message that was responded to */ - const promptMessage = historySlice[historySlice.length - 1]; - - if (message.project) { - this.projectChat - .send(message.project, historySlice, promptMessage.text) - .pipe( - take(1), - tap((response: ChatCompletionMessage) => { - if (response.content) { - message.text = response.content; - message.regenerated = true; - this.saveHistory(); - } - }), - finalize(() => { - this.loading = false; - }) - ) - .subscribe(); - } else if (message.source) { - this.sourceChat - .send(message.source, historySlice, promptMessage.text) - .pipe( - take(1), - tap((response: ChatCompletionMessage) => { - if (response.content) { - message.text = response.content; - message.regenerated = true; - this.saveHistory(); - } - }), - finalize(() => { - this.loading = false; - }) - ) - .subscribe(); - } + ngOnDestroy() { + this.subscription.unsubscribe(); } } diff --git a/src/kc_angular/src/app/components/graph.component.ts b/src/kc_angular/src/app/components/graph.component.ts index 2bb7df8b..52bd84c8 100755 --- a/src/kc_angular/src/app/components/graph.component.ts +++ b/src/kc_angular/src/app/components/graph.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -63,7 +63,7 @@ import { createGraph } from '@app/workers/graph.worker';
diff --git a/src/kc_angular/src/app/components/home.component.ts b/src/kc_angular/src/app/components/home.component.ts index 8b7eabb3..eb8d7b25 100755 --- a/src/kc_angular/src/app/components/home.component.ts +++ b/src/kc_angular/src/app/components/home.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023 Rob Royce + * Copyright (c) 2022-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ import { finalize, map, take, takeUntil, tap } from 'rxjs/operators'; template: `
- - - - - - - -
-
-
- Project Type: {{ kcProject.type || 'Default' | titlecase - }}
- Date Created: {{ kcProject.dateCreated | date : 'medium' - }}
- Last Modified: {{ kcProject.dateModified | date : 'medium' - }}
- Last Accessed: {{ kcProject.dateAccessed | date : 'medium' - }}
-
-
Topics: None
-
- Knowledge Sources: {{ kcProject.knowledgeSource.length - }}
- Subprojects: {{ kcProject.subprojects.length }} -
- Description: -
- {{ kcProject.description || 'None' }}
-
-
-
-
- - - -
-
Subprojects
- - - {{ subproject | projectName }} - - - None - -
-
- - -
- Topics: - -
-
- `, - styles: [], -}) -export class ProjectCardComponent { - /** - * The project to be displayed on this card. - */ - @Input() kcProject?: KcProject | null; - - /** - * Whether to display the "Archive Project" button. - * Default is false. - */ - @Input() showArchive = false; - - /** - * Whether to display the "Remove Project" button. - * Default is true. - */ - @Input() showRemove = true; - - /** - * Whether to display the "Edit Project" button. - * Default is true. - */ - @Input() showEdit = true; - - /** - * Whether to display the "Add Subproject" button. - * Default is true. - */ - @Input() showAddSubproject = true; - - /** - * Whether to display the "GoTo Project" button. - * Default is true. - */ - @Input() showNavigate = true; - - /** - * Whether to display the "GoTo Project" button for subprojects. - * Default is true. - */ - @Input() showSubprojectNavigate = true; - - /** - * Emitted when the "edit" button is pressed. - * Contains the ID of the project to be edited. - */ - @Output() onEditProject = new EventEmitter(); - - /** - * Emitted when the "GoTo" button is pressed. - * Contains the ID of the project to be navigated to. - */ - @Output() onNavigateToProject = new EventEmitter(); - - /** - * Emitted when the "Add Subproject" button is pressed. - * Contains the ID of the parent project. - */ - @Output() onAddSubproject = new EventEmitter(); - - /** - * Emitted when the "Remove Project" button is pressed. - * Contains the ID of the project to be removed. - */ - @Output() onRemoveProject = new EventEmitter(); - - /** - * Emitted when the "Archive Project" button is pressed. - * Contains the ID of the project to be archived. - */ - @Output() onArchiveProject = new EventEmitter(); - - topics: string[] = []; - - constructor(private data: DataService) { - data.ksList.subscribe((ksList) => { - if (ksList.length > 0) { - this.topics = Array.from( - new Set( - ksList - .map((ks) => ks.topics) - .reduce((previousValue, currentValue) => { - if (previousValue && currentValue) { - return previousValue.concat(currentValue); - } else if (previousValue) { - return previousValue; - } else if (currentValue) { - return currentValue; - } else { - return []; - } - }) - ) - ); - } else { - this.topics = []; - } - }); - } - - /** - * Convenience method that transforms a project ID string to UuidModel - * @param id - */ - navigate(id: string) { - this.onNavigateToProject.emit({ value: id }); - } -} diff --git a/src/kc_angular/src/app/components/project-components/project-details.component.ts b/src/kc_angular/src/app/components/project-components/project-details.component.ts index 2db01d45..a5d4f0e7 100755 --- a/src/kc_angular/src/app/components/project-components/project-details.component.ts +++ b/src/kc_angular/src/app/components/project-components/project-details.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,7 +55,7 @@ import { PrimeIcons } from 'primeng/api';
diff --git a/src/kc_angular/src/app/components/project-components/projects-tree.component.ts b/src/kc_angular/src/app/components/project-components/projects-tree.component.ts index 29735bbe..1c801a75 100755 --- a/src/kc_angular/src/app/components/project-components/projects-tree.component.ts +++ b/src/kc_angular/src/app/components/project-components/projects-tree.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023 Rob Royce + * Copyright (c) 2022-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,7 +47,7 @@ import { DragAndDropService } from '@services/ingest-services/drag-and-drop.serv class="h-full" emptyMessage=" " selectionMode="single" - styleClass="border-1" + styleClass="border-1 bg-primary-reverse text-color" scrollHeight="flex" [draggableNodes]="true" [droppableNodes]="true" @@ -61,8 +61,6 @@ import { DragAndDropService } from '@services/ingest-services/drag-and-drop.serv [selection]="currentProject" (onNodeSelect)="selectionChange($event)" (onNodeContextMenuSelect)="onContextMenu($event)" - (onNodeCollapse)="onNodeCollapse($event, true)" - (onNodeExpand)="onNodeCollapse($event, false)" >
@@ -161,8 +159,6 @@ import { DragAndDropService } from '@services/ingest-services/drag-and-drop.serv ], }) export class ProjectsTreeComponent implements OnInit, OnDestroy { - @ViewChild('projectTreeElement', { static: true }) projectTreeElement: any; - @ViewChild('nodeLabel', { static: true }) nodeLabel!: ElementRef; projectId = ''; @@ -246,7 +242,7 @@ export class ProjectsTreeComponent implements OnInit, OnDestroy { } ngOnInit(): void { - this.scrollToActive(1000); + this.projects.scrollToActive(1000); } ngOnDestroy() { @@ -273,13 +269,13 @@ export class ProjectsTreeComponent implements OnInit, OnDestroy { } } - expandAll = async (root: TreeNode[], expand: boolean) => { + expandAll(root: TreeNode[], expand: boolean) { for (const t of root) { t.expanded = expand; if (t.children && t.children.length > 0) - await this.expandAll(t.children, expand); + this.expandAll(t.children, expand); } - }; + } onContextMenu($event: any) { if ($event.node) { @@ -293,36 +289,10 @@ export class ProjectsTreeComponent implements OnInit, OnDestroy { } } - scrollToActive(timeout = 0) { - setTimeout(() => { - const classElement = document.getElementsByClassName( - 'p-treenode-content p-treenode-selectable p-highlight' - ); - if (classElement.length > 0) { - classElement[0].scrollIntoView({ behavior: 'smooth' }); - } - }, timeout); - } - showSelected() { if (this.currentProject) { this.expandPath(this.currentProject); - this.scrollToActive(); - } - } - - onNodeCollapse($event: any, collapsed: boolean) { - const node = $event.node; - if (node?.key) { - const project = this.projects.getProject(node.key); - if (project) { - this.projects.updateProjects([ - { - id: project.id, - expanded: !collapsed, - }, - ]); - } + this.projects.scrollToActive(); } } diff --git a/src/kc_angular/src/app/components/settings/chat-settings.component.ts b/src/kc_angular/src/app/components/settings/chat-settings.component.ts index c8b40cb3..4f075188 100755 --- a/src/kc_angular/src/app/components/settings/chat-settings.component.ts +++ b/src/kc_angular/src/app/components/settings/chat-settings.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,79 +30,10 @@ import { ChatModel, SupportedChatModels } from '@shared/models/chat.model';
- +
-
Chat Settings
-
-
- -
- - - - - - - - - - - - - - - -
-
-
-
- -
- - -
-
OpenAI API
+
Model
@@ -287,10 +218,6 @@ export class ChatSettingsComponent { this.canChat = this.chat.canChat(); this.form = this.formBuilder.group({ - suggestionsEnabled: [this.chatSettings.suggestions.enabled], - suggestionsOnInput: [this.chatSettings.suggestions.onInput], - introductions: [this.chatSettings.display.introductions], - sourceMessages: [this.chatSettings.display.sourceMessages], modelName: [this.chatSettings.model.name], temperature: [this.chatSettings.model.temperature], top_p: [this.chatSettings.model.top_p], @@ -298,7 +225,6 @@ export class ChatSettingsComponent { presence_penalty: [this.chatSettings.model.presence_penalty], frequency_penalty: [this.chatSettings.model.frequency_penalty], }); - this.disable(); this.form.valueChanges .pipe( @@ -334,14 +260,7 @@ export class ChatSettingsComponent { model.frequency_penalty = formValue.frequency_penalty; const chatSettings: ChatSettingsModel = { - display: { - introductions: formValue.introductions, - sourceMessages: formValue.sourceMessages, - }, - suggestions: { - enabled: formValue.suggestionsEnabled, - onInput: formValue.suggestionsOnInput, - }, + suggestions: { enabled: false, onInput: false }, model: model, }; @@ -350,7 +269,6 @@ export class ChatSettingsComponent { } this.chatSettings = chatSettings; - this.disable(); this.set(); setTimeout(() => { @@ -426,12 +344,6 @@ export class ChatSettingsComponent { return JSON.stringify(prev) === JSON.stringify(curr); } - private disable() { - this.chatSettings.suggestions.enabled - ? this.form.get('suggestionsOnInput')?.enable() - : this.form.get('suggestionsOnInput')?.disable(); - } - private set() { this.settings.set({ app: { diff --git a/src/kc_angular/src/app/components/settings/display-settings.component.ts b/src/kc_angular/src/app/components/settings/display-settings.component.ts index 5b595ed9..b62972f5 100755 --- a/src/kc_angular/src/app/components/settings/display-settings.component.ts +++ b/src/kc_angular/src/app/components/settings/display-settings.component.ts @@ -28,7 +28,7 @@ import { debounceTime, distinctUntilChanged, tap } from 'rxjs/operators';
- +
Appearance
@@ -107,7 +107,7 @@ import { debounceTime, distinctUntilChanged, tap } from 'rxjs/operators';
- +
Video
@@ -130,7 +130,7 @@ import { debounceTime, distinctUntilChanged, tap } from 'rxjs/operators';
- +
Animations
diff --git a/src/kc_angular/src/app/components/settings/graph-settings.component.ts b/src/kc_angular/src/app/components/settings/graph-settings.component.ts index e99a4338..063a9bcd 100755 --- a/src/kc_angular/src/app/components/settings/graph-settings.component.ts +++ b/src/kc_angular/src/app/components/settings/graph-settings.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023 Rob Royce + * Copyright (c) 2022-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ import { debounceTime, distinctUntilChanged, tap } from 'rxjs/operators';
-
Graph Settings
+
General
@@ -109,10 +109,10 @@ import { debounceTime, distinctUntilChanged, tap } from 'rxjs/operators';
- +
-
Animation Settings
+
Animation
@@ -151,10 +151,10 @@ import { debounceTime, distinctUntilChanged, tap } from 'rxjs/operators';
- +
-
Physics Simulator
+
Physics
diff --git a/src/kc_angular/src/app/components/settings/ingest-settings.component.ts b/src/kc_angular/src/app/components/settings/ingest-settings.component.ts index 752a21d9..2ec9cd6b 100755 --- a/src/kc_angular/src/app/components/settings/ingest-settings.component.ts +++ b/src/kc_angular/src/app/components/settings/ingest-settings.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ export type SelectButtonOption = {
- +
Autoscan
@@ -107,7 +107,7 @@ export type SelectButtonOption = {
- +
File Manager
@@ -172,7 +172,7 @@ export type SelectButtonOption = {
- +
Browser Extensions
diff --git a/src/kc_angular/src/app/components/settings/search-settings.component.ts b/src/kc_angular/src/app/components/settings/search-settings.component.ts index 28471087..23aec7cd 100755 --- a/src/kc_angular/src/app/components/settings/search-settings.component.ts +++ b/src/kc_angular/src/app/components/settings/search-settings.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ import { debounceTime, distinctUntilChanged, tap } from 'rxjs/operators';
- +
Web Search
@@ -53,7 +53,7 @@ import { debounceTime, distinctUntilChanged, tap } from 'rxjs/operators';
- +
Local Search
diff --git a/src/kc_angular/src/app/components/settings/storage-settings.component.ts b/src/kc_angular/src/app/components/settings/storage-settings.component.ts index 79ddd894..86647a5f 100755 --- a/src/kc_angular/src/app/components/settings/storage-settings.component.ts +++ b/src/kc_angular/src/app/components/settings/storage-settings.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ import { NotificationsService } from '@services/user-services/notifications.serv
- +
Import/Export
diff --git a/src/kc_angular/src/app/components/shared/create.component.ts b/src/kc_angular/src/app/components/shared/create.component.ts index 6e4508ce..b825741e 100755 --- a/src/kc_angular/src/app/components/shared/create.component.ts +++ b/src/kc_angular/src/app/components/shared/create.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023 Rob Royce + * Copyright (c) 2022-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -77,18 +77,16 @@ import { ConfirmationService } from 'primeng/api'; (change)="onNewFile($event)" /> @@ -132,6 +130,10 @@ import { ConfirmationService } from 'primeng/api'; + + + + `, styles: [], }) @@ -142,6 +144,8 @@ export class CreateComponent { files: any[] = []; + showImportDialog = false; + constructor( private dialog: DialogService, private confirm: ConfirmationService, diff --git a/src/kc_angular/src/app/components/shared/web.import.component.ts b/src/kc_angular/src/app/components/shared/web.import.component.ts new file mode 100644 index 00000000..065fbef5 --- /dev/null +++ b/src/kc_angular/src/app/components/shared/web.import.component.ts @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Component, + EventEmitter, + HostListener, + Output, + ViewChild, +} from '@angular/core'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { + KnowledgeSourceFactoryRequest, + KsFactoryService, +} from '@services/factory-services/ks-factory.service'; +import { KnowledgeSource } from '@app/models/knowledge.source.model'; +import { KsCommandService } from '@services/command-services/ks-command.service'; +import { IngestService } from '@services/ingest-services/ingest.service'; +import { ProjectService } from '@services/factory-services/project.service'; +import { tap } from 'rxjs'; +import { ProjectUpdateRequest } from '@app/models/project.model'; + +@Component({ + selector: 'app-import-web', + template: ` +
+ +
+ +
+
+ + + + + Title + + + + + + + + + + + + + + + + + + + + + +
+
+ No Sources +
+
+
+
+ +
+ + +
+
+ `, +}) +export class WebImportComponent { + @Output() close = new EventEmitter(); + + @ViewChild('linkInput', { static: false }) linkInput: any; + + linkForm: FormGroup; + + sources: KnowledgeSource[] = []; + + destinations: { name: string; code: string }[] = [ + { name: 'Inbox', code: '1' }, + { name: 'Project', code: '2' }, + ]; + + destination: { name: string; code: string } = this.destinations[0]; + + constructor( + private formBuilder: FormBuilder, + private factory: KsFactoryService, + private command: KsCommandService, + private ingest: IngestService, + private projects: ProjectService + ) { + this.linkForm = formBuilder.group({ + url: ['', [Validators.required, Validators.pattern('https?://.+[.]+.+')]], + }); + + projects.currentProject + .pipe( + tap((project) => { + if (project) { + this.destinations = [ + { name: 'Inbox', code: '1' }, + { name: `Project (${project.name})`, code: '2' }, + ]; + } + }) + ) + .subscribe(); + } + + // HostListener for pasting text + @HostListener('paste', ['$event']) + onPaste(event: ClipboardEvent) { + const text = event.clipboardData?.getData('text'); + + // Get all URLs in text + const regex = /https?:\/\/[^\s]+/g; + const matches = text?.match(regex); + + // Add URLs to array + if (matches) { + const urls: string[] = []; + matches.forEach((match) => { + // Remove commas from end of URLs + if (match.endsWith(',')) { + match = match.slice(0, -1); + } + urls.push(match); + }); + this.createSource(urls); + } + + // Clear form by resetting URL field, making sure the value is updated + setTimeout(() => { + this.linkForm.setValue({ + url: '', + }); + this.linkForm.reset(); + }); + } + + onSubmit(value: string) {} + + createSource(urls: string[]) { + const requests: KnowledgeSourceFactoryRequest = { + ingestType: 'website', + links: urls, + }; + + this.factory.many(requests).then((sources) => { + for (const source of sources) { + this.sources.push(source); + } + }); + } + + addFlag(source: KnowledgeSource) { + source.flagged = !source.flagged; + } + + removeSource(source: KnowledgeSource) { + this.sources = this.sources.filter((s) => s !== source); + } + + openSource(source: KnowledgeSource) { + this.command.open(source); + } + + previewSource(source: KnowledgeSource) { + this.command.preview(source); + } + + import(destination: string) { + console.log('Importing to ' + destination); + + if (destination === 'Inbox') { + this.ingest.enqueue(this.sources); + } else { + const project = this.projects.getCurrentProjectId(); + if (project) { + const req: ProjectUpdateRequest = { + id: project, + addKnowledgeSource: this.sources, + }; + this.projects.updateProjects([req]); + } + } + + this.linkForm.reset(); + this.sources = []; + this.close.emit(); + } +} diff --git a/src/kc_angular/src/app/components/source-components/ks-card-list.component.html b/src/kc_angular/src/app/components/source-components/ks-card-list.component.html index e4ce7d10..77431767 100755 --- a/src/kc_angular/src/app/components/source-components/ks-card-list.component.html +++ b/src/kc_angular/src/app/components/source-components/ks-card-list.component.html @@ -189,7 +189,7 @@ (onShow)="onKsContextMenu()" [model]="ksMenuItems" appendTo="body" - styleClass="shadow-7" + styleClass="shadow-7 bg-primary-reverse" > diff --git a/src/kc_angular/src/app/components/source-components/ks-card.component.ts b/src/kc_angular/src/app/components/source-components/ks-card.component.ts index f96865ff..ca394941 100755 --- a/src/kc_angular/src/app/components/source-components/ks-card.component.ts +++ b/src/kc_angular/src/app/components/source-components/ks-card.component.ts @@ -35,7 +35,7 @@ import { DragAndDropService } from '@services/ingest-services/drag-and-drop.serv selector: 'app-ks-card', template: `
@@ -184,6 +185,14 @@ import { DragAndDropService } from '@services/ingest-services/drag-and-drop.serv cursor: pointer; } } + + app-action-bar { + opacity: 0.35; + } + + :host:hover app-action-bar { + opacity: 1; + } `, ], }) @@ -254,6 +263,9 @@ export class KsCardComponent implements OnDestroy, OnChanges { /* Set an optional label */ @Input() label?: string; + /* Determines whether to allow dragging of the card (default: true) */ + @Input() allowDrag = true; + /* EventEmitter that is triggered when the "Remove" button is pressed */ @Output() onRemove = new EventEmitter(); @@ -425,10 +437,59 @@ export class KsCardComponent implements OnDestroy, OnChanges { } dragStart($event: DragEvent, ks: KnowledgeSource) { - this.dnd.dragSource($event, ks); + if (this.allowDrag) { + this.dnd.dragSource($event, ks); + } } dragEnd($event: any, rowData: any) { - this.dnd.dragSourceEnd($event, rowData); + if (this.allowDrag) { + this.dnd.dragSourceEnd($event, rowData); + } + } + + openSource() { + if (this.onOpen.observed) { + this.onOpen.emit(this.ks); + } else { + this.command.open(this.ks); + } + } + + removeSource() { + if (this.onRemove.observed) { + this.onRemove.emit(this.ks); + } else { + this.command.remove([this.ks]); + } + } + + previewSource() { + if (this.onPreview.observed) { + this.onPreview.emit(this.ks); + } else { + this.command.preview(this.ks); + } + } + + editSource() { + if (this.onEdit.observed) { + this.onEdit.emit(this.ks); + } else { + this.command.detail(this.ks); + } + } + + chatSource() { + if (this.onChat.observed) { + this.onChat.emit(this.ks); + } else { + this.command.chat(this.ks); + } + } + + flagSource(checked: boolean) { + this.ks.flagged = checked; + this.command.update([this.ks]); } } diff --git a/src/kc_angular/src/app/components/source-components/ks-message.component.ts b/src/kc_angular/src/app/components/source-components/ks-message.component.ts index 27bc1693..01975735 100755 --- a/src/kc_angular/src/app/components/source-components/ks-message.component.ts +++ b/src/kc_angular/src/app/components/source-components/ks-message.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023 Rob Royce + * Copyright (c) 2022-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ import { KnowledgeSource } from '@app/models/knowledge.source.model'; template: `
diff --git a/src/kc_angular/src/app/components/source-components/ks-table.component.ts b/src/kc_angular/src/app/components/source-components/ks-table.component.ts index 8841bf12..2740515c 100755 --- a/src/kc_angular/src/app/components/source-components/ks-table.component.ts +++ b/src/kc_angular/src/app/components/source-components/ks-table.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -215,32 +215,6 @@ import { DragAndDropService } from '@services/ingest-services/drag-and-drop.serv {{ rowData[col.field] | date : 'mediumDate' }}
- -
-
-
-
-
-
- {{ rowData[col.field] | date : 'mediumDate' }} -
-
- {{ rowData[col.field] | countdown }} -
-
- -
- {{ rowData[col.field] | date : 'mediumDate' }} -
-
- {{ rowData[col.field] | countdown }} -
-
-
-
-
@@ -348,7 +322,6 @@ export class KsTableComponent implements OnInit, OnChanges { { field: 'icon', header: '' }, { field: 'title', header: 'Title' }, { field: 'associatedProject', header: 'Project' }, - { field: 'dateDue', header: 'Due Date' }, { field: 'dateCreated', header: 'Created' }, { field: 'dateAccessed', header: 'Accessed' }, { field: 'dateModified', header: 'Modified' }, @@ -439,6 +412,9 @@ export class KsTableComponent implements OnInit, OnChanges { return nA < nB ? 1 : nA > nB ? -1 : 0; }); }); + + // Set pagination to first page + this.first = 0; } } @@ -476,7 +452,6 @@ export class KsTableComponent implements OnInit, OnChanges { }); break; case 'dateCreated': - case 'dateDue': event.data.sort((d1, d2) => { if (!event.field || !event.order) { return 0; diff --git a/src/kc_angular/src/app/components/source-components/ks-thumbnail.component.ts b/src/kc_angular/src/app/components/source-components/ks-thumbnail.component.ts index 94aba4f8..49d05ca8 100755 --- a/src/kc_angular/src/app/components/source-components/ks-thumbnail.component.ts +++ b/src/kc_angular/src/app/components/source-components/ks-thumbnail.component.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023 Rob Royce + * Copyright (c) 2022-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -47,8 +47,7 @@ import { NotificationsService } from '@services/user-services/notifications.serv
@@ -153,7 +152,7 @@ export class KsThumbnailComponent implements OnDestroy, OnChanges { this._thumbnail$.next(this.ks.thumbnail); } - const meta = this.ks.reference.source.website?.metadata?.meta; + const meta = this.ks.meta; if (meta) { const ogImage = meta.find((m) => m.key === 'og:image'); if (ogImage && ogImage.value) { diff --git a/src/kc_angular/src/app/components/source-components/source-note.component.ts b/src/kc_angular/src/app/components/source-components/source-note.component.ts new file mode 100755 index 00000000..edb75f2b --- /dev/null +++ b/src/kc_angular/src/app/components/source-components/source-note.component.ts @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2023-2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Component, EventEmitter, Input, Output } from '@angular/core'; +import { SourceNote } from '@app/models/knowledge.source.model'; + +@Component({ + selector: 'source-note', + template: ` +
+
+
+ {{ note.title }} +
+
+ {{ note.content }} +
+
+
+
+ + +
+
+
+ `, + styles: [ + ` + :host { + } + `, + ], +}) +export class SourceNoteComponent { + @Input() note?: SourceNote; + + @Output() addNote = new EventEmitter(); + + constructor() {} +} diff --git a/src/kc_angular/src/app/components/source-components/source.chat.component.ts b/src/kc_angular/src/app/components/source-components/source.chat.component.ts index b881d6cf..cee1f270 100755 --- a/src/kc_angular/src/app/components/source-components/source.chat.component.ts +++ b/src/kc_angular/src/app/components/source-components/source.chat.component.ts @@ -16,27 +16,14 @@ import { Component, OnInit, ViewChild } from '@angular/core'; import { KnowledgeSource } from '@app/models/knowledge.source.model'; -import { AgentType, ChatMessage } from '@app/models/chat.model'; import { ChatService } from '@services/chat-services/chat.service'; -import { SourceChat } from '@services/chat-services/source'; -import { finalize, take, tap } from 'rxjs/operators'; import { ChatViewComponent } from '@components/chat-components/chat.view.component'; -import { NotificationsService } from '@services/user-services/notifications.service'; -import { BehaviorSubject } from 'rxjs'; @Component({ selector: 'source-chat', template: `
- +
@@ -55,7 +42,6 @@ import { BehaviorSubject } from 'rxjs'; pInputText class="w-full p-fluid" placeholder="Type your question here, then press enter..." - (keydown.enter)="ask(question.value)" />
@@ -67,157 +53,18 @@ export class SourceChatComponent implements OnInit { source!: KnowledgeSource; // Must be passed in during dynamic component creation - history: ChatMessage[] = []; - - private _loading = new BehaviorSubject(false); - loading$ = this._loading.asObservable(); - showQuestion = false; context?: string; - constructor( - private chat: ChatService, - private sourceChat: SourceChat, - private notify: NotificationsService - ) {} + constructor(private chat: ChatService) {} ngOnInit() { - // Check if the source is a supported file type (PDF or other plain-text) - const supportedFileTypes = [ - '.pdf', - '.txt', - '.md', - '.html', - '.htm', - '.py', - '.js', - '.ts', - '.java', - '.c', - '.cpp', - '.h', - '.hpp', - '.cs', - '.go', - '.rs', - '.sh', - ]; - - // Check if the file is supported - if ( - this.source.ingestType === 'file' && - !supportedFileTypes.some((type) => - this.source.accessLink.toString().endsWith(type) - ) - ) { - this.notify.warn( - 'Source Chat', - 'File Chat', - 'Knowledge does not know how to read this file type. Chat features will be limited.', - 'toast', - 6000 - ); - } - - // Load history - this.history = this.chat.loadChat(this.source.id, undefined, this.source); - if (this.history.length === 0) { - this._loading.next(true); - this.addMessage( - AgentType.User, - AgentType.Source, - `Can you introduce me to "${this.source.title}"?` - ); - - this.sourceChat - .intro(this.source) - .pipe( - take(1), - tap((response) => { - if (response.content) { - this.addMessage( - AgentType.Source, - AgentType.User, - response.content - ); - } - }), - finalize(() => { - this._loading.next(false); - }) - ) - .subscribe(); - } + this.chat.setTarget({ source: this.source }); } showQuestionDialog(context?: string) { this.context = context; this.showQuestion = true; } - - addMessage(from: AgentType, to: AgentType, content: string) { - this.history.push( - this.chat.createMessage(from, to, content, undefined, this.source) - ); - if (this.chatView) { - this.chatView.scroll(); - } - this.chat.saveChat(this.history, this.source.id); - } - - submit($event: string) { - this.addMessage(AgentType.User, AgentType.Source, $event); - this._loading.next(true); - this.sourceChat - .send(this.source, this.history, $event) - .pipe( - take(1), - tap((response) => { - if (response.content) { - this.addMessage(AgentType.Source, AgentType.User, response.content); - } - }), - finalize(() => { - this._loading.next(false); - }) - ) - .subscribe(); - } - - regenerate(message: ChatMessage) { - this._loading.next(true); - this.sourceChat - .regenerate(this.source, message, this.history) - .pipe( - tap((response) => { - this.history = this.history.map((m) => { - if (m.id === message.id) { - m.text = response.content ?? ''; - } - return m; - }); - this.chat.saveChat(this.history, this.source.id); - }), - finalize(() => { - this._loading.next(false); - }) - ) - .subscribe(); - } - - delete(messages: ChatMessage[]) { - for (const message of messages) { - this.history = this.history.filter((m) => m.id !== message.id); - } - - this.chat.saveChat(this.history, this.source.id); - } - - ask(value: string) { - this.showQuestion = false; - this.submit( - `Context: ${this.context}\n\n\nPlease answer the following question: ${value}` - ); - } } diff --git a/src/kc_angular/src/app/components/source-components/source.component.ts b/src/kc_angular/src/app/components/source-components/source.component.ts index 9f475971..76a79e01 100755 --- a/src/kc_angular/src/app/components/source-components/source.component.ts +++ b/src/kc_angular/src/app/components/source-components/source.component.ts @@ -30,13 +30,11 @@ import { import { SourceChatComponent } from '@components/source-components/source.chat.component'; import { SourceDetailsComponent } from '@components/source-components/source.details.component'; import { KnowledgeSource } from '@app/models/knowledge.source.model'; -import { SourceMetadataComponent } from '@components/source-components/source.metadata.component'; import { PrimeIcons } from 'primeng/api'; import { SourceDocumentComponent } from '@components/source-components/source.document.component'; import { SourceVideoComponent } from '@components/source-components/source.video.component'; import { SourceTimelineComponent } from './source.timeline.component'; import { ChatService } from '@services/chat-services/chat.service'; -import { NotificationsService } from '@services/user-services/notifications.service'; import { SourceBrowserComponent } from '@components/source-components/source.browser.component'; interface TabDescriptor { @@ -56,6 +54,7 @@ interface TabDescriptor {
- + + + + {{ tab.label }}
@@ -90,19 +92,17 @@ interface TabDescriptor { .tabs { display: flex; width: 100%; - padding-left: 0.25rem !important; - padding-right: 0.25rem !important; } .tab { flex: 1; - padding: 10px; + padding: 1rem !important; cursor: pointer; background-color: var(--surface-ground); } .tab:not(.active) { - background-color: var(--surface-a); + background-color: var(--primary-color-text); } .tab.active { @@ -111,6 +111,10 @@ interface TabDescriptor { background-color: var(--surface-card) !important; } + .tab.loading { + cursor: wait; + } + .tab.disabled { cursor: not-allowed; opacity: 0.5; @@ -127,14 +131,14 @@ export class SourceComponent implements OnInit, OnChanges { tabContainer!: ViewContainerRef; detailsTab: TabDescriptor = { - label: 'Details', + label: 'Source', icon: PrimeIcons.INFO_CIRCLE, hidden: false, disabled: false, loading: false, component: SourceDetailsComponent, - tipHeader: 'Craving the Deets?', - tipMessage: `Head over to the details tab! It's got all the juicy details about the Source, including title, description, and topics. Information galore, right at your fingertips!`, + tipHeader: 'Source Details', + tipMessage: `This tab shows you all the details about the Source. It's like the Source's personal ID card, with all the important information you need to know.`, }; chatTab: TabDescriptor = { @@ -192,17 +196,6 @@ export class SourceComponent implements OnInit, OnChanges { tipMessage: `Meet the video tab! Your personal movie theater, playing Source videos straight from YouTube using the video ID in the Source link. Sit back, relax, and enjoy the show!`, }; - metadataTab: TabDescriptor = { - label: 'Metadata', - icon: PrimeIcons.HASHTAG, - hidden: false, - disabled: false, - loading: false, - component: SourceMetadataComponent, - tipHeader: 'Ready to go Meta?', - tipMessage: `Step into the metadata tab! It's your personal viewing deck for all things meta about the Source. And guess what? You can easily copy any of the metadata to your clipboard with a single click!`, - }; - tabs: TabDescriptor[] = [ this.detailsTab, this.chatTab, @@ -210,7 +203,6 @@ export class SourceComponent implements OnInit, OnChanges { this.browserTab, this.documentTab, this.videoTab, - this.metadataTab, ]; selectedTabIndex = 0; @@ -227,10 +219,11 @@ export class SourceComponent implements OnInit, OnChanges { private componentRef?: ComponentRef; - constructor( - private chat: ChatService, - private notify: NotificationsService - ) {} + constructor(private chat: ChatService) { + this.chat.loading$.subscribe((loading: boolean) => { + this.chatTab.loading = loading; + }); + } ngOnInit() { this.loadSelectedTab(); @@ -244,6 +237,7 @@ export class SourceComponent implements OnInit, OnChanges { if (changes.source) { this.views(changes.source.currentValue); this.loadSelectedTab(); + this.chat.setTarget({ source: this.source }); } if (changes.reset && changes.reset.currentValue) { @@ -254,10 +248,6 @@ export class SourceComponent implements OnInit, OnChanges { } views(source: KnowledgeSource) { - // Metadata is disabled if no metadata exists - this.metadataTab.disabled = - !this.source.reference.source.website?.metadata?.meta; - // If video, enable video tab and disable document tab if (source.ingestType === 'website') { source.accessLink = new URL(source.accessLink); @@ -281,7 +271,6 @@ export class SourceComponent implements OnInit, OnChanges { this.documentTab.disabled = this.documentTab.hidden = true; // If document, enable document table and disable video tab - const fileType = source.reference.source.file?.type; if ( `${source.accessLink}`.endsWith('pdf') || `${source.accessLink}`.endsWith('gif') || @@ -327,12 +316,12 @@ export class SourceComponent implements OnInit, OnChanges { private loadComponent(component: Type) { this.componentRef = this.tabContainer.createComponent(component); + this.componentRef.instance.source = this.source; // Special handling for components with outputs and other special cases switch (component) { case SourceChatComponent: - this.loadChat(this.componentRef); break; case SourceDetailsComponent: this.loadDetails(this.componentRef); @@ -353,14 +342,6 @@ export class SourceComponent implements OnInit, OnChanges { }); } - private loadChat(componentRef: ComponentRef) { - componentRef.instance.loading$.subscribe((loading: boolean) => { - setTimeout(() => { - this.chatTab.loading = loading; - }); - }); - } - private loadBrowser(componentRef: ComponentRef) { componentRef.instance.update.subscribe((source: KnowledgeSource) => { this.source = source; @@ -380,7 +361,7 @@ export class SourceComponent implements OnInit, OnChanges { input.message.substring(0, 8192) ); } else { - this.componentRef?.instance.submit(input.message); + this.chat.submit(input.message); } }, 1000); } diff --git a/src/kc_angular/src/app/components/source-components/source.details.component.ts b/src/kc_angular/src/app/components/source-components/source.details.component.ts index c19bdfc6..2d20e377 100755 --- a/src/kc_angular/src/app/components/source-components/source.details.component.ts +++ b/src/kc_angular/src/app/components/source-components/source.details.component.ts @@ -14,7 +14,14 @@ * limitations under the License. */ -import { Component, EventEmitter, OnInit, Output } from '@angular/core'; +import { + Component, + ElementRef, + EventEmitter, + OnInit, + Output, + ViewChild, +} from '@angular/core'; import { KnowledgeSource } from '@app/models/knowledge.source.model'; import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { debounceTime, distinctUntilChanged } from 'rxjs/operators'; @@ -23,263 +30,262 @@ import { ElectronIpcService } from '@services/ipc-services/electron-ipc.service' import { NotificationsService } from '@services/user-services/notifications.service'; import { SearchService } from '@services/user-services/search.service'; import { skip } from 'rxjs'; +import { Clipboard } from '@angular/cdk/clipboard'; @Component({ selector: 'source-details', template: ` -
+
-
- -
- - -
-
-
-
-
-
-
-

Title

- -
-
-

Topics

- - -
+ +
+
+ +
+
+
+ +
+
+ + +
+
+ + +
- -
+
+ + +
+
+
+
+
-
-
-
+

Metadata

+ + +
+ Click the + button to add metadata... +
+ + + + + Tag + + + + Value + + + + + + + -

Notes

-
- -
+ + + + + + + {{ meta.key }} + + + + + + + + + +
+ {{ meta.value }} +
+
+
+ + +
+
+ +
-
-
- -
-
-
-
-
- -
-
-
-
-
-
-
- -
-
-
-

Model

-
-
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - - -
-
-
-
-
+ + + + +
`, styleUrls: ['./source.styles.scss'], }) export class SourceDetailsComponent implements OnInit { + @ViewChild('tagInput') tagInput!: ElementRef; + + @Output() remove = new EventEmitter(); + + @Output() update = new EventEmitter(); + source!: KnowledgeSource; form: FormGroup; showMarkdownPreview = true; - @Output() remove = new EventEmitter(); + metaUpdates: { [id: number]: { key: string; value: string } } = {}; - @Output() update = new EventEmitter(); + metaEntry: { key: string; value: string } = { key: '', value: '' }; constructor( + private clipboard: Clipboard, private fb: FormBuilder, private command: KsCommandService, private ipc: ElectronIpcService, @@ -310,6 +316,52 @@ export class SourceDetailsComponent implements OnInit { ngOnInit() { this.form.patchValue(this.source); + + if (!this.source.meta) { + this.source.meta = []; + } + + if (this.source.reference.source.website?.metadata?.meta) { + for (const m of this.source.reference.source.website.metadata.meta) { + // Add to source.meta if it doesn't exist + if (!this.source.meta.find((meta) => meta.key === m.key)) { + if (m.key && m.value) { + this.source.meta.push({ key: m.key, value: m.value }); + } + } + } + delete this.source.reference.source.website.metadata.meta; + } + + this.source.meta = [ + { key: 'knowledge:id', value: this.source.id.value }, + { + key: 'knowledge:method', + value: this.source.importMethod ?? 'manual', + }, + { + key: 'knowledge:link', + value: this.source.accessLink.toString(), + }, + { + key: 'knowledge:type', + value: this.source.ingestType, + }, + ...this.source.meta, + ]; + + // Remove any duplicates + this.source.meta = this.source.meta.filter( + (meta, index, self) => + index === + self.findIndex((m) => m.key === meta.key && m.value === meta.value) + ); + + this.update.emit(this.source); + + for (let i = 0; i < this.source.meta.length; i++) { + this.source.meta[i].id = i; + } } flag() { @@ -342,4 +394,77 @@ export class SourceDetailsComponent implements OnInit { this.command.open(this.source); } } + + toClipboard(key: string) { + if (key && key.trim().length > 0) { + this.clipboard.copy(key); + this.notify.success('Source Info', 'Copied to Clipboard', ''); + } + } + + onRowEditSave(meta: { key: string; value: string }, ri: number) { + if (!meta.key) { + return; + } + const original = this.source.meta.find((m) => m.key === meta.key); + const updated = this.metaUpdates[ri]; + // If they are the same, do nothing + if (original && updated) { + if (original.value === updated.value && original.key === updated.key) { + return; + } else { + this.update.emit(this.source); + } + } + } + + onRowEditCancel(meta: { key: string; value: string }, ri: number) { + if (meta.key) { + this.source.meta[ri] = this.metaUpdates[ri]; + delete this.metaUpdates[ri]; + } + } + + onRowEditInit(meta: { key: string; value: string }, ri: number) { + this.metaUpdates[ri] = { ...meta }; + } + + addMeta(metaEntry: { key: string; value: string }) { + const entry = { ...metaEntry }; + + if (entry.key && entry.value) { + // If the key/value pair already exists, don't add it + if ( + this.source.meta.find( + (m) => m.key === entry.key && m.value === entry.value + ) + ) { + this.notify.warn( + 'Source Info', + 'Ignoring Duplicate', + 'That key/value pair already exists...', + 'toast' + ); + return; + } + + this.source.meta = [ + { key: entry.key, value: entry.value }, + ...this.source.meta, + ]; + this.metaEntry = { key: '', value: '' }; + this.update.emit(this.source); + this.tagInput.nativeElement.focus(); + } else { + this.notify.error('Source Info', 'Missing Key or Value', ''); + } + } + + onRowDelete(meta: { key: string; value: string }, ri: number) { + this.source.meta = this.source.meta.filter( + (m) => m.key !== meta.key && m.value !== meta.value + ); + delete this.metaUpdates[ri]; + this.update.emit(this.source); + } } diff --git a/src/kc_angular/src/app/components/source-components/source.metadata.component.ts b/src/kc_angular/src/app/components/source-components/source.metadata.component.ts deleted file mode 100755 index 00ecb367..00000000 --- a/src/kc_angular/src/app/components/source-components/source.metadata.component.ts +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2023 Rob Royce - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Component, OnInit } from '@angular/core'; -import { KnowledgeSource } from '@app/models/knowledge.source.model'; -import { WebsiteMetaTagsModel } from '@shared/models/web.source.model'; -import { NotificationsService } from '@services/user-services/notifications.service'; -import { Clipboard } from '@angular/cdk/clipboard'; -import { ElectronIpcService } from '@services/ipc-services/electron-ipc.service'; -import { KsCommandService } from '@services/command-services/ks-command.service'; - -@Component({ - selector: 'source-metadata', - template: ` - - `, - styleUrls: ['./source.styles.scss'], -}) -export class SourceMetadataComponent implements OnInit { - source!: KnowledgeSource; - - /* If the source is a website, this will contain the metadata for the website and sanitized to prevent XSS attacks */ - ksMetadata: WebsiteMetaTagsModel[] = []; - - constructor( - private clipboard: Clipboard, - private command: KsCommandService, - private ipc: ElectronIpcService, - private notify: NotificationsService - ) {} - - ngOnInit(): void { - this.ksMetadata = - this.source.reference.source.website?.metadata?.meta ?? []; - } - - /* When the user clicks on the "Copy" button, copy the source key to the clipboard */ - toClipboard(key: string) { - if (key && key.trim().length > 0) { - this.clipboard.copy(key); - this.notify.success('Source Info', 'Copied to Clipboard', ''); - } - } -} diff --git a/src/kc_angular/src/app/directives/chat-message.directive.ts b/src/kc_angular/src/app/directives/chat-message.directive.ts new file mode 100644 index 00000000..347f6e2d --- /dev/null +++ b/src/kc_angular/src/app/directives/chat-message.directive.ts @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + Directive, + Input, + OnChanges, + SimpleChanges, + ViewContainerRef, +} from '@angular/core'; +import { AgentType, ChatMessage } from '@app/models/chat.model'; +import { QuizMessage } from '@components/chat-components/message-templates/quiz.message'; +import { TopicMessage } from '@components/chat-components/message-templates/topic.message'; +import { ChatMessageComponent } from '@components/chat-components/chat.message.component'; +import { CategorizeMessage } from '@components/chat-components/message-templates/categorize.message'; + +@Directive({ + selector: '[chatMessage]', +}) +export class ChatMessageDirective implements OnChanges { + @Input() message!: ChatMessage; + + constructor(public viewContainerRef: ViewContainerRef) {} + + ngOnChanges(changes: SimpleChanges) { + if (changes.message) { + this.createComponent(this.message); + } + } + + private createComponent(message: ChatMessage) { + // Create a component based on the message type + // Use the viewContainerRef to create the component + let componentRef; + const parentDiv = this.viewContainerRef.element.nativeElement.parentElement; + + switch (message.sender) { + case AgentType.Category: + componentRef = this.viewContainerRef.createComponent(CategorizeMessage); + componentRef.location.nativeElement.classList.add('p-4', 'text-color'); + parentDiv.classList.add( + 'bg-primary-reverse', + 'border-1', + 'border-round-3xl', + 'flex-row-center-center', + 'hover:shadow-3', + 'm-4', + 'shadow-1', + 'surface-border', + 'w-full', + 'overflow-hidden' + ); + break; + case AgentType.Quiz: + componentRef = this.viewContainerRef.createComponent(QuizMessage); + parentDiv.classList.add( + 'bg-primary-reverse', + 'border-1', + 'border-round-3xl', + 'flex-row-center-center', + 'hover:shadow-3', + 'm-4', + 'shadow-1', + 'surface-border', + 'w-full', + 'overflow-hidden', + 'h-26rem' + ); + break; + case AgentType.Topic: + componentRef = this.viewContainerRef.createComponent(TopicMessage); + parentDiv.classList.add( + 'bg-primary-reverse', + 'border-1', + 'border-round-3xl', + 'flex-row-center-center', + 'hover:shadow-3', + 'm-4', + 'shadow-1', + 'surface-border', + 'w-full', + 'overflow-hidden' + ); + break; + case AgentType.User: + componentRef = + this.viewContainerRef.createComponent(ChatMessageComponent); + componentRef.location.nativeElement.classList.add( + 'shadow-1', + 'hover:shadow-3', + 'p-4', + 'border-round-3xl' + ); + parentDiv.classList.add('flex-row-center-end', 'px-3'); + break; + default: + componentRef = + this.viewContainerRef.createComponent(ChatMessageComponent); + componentRef.location.nativeElement.classList.add( + 'shadow-1', + 'hover:shadow-3', + 'p-4', + 'border-round-3xl' + ); + } + componentRef.setInput('message', message); + + return componentRef; + } +} diff --git a/src/kc_angular/src/app/models/chat.model.ts b/src/kc_angular/src/app/models/chat.model.ts index bf1a498a..c55d5e1a 100755 --- a/src/kc_angular/src/app/models/chat.model.ts +++ b/src/kc_angular/src/app/models/chat.model.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ import { KcProject } from './project.model'; import { KnowledgeSource } from './knowledge.source.model'; +import { ChatModelName } from '@shared/models/chat.model'; export enum AgentType { Assistant = 'assistant', @@ -24,6 +25,10 @@ export enum AgentType { Source = 'source', System = 'system', User = 'user', + Tutor = 'tutor', + Quiz = 'quiz', + Category = 'category', + Topic = 'topic', } export type MessageRating = 'thumbs-up' | 'thumbs-down' | 'none'; @@ -39,4 +44,6 @@ export interface ChatMessage { suggestion?: boolean; regenerated?: boolean; rating?: MessageRating; + ephemeral?: boolean; + model?: ChatModelName; } diff --git a/src/kc_angular/src/app/models/knowledge.source.model.ts b/src/kc_angular/src/app/models/knowledge.source.model.ts index 655d276c..3c3f2fde 100755 --- a/src/kc_angular/src/app/models/knowledge.source.model.ts +++ b/src/kc_angular/src/app/models/knowledge.source.model.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { AuthorModel } from '../../../../kc_shared/models/author.model'; -import { FileSourceModel } from '../../../../kc_shared/models/file.source.model'; -import { UUID } from '../../../../kc_shared/models/uuid.model'; -import { ImportMethod } from '../../../../kc_shared/models/knowledge.source.model'; -import { WebSourceModel } from '../../../../kc_shared/models/web.source.model'; +import { AuthorModel } from '@shared/models/author.model'; +import { FileSourceModel } from '@shared/models/file.source.model'; +import { UUID } from '@shared/models/uuid.model'; +import { ImportMethod } from '@shared/models/knowledge.source.model'; +import { WebSourceModel } from '@shared/models/web.source.model'; export type IngestType = | 'file' @@ -62,6 +62,13 @@ export type KnowledgeSourceEvent = { iconText?: string; }; +export type SourceNote = { + title: string; + content: string; + createdAt: Date; + updatedAt: Date; +}; + export class KnowledgeSource { associatedProject: UUID; authors: AuthorModel[]; @@ -77,6 +84,8 @@ export class KnowledgeSource { ingestType: IngestType; rawText?: string; flagged?: boolean; + meta: { id?: number; key: string; value: string }[] = []; + notes: SourceNote[] = []; title: string; topics?: string[]; accessLink: URL | string; @@ -101,6 +110,7 @@ export class KnowledgeSource { this.dateAccessed = []; this.accessLink = reference.link; this.flagged = false; + this.notes = []; this.topics = []; // TODO: this should be replaced with the new EventModel this.events = [ @@ -112,22 +122,4 @@ export class KnowledgeSource { } } -export class KnowledgeSourceNote { - text: string; - dateCreated: string; - dateModified: string; - dateAccessed: string; - - constructor(text?: string) { - this.text = text ? text : ''; - this.dateCreated = Date(); - this.dateAccessed = Date(); - this.dateModified = Date(); - } - - static blank() { - return new KnowledgeSourceNote('test'); - } -} - export class KnowledgeSourceMarkup {} diff --git a/src/kc_angular/src/app/services/chat-services/chat.factory.service.ts b/src/kc_angular/src/app/services/chat-services/chat.factory.service.ts new file mode 100644 index 00000000..ed90e870 --- /dev/null +++ b/src/kc_angular/src/app/services/chat-services/chat.factory.service.ts @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Injectable } from '@angular/core'; +import { AgentType, ChatMessage } from '@app/models/chat.model'; +import { UuidService } from '../ipc-services/uuid.service'; +import { UUID } from '@shared/models/uuid.model'; +import { KnowledgeSource } from '@app/models/knowledge.source.model'; +import { KcProject } from '@app/models/project.model'; + +@Injectable({ + providedIn: 'root', +}) +export class ChatFactoryService { + constructor(private uuid: UuidService) {} + + message(sender: AgentType, recipient: AgentType, text: string): ChatMessage { + const id: UUID = this.uuid.generate(1)[0]; + + return { + id: id.value, + recipient: recipient, + sender: sender, + text: text, + timestamp: new Date(), + rating: 'none', + ephemeral: sender === AgentType.System, + }; + } + + /** + * Create a message either from or to a Source + * @param text The text of the message + * @param source The Source + * @param sender The sender of the message (User or Source) + */ + sourceMessage( + text: string, + source: KnowledgeSource, + sender: AgentType + ): ChatMessage { + const id: UUID = this.uuid.generate(1)[0]; + + return { + text: text, + id: id.value, + sender: sender, + recipient: + sender === AgentType.Source ? AgentType.User : AgentType.Source, + source: source, + timestamp: new Date(), + rating: 'none', + }; + } + + projectMessage( + text: string, + project: KcProject, + sender: AgentType + ): ChatMessage { + const id: UUID = this.uuid.generate(1)[0]; + + return { + text: text, + id: id.value, + sender: sender, + recipient: + sender === AgentType.Project ? AgentType.User : AgentType.Project, + project: project, + timestamp: new Date(), + rating: 'none', + }; + } + + topicMessage( + text: string, + source?: KnowledgeSource, + project?: KcProject | null + ): ChatMessage { + const id: UUID = this.uuid.generate(1)[0]; + + return { + text: text, + id: id.value, + sender: AgentType.Topic, + recipient: AgentType.User, + source: source, + project: project ? project : undefined, + timestamp: new Date(), + rating: 'none', + }; + } + + loadSourceMessages(id: UUID, source: KnowledgeSource) { + const messages: ChatMessage[] = this.loadMessages(id); + + for (const msg of messages) { + if ( + msg.sender === AgentType.Source || + msg.recipient === AgentType.Source || + msg.sender === AgentType.Topic + ) { + msg.source = source; + } + } + + return messages; + } + + /** + * Load messages for a given project + * @param project The project to load messages for + */ + loadProjectMessages(project: KcProject) { + let messages: ChatMessage[] = this.loadMessages(project.id); + + for (const msg of messages) { + // If the message is to/from a Project or Topic, set the project + if ( + msg.sender === AgentType.Project || + msg.recipient === AgentType.Project || + msg.sender === AgentType.Topic + ) + msg.project = project; + } + + // Filter out any messages that are to/from a Source + messages = messages.filter((msg) => { + return ( + msg.sender !== AgentType.Source && + msg.recipient !== AgentType.Source && + msg.sender !== AgentType.Category + ); + }); + + return messages; + } + + private loadMessages(id: UUID): ChatMessage[] { + let messages: ChatMessage[] = []; + + const stored = localStorage.getItem(`chat-${id.value}`); + if (stored) { + messages = JSON.parse(stored); + + for (const msg of messages) { + msg.timestamp = new Date(msg.timestamp); + } + } + + // Remove any duplicates + messages = messages.filter((msg, index, self) => { + return ( + index === + self.findIndex((m) => { + return m.id === msg.id; + }) + ); + }); + + return messages; + } + + saveChat(newHistory: ChatMessage[], id: UUID) { + localStorage.setItem(`chat-${id.value}`, JSON.stringify(newHistory)); + } +} diff --git a/src/kc_angular/src/app/services/chat-services/chat.service.ts b/src/kc_angular/src/app/services/chat-services/chat.service.ts index 5ed57bf1..383416f5 100755 --- a/src/kc_angular/src/app/services/chat-services/chat.service.ts +++ b/src/kc_angular/src/app/services/chat-services/chat.service.ts @@ -14,34 +14,47 @@ * limitations under the License. */ -import { AgentType, ChatMessage } from '@app/models/chat.model'; +import { AgentType, ChatMessage, MessageRating } from '@app/models/chat.model'; import { HttpClient } from '@angular/common/http'; -import { Inject, Injectable, LOCALE_ID } from '@angular/core'; +import { Injectable } from '@angular/core'; import { KcProject } from '@app/models/project.model'; import { KnowledgeSource } from '@app/models/knowledge.source.model'; import { NotificationsService } from '@services/user-services/notifications.service'; -import { BehaviorSubject, Observable, tap, timeout } from 'rxjs'; +import { + BehaviorSubject, + catchError, + forkJoin, + mergeMap, + Observable, + tap, + timeout, +} from 'rxjs'; import { UUID } from '@shared/models/uuid.model'; -import { UuidService } from '@services/ipc-services/uuid.service'; -import { ElectronIpcService } from '@services/ipc-services/electron-ipc.service'; import { DialogService } from 'primeng/dynamicdialog'; -import { map } from 'rxjs/operators'; +import { debounceTime, distinctUntilChanged, map, take } from 'rxjs/operators'; import { SettingsService } from '@services/ipc-services/settings.service'; import { ChatSettingsModel } from '@shared/models/settings.model'; import { ChatApiComponent } from '@components/chat-components/api.component'; import { ChatCompletionMessage, - CreateChatCompletionRequestMessage, + ChatCompletionMessageParam, } from 'openai/resources/chat/completions'; +import { ChatFactoryService } from '@services/chat-services/chat.factory.service'; + +export interface ChatTarget { + source?: KnowledgeSource; + project?: KcProject | null; +} @Injectable({ providedIn: 'root', }) export class ChatService { - // TODO in v0.8.1 make this more robust (dynamic port selection when necessary) + // TODO: Make this a configurable setting when backend becomes available as standalone private backendUrl = 'http://localhost:21003'; private canConnect = new BehaviorSubject(false); + canConnect$ = this.canConnect.asObservable(); private _settings: ChatSettingsModel = new ChatSettingsModel(); @@ -54,18 +67,28 @@ export class ChatService { tokenLimit$ = this.tokenLimit.asObservable(); + private _messages$ = new BehaviorSubject([]); + + messages$ = this._messages$.asObservable(); + + private _loading$ = new BehaviorSubject(false); + + loading$ = this._loading$.asObservable(); + + private _target$ = new BehaviorSubject({}); + + target$ = this._target$.asObservable(); + constructor( private dialog: DialogService, + private factory: ChatFactoryService, private http: HttpClient, - @Inject(LOCALE_ID) private locale: string, private notify: NotificationsService, - private ipc: ElectronIpcService, - private settings: SettingsService, - private uuid: UuidService + private settings: SettingsService ) { this.checkConnection(); - settings.all + this.settings.all .pipe(map((settings) => settings?.app?.chat ?? new ChatSettingsModel())) .subscribe((settings) => { this._settings = settings; @@ -73,21 +96,264 @@ export class ChatService { settings.model.token_limit - settings.model.max_tokens ); }); + + this.target$ + .pipe( + debounceTime(100), + distinctUntilChanged((a, b) => { + if (a.source && b.source) { + return a.source.id.value === b.source.id.value; + } else if (a.project && b.project) { + return a.project.id.value === b.project.id.value; + } else { + return false; + } + }) + ) + .subscribe((target) => { + this._loading$.next(true); + let history: ChatMessage[] = []; + + if (target.source) { + history = this.factory.loadSourceMessages( + target.source.id, + target.source + ); + } else if (target.project) { + history = this.factory.loadProjectMessages(target.project); + + for (const source of target.project.knowledgeSource) { + const sourceHistory = this.factory.loadSourceMessages( + source.id, + source + ); + + for (const msg of sourceHistory) { + if ( + msg.sender !== AgentType.Quiz && + msg.sender !== AgentType.Category + ) { + history.push(msg); + } + } + } + + // Sort the history by timestamp + history.sort((a: ChatMessage, b: ChatMessage) => { + return a.timestamp > b.timestamp ? 1 : -1; + }); + } + + // Remove all duplicates + history = history.filter((msg, index, self) => { + return index === self.findIndex((m) => m.id === msg.id); + }); + + this._messages$.next(history); + this._loading$.next(false); + }); + + this.messages$ + .pipe( + distinctUntilChanged((a, b) => { + // Determine if the messages are the same + if (a.length !== b.length) { + return false; + } + for (let i = 0; i < a.length; i++) { + if (a[i].id !== b[i].id) { + return false; + } else if (a[i].rating !== b[i].rating) { + return false; + } + } + return true; + }) + ) + .subscribe((messages) => { + // Save the chat history + if (this._target$.value.source) { + this.saveChat(messages, this._target$.value.source.id); + } else if (this._target$.value.project) { + this.saveChat(messages, this._target$.value.project.id); + } + }); } get serverUrl() { return this.backendUrl; } - checkConnection() { - this.http.get(this.backendUrl + '/api/key').subscribe((result: any) => { - if (result) { - this.canConnect.next(result.apiKeySet); + canChat() { + return this.canConnect.value; + } + + addMessage(message: ChatMessage) { + this.messages$.pipe(take(1)).subscribe((messages) => { + this._messages$.next([...messages, message]); + }); + } + + deleteMessage(message: ChatMessage) { + // Remove the message from the chat history using the message ID + this.messages$.pipe(take(1)).subscribe((messages) => { + this._messages$.next(messages.filter((msg) => msg.id !== message.id)); + }); + } + + persistChatHistory() { + this.messages$.pipe(take(1)).subscribe((messages) => { + this.saveChat(messages); + }); + } + + displayText(text: string, sender: AgentType = AgentType.Assistant) { + const message = this.factory.message(sender, AgentType.User, text); + this.addMessage(message); + } + + post(endpoint: string, body: any): Observable { + this._loading$.next(true); + return this.http.post(this.backendUrl + endpoint, body).pipe( + take(1), + catchError((err) => { + console.error('Error: ', err); + this._loading$.next(false); + return []; + }), + map((response: any) => { + return response.choices[0].message.content; + }), + tap(() => { + this._loading$.next(false); + }) + ); + } + + get(endpoint: string, body?: any): Observable { + this._loading$.next(true); + return this.http.get(this.backendUrl + endpoint, body).pipe( + map((response: any) => { + return response.choices[0].message.content; + }), + tap(() => { + this._loading$.next(false); + }) + ); + } + + /** + * Submit a message to the chat service. This will send the message to the backend for inference. + * @param text The text of the message + */ + submit(text: string) { + const observable = forkJoin([ + this.messages$.pipe(take(1)), + this.target$.pipe(take(1)), + ]).pipe( + map((value) => { + const messages = value[0]; + const target = value[1]; + let message: ChatMessage; + + // Add the user message to the chat history + // TODO: Change this to /projects once the backend is updated + let endpoint = '/chat'; + if (target.source) { + message = this.factory.sourceMessage( + text, + target.source, + AgentType.User + ); + endpoint = '/sources'; + } else if (target.project) { + message = this.factory.projectMessage( + text, + target.project, + AgentType.User + ); + } else { + throw new Error('Target not found'); + } + this.addMessage(message); + messages.push(message); + return { messages, target, endpoint }; + }), + mergeMap(({ messages, target, endpoint }) => { + // Send the messages to the backend for inference + return this.post(endpoint, { + messages: this.convertToOpenAI(messages), + source: target.source, + project: target.project, + }).pipe( + take(1), + map((response) => { + return { response, messages, target }; + }) + ); + }) + ); + + observable.subscribe(({ response, target }) => { + // Add the response to the chat history + let message: ChatMessage; + if (target.source) { + message = this.factory.sourceMessage( + response, + target.source, + AgentType.Assistant + ); + } else if (target.project) { + message = this.factory.projectMessage( + response, + target.project, + AgentType.Assistant + ); + } else { + throw new Error('Target not found'); } + this.addMessage(message); }); } - getApiKeyDialog() { + /** + * Set the target for the chat service. This will load the chat history for the target. + * @param target a ChatTarget object containing either a source or a project + * + * Side effects: + * - Updates the target$ Observable. + */ + setTarget(target: ChatTarget) { + this._target$.next(target); + } + + /** + * Delete the OpenAI API key from the backend. + * + * Side effects: + * - Updates the canConnect$ Observable. + */ + deleteApiKey() { + return this.http.delete(this.backendUrl + '/api/key').pipe( + tap((result: any) => { + if (result) { + this.canConnect.next(false); + } + }) + ); + } + + /** + * Open a dialog to get the OpenAI API key from the user. + * + * Side effects: + * - Shows a dialog to the user to get the API key. + * - Shows a success or error notification to the user. + * - API key is persisted by the backend. + * + * @returns {Observable} An observable that emits true if the API key was saved successfully, or false if it was not. + */ + getApiKeyDialog(): Observable { const dialogRef = this.dialog.open(ChatApiComponent, { header: 'OpenAI API Key Required', }); @@ -112,16 +378,14 @@ export class ChatService { ); } - deleteApiKey() { - return this.http.delete(this.backendUrl + '/api/key').pipe( - tap((result: any) => { - if (result) { - this.canConnect.next(false); - } - }) - ); - } - + /** + * Set the OpenAI API key for the chat service, which is used to authenticate with the backend. + * + * Side effects: + * - Updates the canConnect$ Observable. + * + * @param apiKey + */ setApiKey(apiKey: string) { return this.http .post(this.backendUrl + '/api/key', { apiKey: apiKey }) @@ -136,70 +400,24 @@ export class ChatService { ); } - canChat() { - return this.canConnect.value; - } - - /* Create a new chat message */ - createMessage( - sender: AgentType, - recipient: AgentType, - message: string, - project?: KcProject, - source?: KnowledgeSource - ): ChatMessage { - const id: UUID = this.uuid.generate(1)[0]; - return { - id: id.value, - sender: sender, - recipient: recipient, - text: message, - timestamp: new Date(), - project: project, - source: source, - rating: 'none', - }; - } - - /* Convenience method for creating a new chat message based on a previous message */ - elaborate( - history: ChatMessage[], - method: 'ELI5' | 'TLDR' | 'Continue' - ): Observable { - const messages = this.convertToOpenAI(history); - messages.push({ - role: 'user', - content: method, - }); - - return this.send(messages); - } - - intro( - messages: CreateChatCompletionRequestMessage[], - source: KnowledgeSource - ): Observable { - const body = { - messages: messages, - source: source, - }; - const url = this.backendUrl + '/sources/intro'; - return this.http.post(url, body).pipe( - map((response: any) => { - return response.choices[0].message; - }) - ); - } - - convertToOpenAI( - messages: ChatMessage[] - ): CreateChatCompletionRequestMessage[] { + /** + * Convert the chat messages to completion request messages. If a message has a 'thumbs-down' rating, omit it from the request + * @param messages + */ + convertToOpenAI(messages: ChatMessage[]): ChatCompletionMessageParam[] { // Convert the chat messages to completion request messages. If a message has a 'thumbs-down' rating, omit it from the request - const completions: CreateChatCompletionRequestMessage[] = []; + const completions: ChatCompletionMessageParam[] = []; + messages.forEach((message) => { if (message.rating !== 'thumbs-down') { + const role = + message.sender === AgentType.User + ? 'user' + : message.sender === AgentType.System + ? 'system' + : 'assistant'; completions.push({ - role: message.sender === AgentType.User ? 'user' : 'assistant', + role: role, content: message.text, name: message.sender, }); @@ -208,66 +426,113 @@ export class ChatService { // TODO: Need to implement thumbs-up prioritization, but that will require a change to the // request body (i.e. we will need to send the rating info to the backend). }); + return completions; } - send( - messages: CreateChatCompletionRequestMessage[] + /** + * Submit message history to the server for inference. Assumes that the messages are already + * formatted as completion request messages (OpenAI API format). + * @param messages + * @param endpoint + */ + sendChat( + messages: ChatCompletionMessageParam[], + endpoint = '/chat' ): Observable { const body = { messages: messages }; - this.notify.debug('Chat Service', 'Sending OpenAI API Request', body); // Remove any empty messages body.messages = body.messages.filter( - (message) => message.content?.trim() !== '' + (message) => + typeof message.content === 'string' && message.content?.trim() !== '' ); - return this.http.post(this.backendUrl + '/chat', body).pipe( + this._loading$.next(true); + return this.http.post(this.backendUrl + endpoint, body).pipe( timeout(60000), + tap(() => { + this._loading$.next(false); + }), map((response: any) => { return response.choices[0].message; }) ); } - /* Load chat history from local storage */ - loadChat( - id: UUID, - project?: KcProject, - source?: KnowledgeSource - ): ChatMessage[] { - let history: ChatMessage[] = []; - - const chatHistory = localStorage.getItem(`chat-${id.value}`); - if (chatHistory) { - history = JSON.parse(chatHistory); - for (const msg of history) { - msg.timestamp = new Date(msg.timestamp); - - if (project) { - msg.project = project; - } - if (source) { - msg.source = source; - } - } + /** + * Process the input message to count the number of tokens it contains. + * + * Side effects: + * - Updates the tokenCount$ Observable. + * @param message The string to count tokens for + */ + processInput(message: string) { + if (!message.trim()) { + this.tokenCount.next(0); + return; } + this.http + .post(this.backendUrl + '/chat/tokens', { text: message }) + .pipe(map((response: any) => response.tokens)) + .subscribe((count: number) => { + this.tokenCount.next(count); + }); + } - return history; + /** + * Check for a valid connection to the server, which requires a valid API key + * and a successful response from the backend. + */ + private checkConnection() { + this.http.get(this.backendUrl + '/api/key').subscribe((result: any) => { + if (result) { + this.canConnect.next(result.apiKeySet); + } + }); } - /* Save chat history to local storage */ - saveChat(history: ChatMessage[], id?: UUID): ChatMessage[] { - /* If no ID is provided, save chat history for all sources and projects individually */ + /** + * Save chat history to local storage + * + * If an ID is provided, save entire chat history using the ID as the key. + * Otherwise, save individual history for all sources and projects found in the history. + * Messages that are marked as "ephemeral" are not persisted. + * + * @param history The chat history to save + * @param id The ID of the source or project to save the history for + * @returns {ChatMessage[]} The chat history that was saved + */ + private saveChat(history: ChatMessage[], id?: UUID) { this.notify.debug( 'Chat Service', 'Saving Chat', 'Saving chat history to local storage' ); - if (!id) { - // Get a list of all unique sources in the history + + /* If an ID is provided, save entire chat history using the ID as the key */ + if (id) { + const newHistory: ChatMessage[] = []; + for (const msg of history) { + if (!msg.ephemeral) { + newHistory.push({ + id: msg.id, + sender: msg.sender, + recipient: msg.recipient, + text: msg.text, + timestamp: msg.timestamp, + project: undefined, + source: undefined, + rating: msg.rating, + }); + } + } + this.factory.saveChat(newHistory, id); + } else { + // Get a list of all unique sources and projects in the history const sources: KnowledgeSource[] = []; const projects: KcProject[] = []; + for (const msg of history) { if ( msg.source && @@ -284,130 +549,72 @@ export class ChatService { } } - // Save all source history + // Save history for all Sources for (const source of sources) { // Get history for this source const sourceHistory = history.filter( - (m) => m.source?.id.value === source.id.value - ); - - // Save history for this source - localStorage.setItem( - `chat-${source.id.value}`, - JSON.stringify(sourceHistory) + (m) => m.source?.id.value === source.id.value && !m.ephemeral ); + this.factory.saveChat(sourceHistory, source.id); } - // Save all project history + // Save history for all Projects for (const project of projects) { const projectHistory = history.filter( - (m) => m.project?.id.value === project.id.value - ); - localStorage.setItem( - `chat-${project.id.value}`, - JSON.stringify(projectHistory) + (m) => m.project?.id.value === project.id.value && !m.ephemeral ); + this.factory.saveChat(projectHistory, project.id); } - - return history; - } - - /* If an ID is provided, save entire chat history using the ID as the key */ - const newHistory: ChatMessage[] = []; - for (const msg of history) { - newHistory.push({ - id: msg.id, - sender: msg.sender, - recipient: msg.recipient, - text: msg.text, - timestamp: msg.timestamp, - project: undefined, - source: undefined, - }); } - localStorage.setItem(`chat-${id.value}`, JSON.stringify(newHistory)); - - return newHistory; } /** - * Returns a copy of the history, up to and including the target message - * @param message The target message - * @param history The entire history - * @param inclusive Whether or not to include the target message in the context - * @param rules if true, the history will be filtered depending on the source/project - * @returns {ChatMessage[]} The history up to and including the target message. - * If the target message is not in the history, then **an empty array is returned.** + * Rate a message using a thumbs-up. thumbs-down or none rating. + * Sets the message rating and updates the chat history. + * @param message + * @param rating */ - getHistory( - message: ChatMessage, - history: ChatMessage[], - inclusive = true, - rules = false - ): ChatMessage[] { - // Sort history by timestamp to ensure that the index is correct - history.sort((a: ChatMessage, b: ChatMessage) => { - return a.timestamp > b.timestamp ? 1 : -1; - }); - - // Get the index of the target message - const index = history.indexOf(message); - - // If the message is not in the history, return an empty array - if (index < 0) { - return []; - } - - // Get the history up to and including the target message (or not, depending on the inclusive flag) - let context: ChatMessage[] = history.slice( - 0, - inclusive ? index + 1 : index - ); - - // Apply Source rules to the context - if (rules && message.source) { - context = this.getSourceHistory(message, context, message.source); - } - - // Apply Project rules to the context - if (rules && message.project) { - context = this.getProjectHistory(message, context, message.project); - } - - return context; - } - - private getProjectHistory( - message: ChatMessage, - history: ChatMessage[], - project: KcProject - ): ChatMessage[] { - return history.filter((msg: ChatMessage) => { - return msg.project && msg.project.id === project.id; - }); - } - - private getSourceHistory( - message: ChatMessage, - history: ChatMessage[], - source: KnowledgeSource - ): ChatMessage[] { - return history.filter((msg: ChatMessage) => { - return msg.source && msg.source.id === source.id; + rateMessage(message: ChatMessage, rating: MessageRating) { + this.messages$.pipe(take(1)).subscribe((messages) => { + // Find the message using ID + const index = messages.findIndex((msg) => msg.id === message.id); + if (index < 0) { + this.notify.error( + 'Chat Service', + 'Message Not Found', + 'Knowledge encountered an error while trying to rate a message that was not found.', + 'toast' + ); + return; + } else { + // Update the message rating + messages[index].rating = rating; + this._messages$.next(messages); + + if (this._target$.value.source) { + this.saveChat(messages, this._target$.value.source.id); + } else if (this._target$.value.project) { + this.saveChat(messages, this._target$.value.project.id); + } + } }); } - countTokens(message: string) { - if (!message) { - this.tokenCount.next(0); - return; + addTopic(topic: string) { + if (this._target$.value.source) { + if (this._target$.value.source.topics) { + this._target$.value.source.topics.push(topic); + } else { + this._target$.value.source.topics = [topic]; + } + this.setTarget(this._target$.value); + } else if (this._target$.value.project) { + if (this._target$.value.project.topics) { + this._target$.value.project.topics.push(topic); + } else { + this._target$.value.project.topics = [topic]; + } + this.setTarget(this._target$.value); } - - this.http - .post(this.backendUrl + '/chat/tokens', { text: message }) - .pipe(map((response: any) => response.tokens)) - .subscribe((count: number) => { - this.tokenCount.next(count); - }); } } diff --git a/src/kc_angular/src/app/services/chat-services/commands.service.ts b/src/kc_angular/src/app/services/chat-services/commands.service.ts new file mode 100644 index 00000000..3bd11a1e --- /dev/null +++ b/src/kc_angular/src/app/services/chat-services/commands.service.ts @@ -0,0 +1,554 @@ +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Injectable } from '@angular/core'; +import { BehaviorSubject, forkJoin, mergeMap, tap } from 'rxjs'; +import { ChatService } from '@services/chat-services/chat.service'; +import { SettingsService } from '@services/ipc-services/settings.service'; +import { ProjectTreeFactoryService } from '@services/factory-services/project-tree-factory.service'; +import { filter, map, take } from 'rxjs/operators'; +import { ChatFactoryService } from '@services/chat-services/chat.factory.service'; +import { AgentType, ChatMessage } from '@app/models/chat.model'; + +export interface CommandArgs { + label: string; + optional: boolean; + value?: string; +} + +export interface ChatCommand { + command: string; + targets: 'Source' | 'Project' | 'Both'; + description: string; + args?: CommandArgs[]; + argParse?: (args: string) => CommandArgs[]; + execute: (args: CommandArgs[]) => void; +} + +@Injectable({ + providedIn: 'root', +}) +export class ChatCommandService { + private commands: ChatCommand[] = [ + { + command: '/categorize', + description: 'Get a list of Projects that this Source would fit into', + targets: 'Source', + execute: () => this.categorize(), + }, + // { + // command: '/note', + // description: 'Add a note to this Source', + // targets: 'Source', + // args: [ + // { + // label: '', + // optional: false, + // }, + // ], + // argParse(args: string) { + // const tokens = args.trim().split(' '); + // const note = tokens.slice(1).join(' ').trim(); + // console.log('note args: ', note); + // return [ + // { + // label: '', + // optional: false, + // value: note === '' ? undefined : note, + // }, + // ]; + // }, + // execute: (args: CommandArgs[]) => { + // console.log('Adding note to Source: ', args); + // }, + // }, + // { + // command: '/organize', + // description: + // 'Get suggestions on how to organize the Sources in this Project based on Topics, keywords, and other attributes', + // targets: 'Project', + // args: [], + // execute: () => {}, + // }, + // { + // command: '/review', + // description: + // 'Get a high-level overview of your Chat history with this Source or Project', + // targets: 'Both', + // args: [], + // execute: (args: CommandArgs[]) => {}, + // }, + // { + // command: '/search', + // description: + // 'Search a Source or Project using Retrieval-augmented Generation (RAG)', + // targets: 'Both', + // argParse: (args: string) => { + // const tokens = args.trim().split(' '); + // const query = tokens.slice(1).join(' ').trim(); + // return [ + // { + // label: '', + // optional: false, + // value: query === '' ? undefined : query, + // }, + // ]; + // }, + // args: [ + // { + // label: '', + // optional: false, + // }, + // ], + // execute: (args: CommandArgs[]) => {}, + // }, + // { + // command: '/similar', + // description: 'Find similar Sources to the one you are currently viewing', + // targets: 'Source', + // args: [], + // execute: (args: CommandArgs[]) => {}, + // }, + // { + // command: '/structure', + // description: + // 'Get suggestions on how to restructure your Project to make it more organized and easier to navigate', + // targets: 'Project', + // args: [], + // execute: (args: CommandArgs[]) => {}, + // }, + { + command: '/questions', + description: + 'Get a list of frequently asked questions based on your Chat history', + targets: 'Both', + args: [], + execute: () => this.questions(), + }, + { + command: '/summarize', + description: + 'Summarize the contents of this Source based on content and Chat history', + targets: 'Source', + execute: () => this.summarize(), + }, + { + command: '/topics', + description: + 'Get a list of the most common Topics in a Source or Project', + targets: 'Source', + execute: () => this.topics(), + }, + { + command: '/tutor', + description: + 'Decide what to learn next based on your Chat history and goals', + targets: 'Source', + args: [], + execute: () => this.tutor(), + }, + { + command: '/model', + description: 'Get information about the current Chat model', + targets: 'Both', + args: [], + execute: () => this.model(), + }, + { + command: '/quiz', + description: + 'Take a quiz to test your understanding of a Source or Project', + targets: 'Both', + execute: () => this.quiz(), + }, + { + command: '/tree', + description: 'Display the current Project in tree form', + targets: 'Project', + execute: () => this.showTree(), + }, + ]; + + private _commands$ = new BehaviorSubject([]); + + commands$ = this._commands$.asObservable(); + + private _questions$ = new BehaviorSubject([]); + + questions$ = this._questions$.asObservable(); + + scrollView$ = new BehaviorSubject(false); + + target: 'Source' | 'Project' = 'Source'; + + constructor( + private chat: ChatService, + private settings: SettingsService, + private tree: ProjectTreeFactoryService, + private factory: ChatFactoryService + ) { + this.commands.sort((a, b) => a.command.localeCompare(b.command)); + chat.target$ + .pipe( + tap((target) => { + if (target.source) { + this.target = 'Source'; + } else if (target.project) { + this.target = 'Project'; + } + }) + ) + .subscribe(); + } + + isValid(command: string) { + return this.commands.some((c) => c.command === command); + } + + lookup(command: string) { + return this.commands.find((c) => c.command === command); + } + + /** + * Filter the commands list based on the input. Sets the commands$ observable to the filtered list. + * @param command text to filter commands by + */ + filter(command: string) { + if (command.trim() === '') { + this._commands$.next([]); + return; + } else { + const next = this.commands.filter( + (c) => + c.command.startsWith(command.trim().split(' ')[0].trim()) && + (c.targets === this.target || c.targets === 'Both') + ); + this._commands$.next(next); + } + } + + /* Reset the commands list to an empty array */ + reset() { + this._commands$.next([]); + } + + indexed(commandIndex: number) { + const commands = this._commands$.value; + if (commands.length === 0) { + return undefined; + } + return commands[commandIndex]; + } + + private categorize() { + forkJoin({ + messages: this.chat.messages$.pipe(take(1)), + target: this.chat.target$.pipe(take(1)), + tree: this.tree.treeNodes.pipe(take(1)), + }) + .pipe( + map(({ messages, target, tree }) => { + const treeList = []; + for (const treeNode of tree) { + const treeYaml = this.tree.generateJSON(treeNode); + treeList.push(treeYaml); + } + const treeStr = JSON.stringify(treeList); + // Map messages to OpenAI format + return { + messages: this.chat.convertToOpenAI(messages), + source: target.source, + tree: treeStr, + }; + }), + mergeMap(({ messages, source, tree }) => { + // Send messages and source as body to tutor endpoint + return this.chat.post('/sources/categorize', { + messages: messages, + source: source, + tree: tree, + }); + }) + ) + .subscribe((response) => { + this.chat.displayText(response, AgentType.Category); + }); + } + + private questions() { + this.chat.messages$ + .pipe( + take(1), + map((messages) => { + return this.chat.convertToOpenAI(messages); + }), + map((messages) => { + return messages.concat([ + { + role: 'user', + content: `Given our conversation history, what are some questions I might ask about any related concepts or topics? Please list 3 questions.`, + }, + { + role: 'system', + content: `The questions should be relevant to the Project or the Sources discussed in the conversation history.`, + }, + { + role: 'system', + content: `Take time to think about the questions that the user would ask of an expert.`, + }, + { + role: 'system', + content: `Do not respond with anything other than the questions themselves.`, + }, + { + role: 'system', + content: `Separate each question with a newline character.`, + }, + ]); + }), + tap((messages) => { + this.chat + .sendChat(messages) + .pipe(take(1)) + .subscribe((response) => { + if (response.content) { + const questions = response.content.split('\n'); + this._questions$.next(questions); + } else { + this._questions$.next([]); + } + }); + }) + ) + .subscribe(); + } + + private summarize() { + this.chat.target$ + .pipe( + take(1), + filter(({ source }) => source !== undefined), + mergeMap(({ source }) => { + return this.chat + .post('/sources/summarize', { + source: source, + }) + .pipe( + map((response) => { + return { + response: response, + source: source, + }; + }) + ); + }) + ) + .subscribe(({ response, source }) => { + if (source) { + const message = this.factory.sourceMessage( + response, + source, + AgentType.Source + ); + this.chat.addMessage(message); + } + }); + } + + private quiz() { + forkJoin({ + target: this.chat.target$.pipe(take(1)), + messages: this.chat.messages$.pipe(take(1)), + }) + .pipe( + map(({ target, messages }) => { + // Remove any previous quizzes so they don't interfere with the new quiz + const pruned = messages.filter((message) => { + return message.sender !== AgentType.Quiz; + }); + const openAI = this.chat.convertToOpenAI(pruned); + return { + target: target, + messages: openAI, + }; + }), + mergeMap(({ target, messages }) => { + if (target.source) { + return this.chat.post('/sources/quiz', { + source: target.source, + messages: messages, + }); + } else { + return this.chat.post('/projects/quiz', { + project: target.project, + messages: messages, + }); + } + }) + ) + .subscribe((response) => { + this.chat.displayText(response, AgentType.Quiz); + }); + } + + private tutor() { + // Get Messages and Source + forkJoin({ + messages: this.chat.messages$.pipe(take(1)), + target: this.chat.target$.pipe(take(1)), + }) + .pipe( + map(({ messages, target }) => { + // Map messages to OpenAI format + const openAI = this.chat.convertToOpenAI(messages); + return { + messages: openAI, + target: target, + }; + }), + mergeMap(({ messages, target }) => { + // Send messages and source as body to tutor endpoint + const endpoint = target.source ? '/sources/tutor' : '/projects/tutor'; + return this.chat + .post(endpoint, { + messages: messages, + source: target.source, + project: target.project, + }) + .pipe( + map((response) => { + return { + response: response, + source: target.source, + project: target.project, + }; + }) + ); + }) + ) + .subscribe(({ response, source, project }) => { + let message: ChatMessage; + + if (source) { + message = this.factory.sourceMessage( + response, + source, + AgentType.Source + ); + } else if (project) { + message = this.factory.projectMessage( + response, + project, + AgentType.Project + ); + } else { + message = this.factory.message( + AgentType.Tutor, + AgentType.User, + response + ); + } + + this.chat.addMessage(message); + }); + } + + private model() { + const model = this.settings.get().app.chat.model; + this.chat.displayText( + `${model.label}\n---\n` + + `Description: ${model.description}\n` + + '| Property | Value |\n' + + `| -------- | ------- |\n` + + `| Token Limit | ${model.token_limit} |\n` + + `| Input Cost (1k tokens) | $${model.input_kilo_cost} |\n` + + `| Output Cost (1k tokens) | $${model.output_kilo_cost} |\n` + + `| Max Tokens Upper Bound | ${model.max_tokens_upper_bound} |\n` + + `\n\nThe following parameters are configurable (see Chat settings):\n` + + `| Parameter | Value |\n` + + `| -------- | ------- |\n` + + `| Temperature | ${model.temperature} |\n` + + `| Top P | ${model.top_p} |\n` + + `| Max Output Tokens | ${model.max_tokens} |\n` + + `| Presence Penalty | ${model.presence_penalty} |\n` + + `| Frequency Penalty | ${model.frequency_penalty} |`, + AgentType.System + ); + } + + private topics() { + this.chat.target$ + .pipe( + take(1), + filter( + ({ source, project }) => source !== undefined || project !== undefined + ), + mergeMap(({ source, project }) => { + if (source) { + return this.chat + .post('/sources/topics', { + source: source, + }) + .pipe( + map((response) => ({ + text: response, + source: source, + project: undefined, + })) + ); + } else { + return this.chat + .post('/projects/topics', { + project: project, + }) + .pipe( + map((response) => ({ + text: response, + source: undefined, + project: project, + })) + ); + } + }) + ) + .subscribe(({ text, source, project }) => { + const message = this.factory.topicMessage(text, source, project); + this.chat.addMessage(message); + }); + } + + private showTree() { + forkJoin({ + tree: this.tree.treeNodes.pipe(take(1)), + target: this.chat.target$.pipe(take(1)), + }) + .pipe( + tap(({ tree, target }) => { + if (target.project) { + const projectTree = this.tree.findTreeNode( + target.project.id.value, + tree + ); + if (projectTree) { + const treeText = this.tree.generateTree(projectTree, 0, false); + this.chat.displayText(treeText, AgentType.Project); + } + } + }) + ) + .subscribe(); + } +} diff --git a/src/kc_angular/src/app/services/chat-services/prompts.ts b/src/kc_angular/src/app/services/chat-services/prompts.ts index 0e92fbd5..d809dfd4 100755 --- a/src/kc_angular/src/app/services/chat-services/prompts.ts +++ b/src/kc_angular/src/app/services/chat-services/prompts.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ -import { ChatMessage } from '@app/models/chat.model'; import { Injectable } from '@angular/core'; import { map } from 'rxjs/operators'; -import { KnowledgeSource } from '@app/models/knowledge.source.model'; -import { KcProject } from '@app/models/project.model'; -import { ChatService } from '@services/chat-services/chat.service'; import { ChatCompletionMessage, - CreateChatCompletionRequestMessage, + ChatCompletionMessageParam, } from 'openai/resources/chat/completions'; +import { ChatService } from './chat.service'; +import { KcProject } from '../../models/project.model'; +import { KnowledgeSource } from '../../models/knowledge.source.model'; +import { ChatMessage } from '../../models/chat.model'; @Injectable({ providedIn: 'root', @@ -34,7 +34,7 @@ export class ChatPrompts { introPrompts = ( concept: KnowledgeSource | KcProject, type: 'Source' | 'Project' - ): CreateChatCompletionRequestMessage[] => { + ): ChatCompletionMessageParam[] => { const prompts = [ `Information cutoff: December 2021, Current date: ${new Date().toDateString()}.`, `Knowledge is a tool for saving, searching, accessing, and exploring all of your favorite websites, documents and files.`, @@ -100,15 +100,13 @@ export class ChatPrompts { sourceIntro.forEach((prompt) => prompts.push(prompt)); } - const introPrompts = prompts.map((prompt) => { - const message: CreateChatCompletionRequestMessage = { + return prompts.map((prompt) => { + const message: ChatCompletionMessageParam = { role: 'system', content: prompt, }; return message; }); - - return introPrompts; }; predictNextQuestion( @@ -122,13 +120,14 @@ export class ChatPrompts { ); // Map those messages into a ChatCompletionRequest object - const completionRequests: CreateChatCompletionRequestMessage[] = - recentMessages.map((message) => { + const completionRequests: ChatCompletionMessageParam[] = recentMessages.map( + (message) => { return { role: message.sender === 'user' ? 'user' : 'assistant', content: message.text, }; - }); + } + ); // Explain how to generate the questions to the assistant completionRequests.push({ @@ -167,7 +166,7 @@ export class ChatPrompts { } // Use pipe to split the response into an array of strings (one for each question, separated by newline characters) - return this.chat.send(completionRequests).pipe( + return this.chat.sendChat(completionRequests).pipe( map((response: ChatCompletionMessage) => { if (response.content) { return response.content.split('\n'); @@ -175,25 +174,25 @@ export class ChatPrompts { return []; } }), - map((response) => { + map((response: string[]) => { // Remove any elements that don't end in a question mark return response.filter((question: string) => { return question.endsWith('?'); }); }), - map((response) => { + map((response: string[]) => { // Filter out any empty strings in the array return response.filter((question: string) => { return question !== ''; }); }), - map((response) => { + map((response: string[]) => { // Remove any numbers from the beginning of the response return response.map((question: string) => { return question.replace(/^\d+\.\s/, ''); }); }), - map((response) => { + map((response: string[]) => { // Remove any duplicate questions return response.filter( (question: string, index: number, self: string[]) => { @@ -201,13 +200,13 @@ export class ChatPrompts { } ); }), - map((response) => { + map((response: string[]) => { // Remove any characters from the beginning of the response that are not letters or numbers return response.map((question: string) => { return question.replace(/^[^a-zA-Z0-9]+/, ''); }); }), - map((response) => { + map((response: string[]) => { // Return only the first 3 questions return response.slice(0, 3); }) diff --git a/src/kc_angular/src/app/services/chat-services/source.ts b/src/kc_angular/src/app/services/chat-services/source.ts index db01d7f1..25fe6b90 100755 --- a/src/kc_angular/src/app/services/chat-services/source.ts +++ b/src/kc_angular/src/app/services/chat-services/source.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -119,60 +119,6 @@ export class SourceChat { }); } - return this.chat.send(prompts); - } - - intro(source: KnowledgeSource) { - const system = this.prompts.introPrompts(source, 'Source'); - const prompts = this.getSourcePrompts(source); - prompts.forEach((prompt) => system.push(prompt)); - system.push({ - role: 'user', - content: `Can you introduce me to "${source.title}"?`, - }); - this.notify.debug('Source Chat Service', 'Sending Intro Prompts', system); - return this.chat.intro(system, source); - } - - async getText(source: KnowledgeSource) { - // First try to extract text from the source using htmlToText - - if (source.ingestType === 'file') { - // Get text from file using Tika - } else { - // Get text from website using other libraries - } - } - - /** - * Regenerate a specific message - * @param source The source to regenerate the message for - * @param message The message to regenerate - * @param history The history of the chat session - * @returns an Observable that will emit the regenerated message - */ - regenerate( - source: KnowledgeSource, - message: ChatMessage, - history: ChatMessage[] - ) { - // Get standard source prompts - const prompts = this.getSourcePrompts(source); - - // Get the history of the chat before the message to be regenerated - const slice = this.chat.getHistory(message, history, false, true); - - // Convert to OpenAI format and add to prompts - this.chat - .convertToOpenAI(slice) - .forEach((message) => prompts.push(message)); - - // Send chat history - this.notify.debug( - 'Source Chat Service', - 'Sending Regenerate Prompts', - prompts - ); - return this.chat.send(prompts); + return this.chat.sendChat(prompts); } } diff --git a/src/kc_angular/src/app/services/factory-services/ks-context-menu.service.ts b/src/kc_angular/src/app/services/factory-services/ks-context-menu.service.ts index 82aff402..95d03b97 100755 --- a/src/kc_angular/src/app/services/factory-services/ks-context-menu.service.ts +++ b/src/kc_angular/src/app/services/factory-services/ks-context-menu.service.ts @@ -91,10 +91,6 @@ export class KsContextMenuService { label: '', separator: true, }, - { - label: 'Move', - icon: PrimeIcons.REPLY, - }, { label: 'Remove', icon: PrimeIcons.TRASH, diff --git a/src/kc_angular/src/app/services/factory-services/ks-factory.service.ts b/src/kc_angular/src/app/services/factory-services/ks-factory.service.ts index 324d230d..ab41037c 100755 --- a/src/kc_angular/src/app/services/factory-services/ks-factory.service.ts +++ b/src/kc_angular/src/app/services/factory-services/ks-factory.service.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ import { ElectronIpcService } from '@services/ipc-services/electron-ipc.service' import { ExtractorService } from '@services/ingest-services/extractor.service'; import { FaviconService } from '@services/ingest-services/favicon.service'; import { FileSourceModel } from '@shared/models/file.source.model'; -import { HttpClient } from '@angular/common/http'; +import { HttpClient, HttpHeaders } from '@angular/common/http'; import { IngestType, KnowledgeSource, @@ -67,15 +67,19 @@ export class KsFactoryService { examples(): Observable { return this.http - .get( - 'https://knowledge-app.s3.us-west-1.amazonaws.com/examples_v3.json', - { responseType: 'json' } - ) + .get('https://knowledge-app.s3.us-west-1.amazonaws.com/examples.json', { + responseType: 'json', + headers: new HttpHeaders({ + 'Cache-Control': 'no-cache', + Pragma: 'no-cache', + Expires: 'Sat, 01 Jan 2000 00:00:00 GMT', + }), + }) .pipe( map((example: any) => { let examples: ExampleSource[] = example; this.shuffleArray(examples); - examples = examples.slice(0, 5); + examples = examples.slice(0, 6); const requests: KnowledgeSourceFactoryRequest = { ingestType: 'website', links: examples.map((example) => example.accessLink), @@ -216,6 +220,8 @@ export class KsFactoryService { }, link: '', }, + meta: [], + notes: [], ingestType: 'website', associatedProject: { value: '' }, // TODO: remove these to align with new model... @@ -303,10 +309,25 @@ export class KsFactoryService { }); } + private getLink(link: URL): string { + // If the link is a PDF file from arxiv.org, set the link to `https://arxiv.org/abs/${arxivId}` instead of the PDF link + if ( + link.hostname === 'arxiv.org' && + link.pathname.split('/')[1] === 'pdf' + ) { + const arxivId = link.pathname.split('/')[2]; + if (arxivId) { + return `https://arxiv.org/abs/${arxivId}`; + } + } + + // Otherwise, return the original link + return link.href; + } + private getWebsiteMetadata(ks: KnowledgeSource): Promise { return new Promise((resolve) => { - const link = - typeof ks.accessLink === 'string' ? ks.accessLink : ks.accessLink.href; + const link = this.getLink(new URL(ks.accessLink)); this.extractor .extractWebsiteMetadata(link) .then((metadata) => { @@ -337,6 +358,15 @@ export class KsFactoryService { if (ks.reference.source.website) ks.reference.source.website.metadata = metadata; + + ks.meta = []; + if (metadata.meta) { + for (const meta of metadata.meta) { + if (meta.key && meta.value) { + ks.meta.push({ key: meta.key, value: meta.value }); + } + } + } }) .catch((reason) => { console.warn('Unable to extract website metadata because: ', reason); diff --git a/src/kc_angular/src/app/services/factory-services/project-context-menu.service.ts b/src/kc_angular/src/app/services/factory-services/project-context-menu.service.ts index 2a27aa5a..48e736f6 100755 --- a/src/kc_angular/src/app/services/factory-services/project-context-menu.service.ts +++ b/src/kc_angular/src/app/services/factory-services/project-context-menu.service.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +19,8 @@ import { KcProject, ProjectUpdateRequest } from '@app/models/project.model'; import { MenuItem, PrimeIcons } from 'primeng/api'; import { ProjectCommandService } from '@services/command-services/project-command.service'; import { ProjectService } from './project.service'; +import { ProjectTreeFactoryService } from '@services/factory-services/project-tree-factory.service'; +import { Clipboard } from '@angular/cdk/clipboard'; @Injectable({ providedIn: 'root', @@ -26,7 +28,9 @@ import { ProjectService } from './project.service'; export class ProjectContextMenuService { constructor( private projects: ProjectService, - private pCommand: ProjectCommandService + private pCommand: ProjectCommandService, + private tree: ProjectTreeFactoryService, + private clipboard: Clipboard ) {} generate(project: KcProject, projects?: KcProject[]): MenuItem[] { @@ -101,6 +105,32 @@ export class ProjectContextMenuService { this.pCommand.copyJSON(projects ? projects : [project]); }, }, + { + label: 'Copy Project Tree', + icon: PrimeIcons.COPY, + items: [ + { + label: 'YAML', + icon: 'pi pi-file', + command: () => { + const treeNode = this.tree.findTreeNode(project.id.value); + if (!treeNode) return; + const yaml = this.tree.generateYAML(treeNode, 0); + this.clipboard.copy(yaml); + }, + }, + { + label: 'JSON', + icon: 'pi pi-file', + command: () => { + const treeNode = this.tree.findTreeNode(project.id.value); + if (!treeNode) return; + const json = this.tree.generateJSON(treeNode); + this.clipboard.copy(json); + }, + }, + ], + }, { label: '', separator: true, diff --git a/src/kc_angular/src/app/services/factory-services/project-tree-factory.service.ts b/src/kc_angular/src/app/services/factory-services/project-tree-factory.service.ts index a0db6362..9d3bc5fc 100755 --- a/src/kc_angular/src/app/services/factory-services/project-tree-factory.service.ts +++ b/src/kc_angular/src/app/services/factory-services/project-tree-factory.service.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -23,6 +23,11 @@ import { TreeNode } from 'primeng/api'; import { constructTreeNodes } from '@app/workers/tree.worker'; import { delay, take, takeUntil } from 'rxjs/operators'; +interface ProjectJSON { + name: string; + subprojects: ProjectJSON[]; +} + @Injectable({ providedIn: 'root', }) @@ -127,6 +132,93 @@ export class ProjectTreeFactoryService implements OnDestroy { return undefined; } + generateYAML(root: TreeNode, level: number): string { + // Recursively generate YAML, using the following format: + // - name: Project Name + // subprojects: + // - name: Subproject Name + // subprojects: + // - ... + + let yaml = ''; + const indent = '-'.repeat(level); + yaml += `${indent}${root.label}\n`; + if (root.children && root.children.length > 0) { + for (const child of root.children) { + yaml += this.generateYAML(child, level + 1); + } + } + return yaml; + } + + generateJSON(root: TreeNode): string { + // Generate JSON in the following format: + // { + // "name": "Project Name", + // "subprojects": [ + // { + // "name": "Subproject Name", + // "subprojects": [...] + // } + // ] + // } + + if (!root.label) { + return 'Invalid Project Name'; + } + + const json: ProjectJSON = { + name: root.label, + subprojects: [], + }; + if (root.children && root.children.length > 0) { + for (const child of root.children) { + json.subprojects.push(JSON.parse(this.generateJSON(child))); + } + } + + return JSON.stringify(json); + } + + /** + * Generate a tree in the same format as the `tree` command in Unix. + * e.g.: + * Project Name + * ├── Subproject Name + * │ ├── Subsubproject Name + * │ └── Subsubproject Name + * └── Subproject Name + */ + generateTree( + root: TreeNode, + level = 0, + isLastSubproject = false, + prefix = '' + ) { + let tree = ''; + if (level > 0) { + tree += prefix + (isLastSubproject ? '└── ' : '├── ') + root.label + '\n'; + } else { + tree += `${root.label}` + '\n'; + } + + if (root.children && root.children.length > 0) { + const newPrefix = + prefix + (level === 0 ? ' ' : isLastSubproject ? ' ' : '│ '); + root.children.forEach((child, index) => { + if (root.children) { + const isLastChild = index === root.children.length - 1; + tree += this.generateTree(child, level + 1, isLastChild, newPrefix); + } + }); + } + + if (level === 0) { + return '\n' + tree + '\n'; + } + return tree; + } + private update( projectNodes: ProjectTreeNode[], collapsed: boolean, diff --git a/src/kc_angular/src/app/services/factory-services/project.service.ts b/src/kc_angular/src/app/services/factory-services/project.service.ts index 0ddf7afa..079dd00f 100755 --- a/src/kc_angular/src/app/services/factory-services/project.service.ts +++ b/src/kc_angular/src/app/services/factory-services/project.service.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023 Rob Royce + * Copyright (c) 2022-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -231,16 +231,15 @@ export class ProjectService { } this.storageService.kcCurrentProject = newProject.id.value; this.refreshTree(); + return newProject; } /** - * - * @param updates: ProjectUpdateRequest list that contains one or more project updates - * * To update a project that was already modified, submit an update with only the ID. * * Otherwise, include any of the valid ProjectUpdateRequest fields. * e.g. create an update with ID (required) and an array removeKnowledgeSource[], etc. + * @param updates An array of ProjectUpdateRequest objects */ async updateProjects(updates: ProjectUpdateRequest[]) { for (const update of updates) { @@ -273,9 +272,6 @@ export class ProjectService { target.name = update.name; } - // If the update changes the expanded status, update it. - target.expanded = update.expanded ?? target.expanded; - // Handle parentId update if (update.parentId) { const next = this.projectSource.find( @@ -424,6 +420,8 @@ export class ProjectService { id ); } + + this.scrollToActive(1000); } getCurrentProjectId(): UUID | null { @@ -441,30 +439,6 @@ export class ProjectService { return undefined; } - getSubTree(id: UUID | string): ProjectIdentifiers[] { - if (typeof id !== 'string') { - id = id.value; - } - - const project = this.projectSource.find((k) => k.id.value === id); - let ret: ProjectIdentifiers[] = []; - if (!project) { - console.error( - "Attempting to find project that doesn't exist with ID: ", - id - ); - return ret; - } - - ret.push({ id: project.id.value, title: project.name }); - if (project.subprojects && project.subprojects.length > 0) { - for (const subProjectId of project.subprojects) { - ret = ret.concat(this.getSubTree(subProjectId)); - } - } - return ret; - } - getAncestors(id: string): ProjectIdentifiers[] { const project = this.projectSource.find((k) => k.id.value === id); let ret: ProjectIdentifiers[] = []; @@ -483,11 +457,15 @@ export class ProjectService { return ret; } - async setAllExpanded(expanded: boolean) { - for (const project of this.projectSource) { - project.expanded = expanded; - } - this.storageService.saveProjectList(this.projectSource); + scrollToActive(timeout = 0) { + setTimeout(() => { + const classElement = document.getElementsByClassName( + 'p-treenode-content p-treenode-selectable p-highlight' + ); + if (classElement.length > 0) { + classElement[0].scrollIntoView({ behavior: 'smooth' }); + } + }, timeout); } private setDefaultProject() { diff --git a/src/kc_angular/src/app/services/ingest-services/extractor.service.ts b/src/kc_angular/src/app/services/ingest-services/extractor.service.ts index ff3affab..7790065d 100755 --- a/src/kc_angular/src/app/services/ingest-services/extractor.service.ts +++ b/src/kc_angular/src/app/services/ingest-services/extractor.service.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -216,6 +216,7 @@ export class ExtractorService { )) /* Twitter (but not app) */ || n.startsWith('description') /* Description */ || n.startsWith('article:') /* Articles */ || + n.startsWith('citation') /* Citation */ || n.startsWith('keywords')) /* Keywords */ ); diff --git a/src/kc_angular/src/app/services/ingest-services/favicon.service.ts b/src/kc_angular/src/app/services/ingest-services/favicon.service.ts index 3af98979..6fb22529 100755 --- a/src/kc_angular/src/app/services/ingest-services/favicon.service.ts +++ b/src/kc_angular/src/app/services/ingest-services/favicon.service.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -89,8 +89,8 @@ export class FaviconService { promises.push(fetch(getUrl).then((response) => response.blob())); } - forkJoin(promises).subscribe( - (blobs) => { + forkJoin(promises).subscribe({ + next: (blobs) => { if (!blobs || blobs.length !== promises.length) { this.notifications.warn( 'Favicon Extractor', @@ -113,14 +113,14 @@ export class FaviconService { } resolve(icons); }, - (error) => { + error: (error) => { this.notifications.error( 'Favicon Extractor', 'Failed to get Favicon', error ); - } - ); + }, + }); }); } diff --git a/src/kc_angular/src/app/services/ingest-services/ingest.service.ts b/src/kc_angular/src/app/services/ingest-services/ingest.service.ts index fe854f01..6f3b28c8 100755 --- a/src/kc_angular/src/app/services/ingest-services/ingest.service.ts +++ b/src/kc_angular/src/app/services/ingest-services/ingest.service.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ * limitations under the License. */ import { AutoscanService } from './autoscan.service'; -import { BehaviorSubject } from 'rxjs'; +import { BehaviorSubject, skip } from 'rxjs'; import { DomSanitizer } from '@angular/platform-browser'; import { ElectronIpcService } from '../ipc-services/electron-ipc.service'; import { ExtensionService } from './extension.service'; @@ -78,14 +78,15 @@ export class IngestService implements OnDestroy { ksQueue.find( (k) => k.id.value === ks.id.value || - k.accessLink == ks.accessLink || + k.accessLink.toString() == ks.accessLink.toString() || k.title === ks.title ) ) { this.notify.warn( 'Ingest Service', 'Ignoring Duplicate', - `Source: ${ks.title}` + `Source: ${ks.title}`, + 'toast' ); continue; } @@ -93,7 +94,27 @@ export class IngestService implements OnDestroy { ksNext.push(ks); } + if (ksNext.length <= 0) { + return; + } + ksQueue = ksQueue.concat(ksNext); + + ksQueue.forEach((ks) => { + ks.dateCreated = new Date(ks.dateCreated); + }); + + // Sort by date created + ksQueue.sort((a, b) => { + if (a.dateCreated < b.dateCreated) { + return -1; + } + if (a.dateCreated > b.dateCreated) { + return 1; + } + return 0; + }); + this._queue.next(ksQueue); if (ksList.length <= 0) { @@ -103,7 +124,7 @@ export class IngestService implements OnDestroy { this.notify.success( 'IngestService', 'Source Imported', - `Imported ${ksList.length} Source${ksList.length > 1 ? 's' : ''}.` + `Imported ${ksNext.length} Source${ksNext.length > 1 ? 's' : ''}.` ); } @@ -225,7 +246,7 @@ export class IngestService implements OnDestroy { * @private */ private extensionSubscribe() { - this.extension.links.subscribe((webSource) => { + this.extension.links.pipe(skip(1)).subscribe((webSource) => { if (!webSource || !webSource.accessLink) { this.notify.warn( 'IngestService', diff --git a/src/kc_angular/src/app/services/user-services/search.service.ts b/src/kc_angular/src/app/services/user-services/search.service.ts index d869d64b..56234e3e 100755 --- a/src/kc_angular/src/app/services/user-services/search.service.ts +++ b/src/kc_angular/src/app/services/user-services/search.service.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2023 Rob Royce + * Copyright (c) 2022-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -79,7 +79,7 @@ export class SearchService { }, { name: 'description', - weight: 1, + weight: 4, }, { name: 'rawText', @@ -154,7 +154,7 @@ export class SearchService { }, { name: 'description', - weight: 1, + weight: 4, }, { name: 'accessLink', diff --git a/src/kc_angular/src/app/workers/tree.worker.ts b/src/kc_angular/src/app/workers/tree.worker.ts index 2f96270e..7ade7320 100755 --- a/src/kc_angular/src/app/workers/tree.worker.ts +++ b/src/kc_angular/src/app/workers/tree.worker.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,7 +27,6 @@ export function constructTreeNodes( for (const node of nodes) { const treeNode: TreeNode = { label: node.name, - expanded: collapsed ? false : node.expanded, leaf: node.subprojects.length === 0, selectable: true, draggable: true, diff --git a/src/kc_angular/src/assets/sounds/complete.wav b/src/kc_angular/src/assets/sounds/complete.wav new file mode 100644 index 00000000..88e558c6 Binary files /dev/null and b/src/kc_angular/src/assets/sounds/complete.wav differ diff --git a/src/kc_angular/src/assets/sounds/correct.wav b/src/kc_angular/src/assets/sounds/correct.wav new file mode 100644 index 00000000..04c9205d Binary files /dev/null and b/src/kc_angular/src/assets/sounds/correct.wav differ diff --git a/src/kc_angular/src/assets/sounds/incorrect.wav b/src/kc_angular/src/assets/sounds/incorrect.wav new file mode 100644 index 00000000..07a2a299 Binary files /dev/null and b/src/kc_angular/src/assets/sounds/incorrect.wav differ diff --git a/src/kc_angular/src/styles.scss b/src/kc_angular/src/styles.scss index 29294743..87f68f78 100755 --- a/src/kc_angular/src/styles.scss +++ b/src/kc_angular/src/styles.scss @@ -1,5 +1,5 @@ /*! - * Copyright (c) 2022-2023 Rob Royce + * Copyright (c) 2022-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ * { - font-size: 12px; + font-size: 13px; } html, body { @@ -79,6 +79,9 @@ app-ks-ingest { min-width: min(100%, 20rem); height: 100%; flex-basis: 20%; + border-right-width: 1px !important; + border-right-style: solid; + border-color: var(--surface-border) !important; } .app-splitter-divider { @@ -157,6 +160,44 @@ app-ks-ingest { margin: 0.5rem 0; } +.user-message { + background-color: var(--primary-color) !important; + color: var(--primary-color-text) !important; + border-radius: 10px; + padding: 0 10px 0 10px; + margin: 0.5rem; + text-align: right; + max-width: min(80rem, 100%); +} + +.agent-message { + color: var(--primary-color) !important; + background-color: var(--primary-color-text) !important; + border-radius: 10px; + padding: 0 10px 0 10px; + margin: 1.5rem; + text-align: left; + max-width: min(110rem, 100%); +} + +.next-question-suggestion:hover { + opacity: 1 !important; +} + +.html2canvas-container { + width: 3000px !important; + height: 3000px !important; +} + +.chat-history-active { + // If the command bar is active, make the chat history slightly opaque and transparent + opacity: 0.5; + background-color: var(--surface-ground); + + // Make it blurry + filter: blur(5px); +} + .topic-markdown { background-color: var(--surface-ground); color: var(--primary-color); @@ -397,7 +438,6 @@ source-chat, source-details, source-document, source-metadata, source-timeline, height: 100% !important; width: 100% !important; overflow: auto !important; - padding: 1.5rem !important; max-height: 100% !important; @include styleclass('surface-hover fadein'); } @@ -406,7 +446,7 @@ source-chat, source-details, source-document, source-metadata, source-timeline, @include styleclass('w-full flex flex-wrap p-2 max-h-full'); .source-header-div { - @include styleclass('col-12 lg:col-6 flex flex-column max-h-full'); + @include styleclass('flex flex-column max-h-full w-full'); .source-thumbnail { @include styleclass('w-full flex flex-column flex-auto'); @@ -438,29 +478,15 @@ source-chat, source-details, source-document, source-metadata, source-timeline, } .source-model { - @include styleclass('w-full flex col-12 flex-wrap p-2 flex-column'); + @include styleclass('w-full flex col-12 flex-wrap flex-column'); } .source-youtube-player { @include styleclass('w-full h-full'); } -.source-metadata { - @media (min-width: 0px) and (max-width: 999px) { - max-width: min(45vw, 70rem) !important; - } - - @media (min-width: 1000px) and (max-width: 1199px) { - max-width: min(50vw, 80rem) !important; - } - - @media (min-width: 1200px) and (max-width: 1599px) { - max-width: min(45vw, 70rem) !important; - } - - @media (min-width: 1600px) { - max-width: min(60vw, 1000rem) !important; - } +.source-metadata, .source-notes { + @include styleclass('w-full flex flex-wrap p-2 max-h-full'); } .title-bar { diff --git a/src/kc_angular/src/styles/kc-styles.scss b/src/kc_angular/src/styles/kc-styles.scss index 8651cc65..334f7387 100755 --- a/src/kc_angular/src/styles/kc-styles.scss +++ b/src/kc_angular/src/styles/kc-styles.scss @@ -1,5 +1,5 @@ /*! - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,6 +26,17 @@ cursor: move; } +.chat-command-banner { + border-radius: 1.5rem 1.5rem 0 0; + overflow: hidden; + + // Remove margin from children elements + > * { + margin: 0; + border-radius: 1.5rem 1.5rem 0 0; + } +} + .pulsate-fwd { animation: pulsate-fwd 3s ease-in-out infinite forwards; } @@ -111,12 +122,31 @@ @keyframes bg-pan-left { 0% { background-position: 100% 50%; - background: var(--surface-ground); + background: var(--surface-d); } 100% { background-position: 0 50%; + background: var(--primary-color-text); + } +} + +.bg-color-warp { + animation: bg-color-warp 6s ease-in-out infinite alternate both; +} + +@keyframes bg-color-warp { + 0% { background: var(--surface-a); } + 33.3333% { + background: var(--surface-b); + } + 66.666% { + background: var(--surface-c); + } + 100% { + background: var(--primary-color-text); + } } .kenburns-top { diff --git a/src/kc_electron/package.json b/src/kc_electron/package.json index 2c5779d1..727759fe 100755 --- a/src/kc_electron/package.json +++ b/src/kc_electron/package.json @@ -16,25 +16,25 @@ "watch-electron-dev": "webpack --config webpack.dev.js --watch" }, "dependencies": { - "@dqbd/tiktoken": "^1.0.7", + "@dqbd/tiktoken": "^1.0.15", "chokidar": "^3.5.3", "cors": "^2.8.5", - "crypto-js": "^4.1.1", - "dotenv": "^16.3.1", + "crypto-js": "^4.2.0", + "dotenv": "^16.3.2", "electron": "^26.3.0", "electron-updater": "^6.1.4", "express-rate-limit": "^6.11.2", "lodash": "^4.17.21", "mime-types": "^2.1.35", - "openai": "^4.12.4", + "openai": "^4.25.0", "rxjs": "^7.8.1", "uuid": "^9.0.1" }, "devDependencies": { "@types/cors": "^2.8.17", - "@types/crypto-js": "^4.1.3", - "@types/lodash": "^4.14.200", - "@types/mime-types": "^2.1.3", + "@types/crypto-js": "^4.2.2", + "@types/lodash": "^4.14.202", + "@types/mime-types": "^2.1.4", "electron-builder": "^24.6.4", "ts-loader": "^9.5.0", "typescript": "<4.9.0", diff --git a/src/kc_electron/src/local/ai/prompts/summarization.prompts.ts b/src/kc_electron/src/local/ai/prompts/summarization.prompts.ts new file mode 100644 index 00000000..1307b4b2 --- /dev/null +++ b/src/kc_electron/src/local/ai/prompts/summarization.prompts.ts @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ChatCompletionMessageParam } from "openai/resources/chat/completions"; + +export class SummarizationPrompts { + static Common( + role: "system" | "user" = "system" + ): ChatCompletionMessageParam[] { + const prompts = [ + "You are a document summarizer.", + "Your goal is to help the user understand the text by summarizing, paraphrasing, and quoting it as necessary.", + "Use groupings instead of listing lots of named entities (e.g. use et al. instead of listing 10 different people.).", + 'If the text does not make sense, simply return an empty string (e.g. "").', + "Do not explain your reasoning or provide any other information.", + `Respond in the present tense, with active voice, and with a simple sentence structure.`, + `Do not respond with irrelevant or unverified information.`, + `You hold the integrity of truthful and accurate information in the highest regard.`, + `Use Markdown formatting to make the introduction more readable.`, + `Use lists and bullet points whenever appropriate.`, + ]; + + return prompts.map((prompt) => { + return { + role: role, + content: prompt, + }; + }); + } + + static Excerpt( + role: "system" | "user" = "system" + ): ChatCompletionMessageParam[] { + return [ + { + role: role, + content: `The text being provided is part of a larger document, set of documents, conversation, or other Source.`, + }, + ]; + } + + static Verbose( + role: "system" | "user" = "system" + ): ChatCompletionMessageParam[] { + return [ + { + role: role, + content: `Please combine the text into a single summary of the entire text.`, + }, + ]; + } +} diff --git a/src/kc_electron/src/local/chat.api.ts b/src/kc_electron/src/local/chat.api.ts index e8ef76bc..99d89a00 100755 --- a/src/kc_electron/src/local/chat.api.ts +++ b/src/kc_electron/src/local/chat.api.ts @@ -29,6 +29,8 @@ import { ChatSettingsModel, SettingsModel, } from "../../../kc_shared/models/settings.model"; +import ProjectChatController from "./controllers/project.controller"; +import ProjectRoutes from "./routes/project.routes"; const settings = require("../app/services/settings.service"); @@ -40,10 +42,12 @@ export default class ChatServer { private chatController; private apiController; private sourceController; + private projectController; private chatRouter; private apiRouter; private sourceRouter; + private projectRouter; constructor() { this.tokenizerUtils = new TokenizerUtils(); @@ -64,14 +68,20 @@ export default class ChatServer { this.tokenizerUtils, this.chatController ); + this.projectController = new ProjectChatController( + this.tokenizerUtils, + this.chatController + ); const chatRoutes = new ChatRoutes(this.chatController); const apiRoutes = new ApiRoutes(this.apiController); const sourceRoutes = new SourceRoutes(this.sourceController); + const projectRoutes = new ProjectRoutes(this.projectController); this.chatRouter = chatRoutes.getRouter(); this.apiRouter = apiRoutes.getRouter(); this.sourceRouter = sourceRoutes.getRouter(); + this.projectRouter = projectRoutes.getRouter(); this.app = express(); this.setupMiddleware(); @@ -93,5 +103,6 @@ export default class ChatServer { this.app.use("/api", ErrorHandler.catchErrors(this.apiRouter)); this.app.use("/sources", ErrorHandler.catchErrors(this.sourceRouter)); this.app.use("/chat", ErrorHandler.catchErrors(this.chatRouter)); + this.app.use("/projects", ErrorHandler.catchErrors(this.projectRouter)); } } diff --git a/src/kc_electron/src/local/constants/source.prompts.ts b/src/kc_electron/src/local/constants/source.prompts.ts index 38060c24..9c4a6b75 100755 --- a/src/kc_electron/src/local/constants/source.prompts.ts +++ b/src/kc_electron/src/local/constants/source.prompts.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,37 +14,34 @@ * limitations under the License. */ -import { Completions } from "openai/resources/chat"; -import CreateChatCompletionRequestMessage = Completions.CreateChatCompletionRequestMessage; +import { ChatCompletionMessageParam } from "openai/resources/chat/completions"; export const introPrompts = ( concept: any, - type: "source" | "project" -): CreateChatCompletionRequestMessage[] => { + type: "source" | "project", + text: string +): ChatCompletionMessageParam[] => { const prompts = [ - `Information cutoff: December 2021, Current date: ${new Date().toDateString()}.`, - `Knowledge is a tool for saving, searching, accessing, and exploring all of your favorite websites, documents and files.`, + `Today's date is ${new Date().toDateString()}.`, + `Knowledge is a tool for saving, searching, accessing, exploring, and chatting with all of your favorite websites, documents and files.`, `Knowledge is an application that can organize your research and learning materials.`, `A Source is a URL or local file path that contains information, data, or other digital resources. Users import Sources into Knowledge.`, - `You can chat with a Source and ask it questions.`, - `Example: [Source="https://en.wikipedia.org/wiki/France"]\nQ: What is the capital of France?\nA: Paris`, - `Example: [Source="/home/users/user/Documents/MyFile.txt"]\nQ: How many times does the word "prototype" appear in MyFile.txt?\nA: 5`, `A Project is a collection of Sources that you can organize into a hierarchy.`, `A Project can have multiple Subprojects, each of which can have their own Subprojects.`, `You are the ${type.toLocaleUpperCase()} agent, and your job is to answer questions about ${type.toLocaleUpperCase()}s.`, `Each Source has a dedicated agent that works to help the user understand the content and provide key insights and concepts.`, `When asked for an introduction to a Source, introduce it as a concept to the user in the most effective way possible.`, `An effective introduction should be clear, concise, and informative.`, - `Write the introduction in the present tense, passive voice, indicative mood, and with a simple sentence structure.`, + `Write the introduction in the present tense, active voice, and with a simple sentence structure.`, `The introduction is 2-4 paragraphs in length, each paragraph is 4-5 sentences long.`, `The introduction is engaging and encourages the user to learn more.`, `The introduction is no shorter than 150 words.`, `You should not respond with irrelevant or unverified information.`, + `You hold the integrity of truthful and accurate information in the highest regard.`, `If you are unsure of the answer, you should respond with "I don't know."`, + `You must never lie to or mislead the user.`, `Use Markdown formatting to make the introduction more readable.`, - `Example:\nQ:What is the population of France?\nA: **67 million**`, `Use lists and bullet points whenever appropriate.`, - `Example:\nQ:What are some topics covered by this Source?\nA:\n- **France**: France is...\n- **Europe**: ...\n- **European Union**: The European Union is ...`, ]; if (type === "project") { @@ -66,27 +63,40 @@ export const introPrompts = ( const description = concept.description?.substring(0, 4096); const sourceIntro = [ - `The Source is called "${concept.title}" and is in the form of ${ingestType}`, - topics && topics.length > 0 - ? `The Source has been tagged with the following topics: "${topics}"` - : `The Source has not been tagged with topics yet`, - `The Source has the following description: "${description}"`, - `Assume the user is asking you a question with the intent to learn and understand`, - `Consider what the user might want to learn from the Source`, - `Explain why the Source is important or relevant`, - `Introduction section 1: Summary of the Source`, - `Introduction section 2: List of important concepts and ideas mentioned in the Source`, - `Introduction section 3: Follow-up questions or recommendations for additional reading based on the Source`, - `If you have extra room, recommend new topics for the user to consider adding to the Source.`, - `You hold the integrity of truthful and accurate information in the highest regard.`, - `You must never lie to or mislead the user.`, + `The Source is titled "${concept.title}".`, `You must only use information available in the Source`, ]; + + sourceIntro.push("Explain why the Source is important or relevant."); + + if (description && description.length > 0) { + sourceIntro.push(`Source description:\n===\n> ${description}\n===\n`); + } + + sourceIntro.push( + "Text extracted from the Source " + + ingestType + + ":\n===\n" + + "> " + + text + + "\n===\n" + ); + + if (topics && topics.length > 0) { + sourceIntro.push("Important Source topics: " + topics); + } + + sourceIntro.push( + "Please organize your response as follows:" + + "# Summary of the Source" + + "\n{{succinct summary}}" + ); + sourceIntro.forEach((prompt) => prompts.push(prompt)); } return prompts.map((prompt) => { - const message: CreateChatCompletionRequestMessage = { + const message: ChatCompletionMessageParam = { role: "system", content: prompt, }; @@ -109,7 +119,7 @@ export const mapReducePrompts = () => { ]; return prompts.map((prompt) => { - const message: CreateChatCompletionRequestMessage = { + const message: ChatCompletionMessageParam = { role: "system", content: prompt, }; diff --git a/src/kc_electron/src/local/controllers/chat.controller.ts b/src/kc_electron/src/local/controllers/chat.controller.ts index de4e5e5b..a3624162 100755 --- a/src/kc_electron/src/local/controllers/chat.controller.ts +++ b/src/kc_electron/src/local/controllers/chat.controller.ts @@ -23,10 +23,16 @@ import { } from "../../../../kc_shared/models/settings.model"; import TokenizerUtils from "../utils/tokenizer.utils"; import { OpenAI } from "openai"; -import { CreateChatCompletionRequestMessage } from "openai/resources"; +import { SummarizationPrompts } from "../ai/prompts/summarization.prompts"; +import { ChatCompletionMessageParam } from "openai/resources/chat/completions"; const settings = require("../../app/services/settings.service"); +/** + * The ChatController is the only controller that interacts directly with the OpenAI API. + * All other controllers should interact with the ChatController to access the API. + * This makes it easier to manage the API key and other settings. + */ export default class ChatController { private openai?: OpenAI; @@ -34,11 +40,16 @@ export default class ChatController { private tokenizerUtils: TokenizerUtils; + /** + * @param tokenizerUtils The tokenizer utils to use for this controller + */ constructor(tokenizerUtils: TokenizerUtils) { this.tokenizerUtils = tokenizerUtils; + // Initialize the OpenAI API this.init(); + // Listen for changes to the chat settings and update the settings as necessary settings.all .pipe( skip(1), @@ -56,114 +67,181 @@ export default class ChatController { return this.settings; } - limitText(text: string) { - return this.tokenizerUtils.limitText(text); - } - - async summarize(text: string, succinct = true, verbose = false) { - if (!(await this.veryifyApi()) || !this.openai) { - return text; + async summarizeChunk(text: string) { + if (!(await this.verifyAPI()) || !this.openai) { + console.warn( + "[ChatController]: OpenAI API not initialized or unavailable, skipping summarization..." + ); + return ""; } + const messages = ([] as ChatCompletionMessageParam[]).concat( + SummarizationPrompts.Common(), + SummarizationPrompts.Excerpt() + ); + + const messageTokens = this.tokenizerUtils.countMessageTokens(messages); + const limited = this.tokenizerUtils.limitText.bind(this.tokenizerUtils); - const limitedChunks = this.tokenizerUtils.chunkLimitText.bind( - this.tokenizerUtils + + text = limited(text.replace("\n", " "), messageTokens + 400).replace( + "\n", + " " ); + messages.push({ + role: "user", + content: text, + }); + try { - const messages: CreateChatCompletionRequestMessage[] = [ - { - role: "system", - content: - "Your goal is to help the user understand the text by summarzing, paraphrasing, and quoting it as necessary.", - }, - { - role: "system", - content: `Use groupings instead of listing lots of named entities (e.g. use et al. instead of listing 10 different people.).`, - }, - { - role: "system", - content: `If the text does not make sense, simply return an empty string (e.g. "").`, - }, - ]; - - if (succinct && !verbose) { - messages.push({ - role: "user", - content: `It is important to be as succinct as possible.`, - }); - messages.push({ - role: "system", - content: "The text is a small excerpt from a larger document.", - }); + const response = await this.openai.chat.completions.create({ + model: this.settings.model.name, + temperature: this.settings.model.temperature, + top_p: this.settings.model.top_p, + max_tokens: this.settings.model.max_tokens, + presence_penalty: this.settings.model.presence_penalty, + frequency_penalty: this.settings.model.frequency_penalty, + messages: messages, + }); + return response.choices[0].message.content; + } catch (error) { + console.error("Failed to summarize chunk..."); + console.error(error); + return ""; + } + } - const limitedText = limited(text); + async summarizeChunkResponses(responses: string[], windowSize = 3) { + if (!(await this.verifyAPI()) || !this.openai) { + console.warn( + "[ChatController]: OpenAI API not initialized or unavailable, skipping summarization..." + ); + return ""; + } - messages.push({ - role: "user", - content: `\n=========\n"""${limitedText}"""\n=========`, - }); - } else if (verbose) { - messages.push({ - role: "system", - content: - "The text is a concatenation of your previous responses to the user.", - }); - messages.push({ - role: "system", - content: - "For each section of your response, consider all of the relevant information from your previous responses.", + /** + * At this point, we have an array of responses, each one summarizing a small chunk of the text. + * We want to summarize all of these responses into a single summary. + * It is important for there to be some overlap between the chunks, in order to provide context + * to the summarizer. We will create a sliding window of 3 chunks, where the 3rd chunk of array + * N is the 1st chunk of array N+1. After all of the chunks have been summarized, we will + * summarize the resulting array of summaries. + */ + + // Create the sliding windows + const limited = this.tokenizerUtils.limitText.bind(this.tokenizerUtils); + const windows: string[] = []; + for ( + let i = 0; + i <= responses.length - windowSize + 1; + i += windowSize - 1 + ) { + const endIndex = + i + windowSize - 1 > responses.length - 1 + ? responses.length - 1 + : i + windowSize - 1; + const window = responses.slice(i, endIndex); + const windowText = limited(window.join("\n> ")); + windows.push(windowText); + } + + // Summarize each window + const summaries: string[] = []; + + for (let i = 0; i < windows.length; i++) { + const messages = ([] as ChatCompletionMessageParam[]).concat( + SummarizationPrompts.Common(), + SummarizationPrompts.Verbose(), + { role: "user", content: windows[i] } + ); + + try { + const response = await this.openai.chat.completions.create({ + model: this.settings.model.name, + temperature: this.settings.model.temperature, + top_p: this.settings.model.top_p, + max_tokens: this.settings.model.max_tokens, + presence_penalty: this.settings.model.presence_penalty, + frequency_penalty: this.settings.model.frequency_penalty, + messages: messages, }); - const limitedChunksText = limitedChunks(text); - let chunkedResponse = ""; - - for (let i = 0; i < limitedChunksText.length; i++) { - // If the last message is a user message, remove it from the array - // This is to ensure that each chunk is removed from the previous query - if (messages[messages.length - 1].role === "user") { - messages.pop(); - } - - const chunk = limited(limitedChunksText[i]); - - messages.push({ - role: "user", - content: `=========\n"""${chunk}"""\n=========`, - }); - - // Sleep for 0.25 seconds to avoid rate limiting - await new Promise((resolve) => setTimeout(resolve, 250)); - - const response = await this.openai.chat.completions.create({ - model: this.settings.model.name, - temperature: this.settings.model.temperature, - top_p: this.settings.model.top_p, - max_tokens: this.settings.model.max_tokens, - presence_penalty: this.settings.model.presence_penalty, - frequency_penalty: this.settings.model.frequency_penalty, - messages: messages, - }); - - const summary = response.choices[0].message.content; - chunkedResponse += summary + "\n\n"; - - if (messages[messages.length - 1].role === "user") { - messages.pop(); - } - - messages.push({ - role: "system", - content: `It is important to be as detailed as possible. Your summary should be 2-3 paragraphs in length.`, - }); - - messages.push({ - role: "user", - content: `=========\n"""${chunkedResponse}"""\n=========`, - }); + const summary = response.choices[0].message.content; + + if (summary) { + summaries.push(summary); } + } catch (error) { + console.error("Could not get response from OpenAI API..."); + console.error(error); + return null; + } + } + + // Summarize the summaries + const messages = ([] as ChatCompletionMessageParam[]).concat( + SummarizationPrompts.Common(), + SummarizationPrompts.Verbose(), + { + role: "system", + content: + "Make sure you include the headings (# and ##) and markdown in your summary!", + }, + { + role: "system", + content: `Required sections + ## (

) Brief + {{Explain like I'm 5, in 2 to 3 sentences}} + ## (

) Summary + {{2 to 3 paragraphs summarizing the Source}} + `, + }, + { + role: "user", + content: `Text to summarize:\n===\n${limited( + summaries.join(" ") + )}\n===\n`, } + ); + + // Create 4 calls to the API for each of the 4 sections of the summary + const config = { + model: this.settings.model.name, + temperature: this.settings.model.temperature, + top_p: this.settings.model.top_p, + max_tokens: 1024, + presence_penalty: this.settings.model.presence_penalty, + frequency_penalty: this.settings.model.frequency_penalty, + }; + + try { + const response = await this.openai.chat.completions.create({ + ...config, + messages: messages, + }); + return response.choices[0].message.content; + } catch (error) { + console.error("Could not get response from OpenAI API..."); + console.error(error); + return null; + } + } + async passthrough(req: Request, res: Response): Promise { + if (!req.body.messages) { + return res.status(400).json({ + error: "Missing messages", + }); + } + + if (!(await this.verifyAPI()) || !this.openai) { + return res.status(500).json({ + error: "OpenAI API not initialized", + }); + } + + try { const response = await this.openai.chat.completions.create({ model: this.settings.model.name, temperature: this.settings.model.temperature, @@ -171,19 +249,27 @@ export default class ChatController { max_tokens: this.settings.model.max_tokens, presence_penalty: this.settings.model.presence_penalty, frequency_penalty: this.settings.model.frequency_penalty, - messages: messages, + messages: req.body.messages, }); - return response.choices[0].message.content; + // If the query fails, return an error + if (!response || !response.choices || response.choices.length === 0) { + return res.status(500).json({ + error: "OpenAI API returned no response", + }); + } + + return res.json(response); } catch (error) { - console.error("Error limiting text..."); console.error(error); - return text; + return res.status(500).json({ + error: `Error calling OpenAI API: ${error}`, + }); } } async chat(req: Request, res: Response): Promise { - if (!(await this.veryifyApi()) || !this.openai) { + if (!(await this.verifyAPI()) || !this.openai) { return res.status(500).json({ error: "OpenAI API not initialized", }); @@ -192,14 +278,6 @@ export default class ChatController { // Remove any duplicate messages let messages = this.tokenizerUtils.deduplicate(req.body.messages); - // Insert the existing summary into the messages - if (req.body.summary) { - messages.push({ - role: "assistant", - content: `Here's a summary of the source:\n===\n${req.body.summary}"""\n===\n`, - }); - } - try { messages = this.tokenizerUtils.limitTokens( messages, @@ -213,10 +291,6 @@ export default class ChatController { }); } - // console.log("After token limit: ", messages); - - console.log("Using chat settings: ", this.settings); - console.log("Using messages: ", messages); try { const response = await this.openai.chat.completions.create({ model: this.settings.model.name, @@ -227,6 +301,14 @@ export default class ChatController { frequency_penalty: this.settings.model.frequency_penalty, messages: messages, }); + + // If the query fails, return an error + if (!response || !response.choices || response.choices.length === 0) { + return res.status(500).json({ + error: "OpenAI API returned no response", + }); + } + return res.json(response); } catch (error) { console.error(error); @@ -236,6 +318,34 @@ export default class ChatController { } } + async send(messages: ChatCompletionMessageParam[]) { + if (!(await this.verifyAPI()) || !this.openai) { + return null; + } + + try { + const response = await this.openai.chat.completions.create({ + model: this.settings.model.name, + temperature: this.settings.model.temperature, + top_p: this.settings.model.top_p, + max_tokens: this.settings.model.max_tokens, + presence_penalty: this.settings.model.presence_penalty, + frequency_penalty: this.settings.model.frequency_penalty, + messages: messages, + }); + + // If the query fails, return an error + if (!response || !response.choices || response.choices.length === 0) { + return null; + } + + return response; + } catch (error) { + console.error(error); + return null; + } + } + async tokens(req: Request, res: Response): Promise { const text = req.body.text; if (!text) { @@ -250,18 +360,21 @@ export default class ChatController { }); } - private async veryifyApi() { + private async verifyAPI() { if (!this.openai) { - await this.init(); + try { + await this.init(); + } catch (e) { + console.error( + `Unable to initialize OpenAI API upon verification... something is wrong.` + ); + console.error(e); + return false; + } } return true; } - private async getApiKey() { - const apiKeyPath = this.getApiKeyPath(); - return await chatEncrypt.readAndDecryptApiKey(apiKeyPath, "unsecured"); - } - private getApiKeyPath(): string { const userDataPath = settings.getSettings().system.appPath; return `${userDataPath}/openai.encrypted`; diff --git a/src/kc_electron/src/local/controllers/project.controller.ts b/src/kc_electron/src/local/controllers/project.controller.ts index 7dca5f72..8361f342 100755 --- a/src/kc_electron/src/local/controllers/project.controller.ts +++ b/src/kc_electron/src/local/controllers/project.controller.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,8 +15,15 @@ */ import { Request, Response } from "express"; +import TokenizerUtils from "../utils/tokenizer.utils"; +import ChatController from "./chat.controller"; export default class ProjectChatController { + constructor( + private tokenizerUtils: TokenizerUtils, + private chatController: ChatController + ) {} + async chat(req: Request, res: Response): Promise { // Load the entire source here return res.status(500).json({ message: "Not implemented" }); @@ -27,4 +34,93 @@ export default class ProjectChatController { console.log("Got request on sources/intro..."); return res.status(500).json({ message: "Not implemented" }); } + + async quiz(req: Request, res: Response): Promise { + const messages = this.tokenizerUtils.limitTokens( + this.tokenizerUtils.deduplicate(req.body.messages) + ); + + const prompts = [ + ...messages, + { + role: "system", + content: + "You are a teacher giving a quiz about a Project and its Sources.", + }, + { + role: "system", + content: + "Your goal is to test the students understanding of the Sources.", + }, + { + role: "system", + content: + "You will be provided with the chat history between you and the student.", + }, + { + role: "system", + content: + "You may use any questions from the Project and Sources as part of the quiz.", + }, + { + role: "system", + content: + "You may also use any questions that you think would be appropriate.", + }, + { + role: "system", + content: "Questions must be in multiple choice format.", + }, + { + role: "system", + content: + "Provide the student with 5-7 questions that have not been asked before.", + }, + { + role: "system", + content: "Do not ask the same question twice in the same quiz.", + }, + { + role: "system", + content: "Do not ask questions about people or places.", + }, + { + role: "system", + content: + "Avoid asking too questions where the answer is 'all of the above'.", + }, + { + role: "system", + content: "Please avoid asking questions that are too easy or obvious.", + }, + ]; + + prompts.push({ + role: "system", + content: + "Please use the following format for each question." + + "Q1. {{Question}}\n" + + "A) {{Choice}}\n" + + "B) {{Choice}}\n" + + "C) {{Choice}}\n" + + "D) {{Choice}}\n" + + "Answer: {{letter of the answer}}\n\n\n" + + "Q2. {{Question}}\n" + + "A) {{Choice}}\n" + + "B) {{Choice}}\n" + + "C) {{Choice}}\n" + + "D) {{Choice}}\n" + + "Answer: {{letter of the answer}}\n\n\n" + + "...", + }); + + prompts.push({ + role: "user", + content: "Please give me a quiz about this Project and its Sources.", + }); + + req.body.messages = prompts; + + return this.chatController.passthrough(req, res); + } } diff --git a/src/kc_electron/src/local/controllers/source.controller.ts b/src/kc_electron/src/local/controllers/source.controller.ts index eff1a45c..bfa40dc4 100755 --- a/src/kc_electron/src/local/controllers/source.controller.ts +++ b/src/kc_electron/src/local/controllers/source.controller.ts @@ -15,10 +15,10 @@ */ import { Request, Response } from "express"; -import { introPrompts } from "../constants/source.prompts"; import { GlobalWorkerOptions } from "pdfjs-dist"; import ChatController from "./chat.controller"; import TokenizerUtils from "../utils/tokenizer.utils"; +import { ChatCompletionMessageParam } from "openai/resources/chat/completions"; GlobalWorkerOptions.workerSrc = require("pdfjs-dist/build/pdf.worker.entry"); @@ -33,15 +33,21 @@ export default class SourceChatController { } async chat(req: Request, res: Response): Promise { + // Insert the summary into the messages array + if (req.body.summary) { + req.body.messages = req.body.messages.concat([ + { + role: "system", + content: `Summary: ${req.body.summary}`, + }, + ]); + } return this.chatController.chat(req, res); } - async intro(req: Request, res: Response): Promise { + async summarize(req: Request, res: Response): Promise { // If the summary already exists, simply return it as a ChatCompletion if (req.body.summary) { - console.debug( - "[Knowledge]: Source summary already exists, returning it..." - ); return res.json({ choices: [ { @@ -52,39 +58,434 @@ export default class SourceChatController { }, ], }); + } else { + return res.status(500).json({ + error: "Could not generate a summary. Please try again later.", + }); } + } - const source = req.body.source; - const messages = req.body.messages || []; - const noPrompts = messages.length === 0; - - // Get Source prompts based on this specific source, prepend them to messages - if (noPrompts) { - const sourceIntroPrompts = introPrompts(source, "source"); - sourceIntroPrompts.forEach((message) => { - messages.push(message); - }); + async categorize(req: Request, res: Response) { + // The request body should contain a source title and the entire project tree in YAML format + const sourceTitle = req.body.source.title; + const projectTree = JSON.parse(req.body.tree); + const projectNodes = projectTree.map((nodeText: string) => { + return JSON.parse(nodeText); + }); + let choices = projectNodes.map((node: any) => { + return node.name; + }); + + const messages: ChatCompletionMessageParam[] = this.subcategories( + choices, + req.body.text + ); + + const response = await this.chatController.send(messages); + if (!response) { + return res.status(500); } - // Extract text from web page to feed into the API - const text = this.tokenizerUtils.limitText(req.body.text); + const roots = response.choices[0].message.content?.split("\n"); - // Append the extracted text to the messages before sending - messages.push({ - role: "system", - content: `The following is the text extracted from the Source: "${source.title}":\n=========\n"""${text}"""\n=========`, + // Get the items that correspond to the selected roots + choices = projectNodes.filter((node: any) => { + return roots?.includes(node.name); }); - if (noPrompts) { - messages.push({ + // Recursively check all projects and subprojects. Remove the subprojects field if it is empty. + const removeEmptySubprojects = (node: any) => { + if (node.subprojects) { + node.subprojects = node.subprojects.filter(removeEmptySubprojects); + if (node.subprojects.length === 0) { + delete node.subprojects; + } + } + return node; + }; + choices = choices.map(removeEmptySubprojects); + + // Convert the choices to YAML + choices = JSON.stringify(choices, null, 2); + choices = choices.replace(/"name":/g, "-"); + choices = choices.replace(/"subprojects":/g, " subprojects:"); + + // Remove all brackets, quotes, commas, and visible newlines + choices = choices.replace(/[[\]{}"']/g, ""); + choices = choices.replace(/,/g, ""); + choices = choices.replace(/\n\s*\n/g, "\n"); + choices = choices.replace(/\n/g, "\n-"); + + req.body.messages = [ + { + role: "system", + content: + "You are a librarian who is adept at making ontologies and categorizing Sources.", + }, + { + role: "system", + content: + "You will be provided with information about the source and a list of available Projects.", + }, + { + role: "system", + content: + "Your goal is to recommend 3 to 5 projects that the source should be categorized under.", + }, + { + role: "system", + content: + "You should always prefer to use existing projects. You may only recommend new Projects if none of the existing Projects are appropriate.", + }, + { + role: "system", + content: + "If you suggest new Projects, you may not suggest any that are already in the list.", + }, + { + role: "system", + content: + "Projects are hierarchical, where higher levels of the hierarchy or more general. Projects can have subprojects that get increasingly specific.", + }, + { + role: "system", + content: + "Analyze the key themes and concepts before providing your recommendation. Identify any specific methodologies, geographic locations, historical periods, technologies, or theoretical frameworks mentioned or implied.", + }, + { + role: "system", + content: + "Ask yourself what a proper ontology or taxonomy would look like for the source.", + }, + { + role: "system", + content: + "Ultimately, the Project hierarchy represents the users mind-map of knowledge. The goal is to create a hierarchy that is intuitive and easy to navigate, but you should not try to change the way they structure their knowledge.", + }, + { + role: "system", + content: + "In your response, each project should be on its own line and must start with a dash (-).", + }, + { + role: "system", + content: + 'Example:\n===Source: "Bogleheads investing start-up kit"\n' + + "Source Topics: #investing #startups #investmentplan #diversification" + + "Suggested Projects:\n" + + '- Finance > Personal Finance > Investing : The document\'s focus on "investing" as a start-up kit implies a fundamental approach to managing personal assets and financial resources for growth or income.\n' + + '- Finance > Personal Finance > Investing > Strategies : The inclusion of "Bogleheads" in the title suggests a specific approach or philosophy toward investing, which aligns with the concept of distinct investment strategies.\n' + + "- Finance > Retirement Planning > Tax-Advantaged Accounts : The emphasis on investment planning and diversification within the topics implies considerations for long-term financial goals, such as retirement, where tax-advantaged accounts play a crucial role.\n" + + "===\n", + }, + { + role: "system", + content: + 'Example:\n===Source: "The Open Graph Protocol"\n' + + "Source Topics: #web3, #socialgraphs, #metadata, #rdfs" + + "Suggested Projects:\n" + + "- Internet and Web > Protocols and Standards > The Open Graph Protocol : The title indicates a specific protocol used for integrating web content with social media platforms, fitting under the broader category of internet protocols and standards.\n" + + "- Data and Information > Semantic Technologies > RDFs : RDFs are mentioned, which are a foundational technology of the Semantic Web, emphasizing structured data and metadata to enable machines to understand and interpret web content.\n" + + '- Social Media and Networking > Content Optimization and Sharing > Social Graphs : The focus on "Social Graphs" within the context of the Open Graph Protocol highlights techniques for mapping and analyzing relationships and interactions on social media platforms, pertinent to content sharing and optimization strategies.\n' + + "===\n", + }, + { + role: "system", + content: + "You should include at least one existing Project. Existing projects are listed below and use dashes to indicate hierarchy.", + }, + { + role: "system", + content: `Existing Projects: +=== +${choices} +===`, + }, + { + role: "user", + content: + `Source: "${sourceTitle}"\n` + + `Summary:\n===\n${this.tokenizerUtils.limitText( + req.body.summary, + 500 + )}\n===\n`, + }, + ]; + + return this.chatController.passthrough(req, res); + } + + private subcategories( + choices: string[], + input: string + ): ChatCompletionMessageParam[] { + return [ + { + role: "system", + content: + "You are a librarian who is adept at making ontologies and categorizing Sources.", + }, + { + role: "system", + content: + "You will be provided with information about a Source, and a list of possible categories.", + }, + { + role: "system", + content: + "Your goal is to select the most appropriate categories for the Source.", + }, + { + role: "system", + content: + "IF AND ONLY IF none of the categories are appropriate, you may suggest 2 to 3 new categories.", + }, + { + role: "system", + content: + "If you suggest new categories, you may not suggest any that are already in the list.", + }, + { + role: "system", + content: + "You must separate each category with a new line character (\\n).", + }, + { + role: "system", + content: `Source:\n===\n${this.tokenizerUtils.limitText( + input, + 500 + )}\n===`, + }, + { + role: "system", + content: `Categories:\n===\n${choices.join("\n")}\n===`, + }, + { role: "user", - content: `Can you introduce me to "${source.title}"?`, + content: + "Please select the most appropriate categories for this Source.", + }, + ]; + } + + async tutor(req: Request, res: Response) { + const prompts = [ + { + role: "system", + content: "You are a tutor teaching a student about a Source.", + }, + { + role: "system", + content: + "Your goal is to help fill in the gaps in the student's understanding.", + }, + { + role: "system", + content: + "You will be provided with your chat history with the student.", + }, + { + role: "system", + content: + "Your explanations should be clear and in simple terms when possible", + }, + { + role: "system", + content: + "If you think the student would benefit from additional resources, suggest that they start by researching particular topics or concepts.", + }, + { + role: "system", + content: `You may also consider asking what the students goals are, and what they hope to learn from the Source.`, + }, + { + role: "system", + content: `It is especially important to make sure the student has an understanding of any prerequisites required to understand the Source.`, + }, + { + role: "system", + content: "Give the student a list of prerequisite topics to research.", + }, + { + role: "system", + content: `The Source is titled "${req.body.source.title}".`, + }, + ]; + + prompts.push({ + role: "system", + content: `The following is a brief summary of the Source: ${req.body.summary}.`, + }); + + for (const message of req.body.messages) { + prompts.push(message); + } + + prompts.push({ + role: "user", + content: "Please tutor me about this Source.", + }); + + req.body.messages = prompts; + + return this.chatController.passthrough(req, res); + } + + quiz(req: Request, res: Response) { + const source = req.body.source; + + const prompts = [ + ...req.body.messages, + { + role: "system", + content: "You are a teacher giving a quiz about a Source.", + }, + { + role: "system", + content: + "Your goal is to test the students understanding of the Source.", + }, + { + role: "system", + content: + "You will be provided with your chat history with the student.", + }, + { + role: "system", + content: + "You may use any questions from the Source as part of the quiz.", + }, + { + role: "system", + content: + "You may also use any questions that you think would be appropriate.", + }, + { + role: "system", + content: "Questions must be in multiple choice format.", + }, + { + role: "system", + content: + "Provide the student with 5 questions that have not been asked before.", + }, + { + role: "system", + content: "Do not ask the same question twice in the same quiz.", + }, + { + role: "system", + content: + "You are not allowed to ask more than 2 questions where the answer is 'all of the above'", + }, + { + role: "system", + content: "The Source is titled " + source.title + ".", + }, + ]; + + if (req.body.summary) { + prompts.push({ + role: "system", + content: `The following is a summary of the Source: ${req.body.summary}.`, }); } - return this.chatController.chat(req, res); + + prompts.push({ + role: "system", + content: + "Please use the following format for each question." + + "Q1. {{Question}}\n" + + "A) {{Choice}}\n" + + "B) {{Choice}}\n" + + "C) {{Choice}}\n" + + "D) {{Choice}}\n" + + "Answer: {{letter of the answer}}\n\n\n" + + "Q2. {{Question}}\n" + + "A) {{Choice}}\n" + + "B) {{Choice}}\n" + + "C) {{Choice}}\n" + + "D) {{Choice}}\n" + + "Answer: {{letter of the answer}}\n\n\n" + + "...", + }); + + prompts.push({ + role: "user", + content: "Please give me a quiz about this Source.", + }); + + req.body.messages = prompts; + + return this.chatController.passthrough(req, res); } - async regenerate(req: Request, res: Response) { - return undefined; + topics(req: Request, res: Response) { + const text = this.tokenizerUtils.limitText(req.body.text, 500); + + const prompts = [ + ...(req.body.messages || []), + { + role: "system", + content: "You are analyzing a Source to identify its topics.", + }, + { + role: "system", + content: + "Identify the most relevant words or phrases in the text that represent key concepts.", + }, + { + role: "system", + content: + "Calculate the importance of words in the text based on how frequently they appear and how unique they are across the entire corpus.", + }, + { + role: "system", + content: + "Assume the text is a mixture of a small number of topics and that each word's presence is attributable to one of the document's topics.", + }, + { + role: "system", + content: + "Describe the topics as they relate to the text and the main concepts.", + }, + { + role: "system", + content: "Return the top 5 topics and their associated descriptions.", + }, + { + role: "system", + content: + "You MUST use the following format for each topic: - {{Topic}}: {{Description}}", + }, + { + role: "system", + content: + "You MUST separate the title and the description with a colon (:).", + }, + { + role: "system", + content: + "Do not respond with anything other than the topics and descriptions. Do not explain yourself. Do not attempt to number the topics. Do not include any other information.", + }, + { + role: "system", + content: `Topics may be no more than 2 to 3 words long. Descriptions may be no more than 10 words long.`, + }, + { + role: "system", + content: `If the text contains minor errors, like inappropriate whitespace, please correct it before analyzing.`, + }, + ]; + + prompts.push({ + role: "user", + content: `Please identify the topics in the following passage:\n===\n${text}\n===`, + }); + + req.body.messages = prompts; + + return this.chatController.passthrough(req, res); } } diff --git a/src/kc_electron/src/local/middleware/DocumentSummarizer.ts b/src/kc_electron/src/local/middleware/DocumentSummarizer.ts index 08f2486a..7cd9df1d 100644 --- a/src/kc_electron/src/local/middleware/DocumentSummarizer.ts +++ b/src/kc_electron/src/local/middleware/DocumentSummarizer.ts @@ -20,7 +20,6 @@ import ChatController from "../controllers/chat.controller"; export class DocumentSummarizer { private chatController: ChatController; constructor(chatController: ChatController) { - console.log("DocumentSummarizer initializing..."); this.chatController = chatController; } @@ -28,39 +27,59 @@ export class DocumentSummarizer { return async (req: Request, res: Response, next: NextFunction) => { // If the request already contains a summary, skip this middleware if (req.body.summary) { + console.log( + "[DocumentSummarizer]: Summary already exists, skipping DocumentSummarizer" + ); return next(); } - let text = req.body.text; + // Break the text into chunks under the assumption that each token represents approximately 3 characters + const CHUNK_SIZE = + (this.chatController.getSettings().model.token_limit - + this.chatController.getSettings().model.max_tokens) * + 3; + const chunks = req.body.text.match(new RegExp(`.{1,${CHUNK_SIZE}}`, "g")); + + console.log( + `[DocumentSummarizer]: Splitting text into ${chunks.length} chunks of size ${CHUNK_SIZE}...` + ); + + if (chunks && chunks.length > 0) { + const summaries = await Promise.all( + chunks.map( + this.chatController.summarizeChunk.bind(this.chatController) + ) + ); + + if (summaries.length === 1) { + req.body.summary = summaries[0]; + } else { + const response = await this.chatController.summarizeChunkResponses( + summaries + ); - // Absolute limit of text length is 100,000 characters, truncate anything above - if (text.length > 100000) { + if (response === "" || response === undefined || response === null) { + return res.status(500).send({ + message: "OpenAI API Error - Unable to summarize text.", + }); + } + req.body.summary = response; + } + } else { console.warn( - `[DocumentSummarizer]: truncating text of size ${text.length} to 100,000 characters` + "[DocumentSummarizer]: unable to split text into chunks..." ); - req.body.text = text.substring(0, 100000); - text = req.body.text; + req.body.summary = ""; } - // Generate a summary of the text - const CHUNK_SIZE = 10000; - const chunks = text.match(new RegExp(`.{1,${CHUNK_SIZE}}`, "g")); + next(); + }; + } - if (chunks) { - const summaries = await Promise.all( - chunks.map(this.chatController.summarize.bind(this.chatController)) - ); - const initialSummaries = summaries.join("\n").trim(); - const summary = await this.chatController.summarize( - initialSummaries, - false, - true - ); - req.body.summary = summary; - next(); - } else { - console.warn("DocumentSummarizer: unable to split text into chunks..."); - next(); + topics() { + return async (req: Request, res: Response, next: NextFunction) => { + if (req.body.topcs) { + return next(); } }; } diff --git a/src/kc_electron/src/local/middleware/SourceLoader.ts b/src/kc_electron/src/local/middleware/SourceLoader.ts index 136b0cef..f2040a13 100644 --- a/src/kc_electron/src/local/middleware/SourceLoader.ts +++ b/src/kc_electron/src/local/middleware/SourceLoader.ts @@ -22,9 +22,8 @@ const settingsService = require("../../app/services/settings.service"); export class SourceLoader { static async load(req: Request, res: Response, next: NextFunction) { - const source = req.body.source; - const id = source.id; - const filename = `${id.value}.json`; + const id = req.body.id; + const filename = `${id}.json`; const filepath = path.resolve( settingsService.getSettings().system.appPath, @@ -45,30 +44,16 @@ export class SourceLoader { const data = JSON.parse(json); req.body.text = data[0] ?? undefined; req.body.summary = data[1] ?? undefined; - } else { - console.debug("Source not found in local storage: ", req.body.source); } - console.debug("Loaded source from local storage: ", req.body.source); - console.debug( - "Length of text loaded from local storage: ", - req.body.text?.length - ); - console.debug( - "Length of summary loaded from local storage: ", - req.body.summary?.length - ); - next(); } static async store(req: Request, res: Response, next: NextFunction) { - const source = req.body.source; const text = req.body.text; const summary = req.body.summary; - - const id = source.id; - const filename = `${id.value}.json`; + const id = req.body.id; + const filename = `${id}.json`; const filepath = path.resolve( settingsService.getSettings().system.appPath, @@ -83,8 +68,6 @@ export class SourceLoader { fs.mkdirSync(dirname, { recursive: true }); } - console.debug("Storing source text and summary at: ", filepath); - // Store the Source into local JSON file using ID as filename. const json = JSON.stringify([text, summary]); fs.writeFileSync(filepath, json, "utf8"); diff --git a/src/kc_electron/src/local/middleware/SourceParser.ts b/src/kc_electron/src/local/middleware/SourceParser.ts index 0487d425..c37eee73 100644 --- a/src/kc_electron/src/local/middleware/SourceParser.ts +++ b/src/kc_electron/src/local/middleware/SourceParser.ts @@ -20,34 +20,49 @@ import { getDocument } from "pdfjs-dist"; import { htmlToText, HtmlToTextOptions } from "html-to-text"; import fs from "fs"; import TextUtils from "../utils/text.utils"; +import { codeMarkdownMap } from "../../../../kc_shared/constants/supported.file.types"; export class SourceParser { + static async isPdf(url: string) { + try { + const response = await fetch(url, { method: "HEAD" }); + const contentType = response.headers.get("Content-Type"); + return contentType === "application/pdf"; + } catch (error) { + console.error("Error fetching the URL:", error); + return false; + } + } + static async getText(req: Request, res: Response, next: NextFunction) { - // If the request already contains text, skip this middleware if (req.body.text) { - console.debug( - "Text already extracted from source, skipping SourceParser" - ); + // If the request already contains text, skip this middleware return next(); } - const source = req.body.source; - const ingestType = source.ingestType; + const ingestType = req.body.ingestType; + const accessLink = req.body.accessLink; if (ingestType === "file") { - req.body.text = await SourceParser.fromFile(source.accessLink); + req.body.text = await SourceParser.fromFile(accessLink); } else { - const accessLink = new URL(source.accessLink); - req.body.text = await SourceParser.fromWeb(accessLink); + req.body.text = await SourceParser.fromWeb(new URL(accessLink)); } - // Clean the text - req.body.text = TextUtils.clean(req.body.text); - - console.debug( - "Length of text extracted from source: ", - req.body.text.length - ); + if ( + req.body.text && + typeof req.body.text === "string" && + req.body.text.trim().length > 0 + ) { + // Clean the text + req.body.text = TextUtils.clean(req.body.text); + + // Absolute limit of text length is 100,000 characters, truncate anything above + // TODO: make this a more meaningful value + req.body.text = TextUtils.limit(req.body.text, 100000); + } else { + req.body.text = ""; + } next(); } @@ -55,45 +70,44 @@ export class SourceParser { static async fromFile(filePath: string) { // Check if the file is a PDF if (filePath.endsWith(".pdf")) { - console.debug("PDF detected, attempting to extract text from PDF..."); return SourceParser.fromPdf(filePath); - } else { - const supportedFileTypes = [".txt", ".md", ".html", ".htm"]; - const codeFileTypes = [ - ".py", - ".js", - ".ts", - ".java", - ".c", - ".cpp", - ".h", - ".hpp", - ".cs", - ".go", - ".rs", - ".sh", - ]; + } + try { // Read the file and check if it is a plain text file + const fileExtension = "." + filePath.split(".").pop(); + const supportedFileTypes = [".txt", ".rst"].concat( + Object.keys(codeMarkdownMap) + ); + + // If the file is not a supported file type, return an empty string + if (!fileExtension || !supportedFileTypes.includes(fileExtension)) { + console.debug("SourceParser: unsupported file type: ", fileExtension); + return ""; + } + + // Otherwise, read the file and return the contents const dataBuffer = fs.readFileSync(filePath); const text = dataBuffer.toString(); - if (supportedFileTypes.some((type) => filePath.endsWith(type))) { - console.log("Supported file type detected, returning text..."); - return `This file contains the following text: ${text}`; - } else if (codeFileTypes.some((type) => filePath.endsWith(type))) { - console.log("Code file detected, returning text..."); - return `This file contains the following source code: - === BEGIN CODE - ${text} - === END CODE - `; - } else if (TextUtils.isPlainText(text)) { - console.log("Plain text file detected, returning text..."); - return `This file contains the following plain text: ${text}`; + const isCode = Object.keys(codeMarkdownMap).some((type) => + filePath.endsWith(type) + ); + + if (isCode) { + // Iterate over all key-value pairs to get fileType + Object.entries(codeMarkdownMap).forEach(([key, fileType]) => { + if (filePath.endsWith(key)) { + return "```" + fileType + "\n" + text + "\n```"; + } + }); + + return "```\n" + text + "\n```"; } - return "UNREADABLE FILE"; + return text; + } catch (err) { + return ""; } } @@ -101,44 +115,46 @@ export class SourceParser { const dataBuffer = fs.readFileSync(filePath); const uint8Array = new Uint8Array(dataBuffer.buffer); - // Load the PDF file using PDF.js - const loadingTask = getDocument({ - data: uint8Array, - }); - const pdf = await loadingTask.promise; + try { + // Load the PDF file using PDF.js + const loadingTask = getDocument({ + data: uint8Array, + }); + const pdf = await loadingTask.promise; - let finalText = ""; + let finalText = " "; - // Loop over each page in the PDF (1-indexed) - for (let i = 1; i <= pdf.numPages; i++) { - const page = await pdf.getPage(i); - const content = await page.getTextContent({ - includeMarkedContent: false, - }); + // Loop over each page in the PDF (1-indexed) + for (let i = 1; i <= pdf.numPages; i++) { + const page = await pdf.getPage(i); + const content = await page.getTextContent({ + includeMarkedContent: false, + }); - // Concatenate content items to assemble final text - // @ts-ignore -- required because the compiler thinks item.str does not exist - const strings = content.items.map((item) => item.str); - finalText += strings.join(" ") + "\n"; - } + // Concatenate content items to assemble final text + // @ts-ignore -- required because the compiler thinks item.str does not exist + const strings = content.items.map((item) => item.str); + finalText += strings.join(" ") + "\n"; + } - return finalText; + return finalText; + } catch (err) { + return ""; + } } static async fromWeb(url: URL) { const reqUrl = url.toString(); - if (url.href.endsWith(".pdf")) { - console.log("PDF detected, attempting to extract text from PDF..."); - + const isPDF = await SourceParser.isPdf(reqUrl); + if (url.href.endsWith(".pdf") || isPDF) { // Use http to get the PDF, then use pdfjs to extract the text const response = await axios.get(reqUrl, { responseType: "arraybuffer" }); const data = new Uint8Array(response.data); const doc = await getDocument(data).promise; // Extract text from PDF - let text = - "The following is extracted from the PDF (it may be incomplete but you should still attempt to summarize it)."; + let text = ""; for (let i = 1; i <= doc.numPages; i++) { const page = await doc.getPage(i); const content = await page.getTextContent({ @@ -159,12 +175,13 @@ export class SourceParser { selectors: [ "article", "p", + "code", "blockquote", + "textarea", "ol", "ul", "h1", "h2", - "code", "mark", "table", "h3", diff --git a/src/kc_electron/src/local/middleware/SourceValidator.ts b/src/kc_electron/src/local/middleware/SourceValidator.ts new file mode 100644 index 00000000..223d8a0f --- /dev/null +++ b/src/kc_electron/src/local/middleware/SourceValidator.ts @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { NextFunction, Request, Response } from "express"; + +export class SourceValidator { + static async validate(req: Request, res: Response, next: NextFunction) { + try { + if (!req.body.source) { + return res.status(400).send("Missing Source"); + } + } catch (err) { + return res.status(400).send("Invalid source"); + } + + try { + if (!req.body.source.id?.value) { + return res.status(400).send("Missing Source ID"); + } else { + req.body.id = req.body.source.id.value; + } + } catch (err) { + return res.status(400).send("Invalid source ID"); + } + + try { + if (!req.body.source.title) { + return res.status(400).send("Missing Source Title"); + } + } catch (err) { + return res.status(400).send("Invalid Source Title"); + } + + try { + if (!req.body.source.ingestType) { + return res.status(400).send("Missing Source Type"); + } + req.body.ingestType = req.body.source.ingestType; + } catch (err) { + return res.status(400).send("Invalid Source Type"); + } + + try { + if (!req.body.source.accessLink) { + return res.status(400).send("Missing Source Access Link"); + } + req.body.accessLink = req.body.source.accessLink; + } catch (err) { + return res.status(400).send("Invalid Source Access link"); + } + + next(); + } + + static async hasText(req: Request, res: Response, next: NextFunction) { + try { + if (!req.body.text || req.body.text.trim().length === 0) { + const helpLink = + "https://github.com/KnowledgeCanvas/knowledge/wiki/Chat#supported-file-types"; + return res.json({ + choices: [ + { + message: { + content: `I'm sorry, but I am unable to read the associated ${req.body.ingestType}. Please see ${helpLink} for more info on supported file types.`, + role: "assistant", + }, + }, + ], + }); + } + } catch (err) { + return res.status(400).send("Invalid Source Text"); + } + next(); + } +} diff --git a/src/kc_angular/src/app/components/source-components/source.notes.component.ts b/src/kc_electron/src/local/routes/project.routes.ts old mode 100755 new mode 100644 similarity index 53% rename from src/kc_angular/src/app/components/source-components/source.notes.component.ts rename to src/kc_electron/src/local/routes/project.routes.ts index a15bb744..8cf1b176 --- a/src/kc_angular/src/app/components/source-components/source.notes.component.ts +++ b/src/kc_electron/src/local/routes/project.routes.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,13 +14,24 @@ * limitations under the License. */ -import { Component } from '@angular/core'; +import express from "express"; +import ProjectChatController from "../controllers/project.controller"; -@Component({ - selector: 'source-notes', - template: ``, - styles: [``], -}) -export class SourceNotesComponent { - constructor() {} +const router = express.Router(); + +export default class ProjectRoutes { + private projectController: ProjectChatController; + + constructor(controller: ProjectChatController) { + this.projectController = controller; + } + + getRouter() { + router.post( + "/quiz", + this.projectController.quiz.bind(this.projectController) + ); + + return router; + } } diff --git a/src/kc_electron/src/local/routes/source.routes.ts b/src/kc_electron/src/local/routes/source.routes.ts index 6656cbd2..b41ba188 100755 --- a/src/kc_electron/src/local/routes/source.routes.ts +++ b/src/kc_electron/src/local/routes/source.routes.ts @@ -19,6 +19,7 @@ import SourceChatController from "../controllers/source.controller"; import { SourceParser } from "../middleware/SourceParser"; import { SourceLoader } from "../middleware/SourceLoader"; import { DocumentSummarizer } from "../middleware/DocumentSummarizer"; +import { SourceValidator } from "../middleware/SourceValidator"; const router = express.Router(); @@ -29,7 +30,6 @@ export default class SourceRoutes { constructor(controller: SourceChatController) { this.sourceController = controller; this.summarizer = new DocumentSummarizer(controller.getChatController()); - this.getRouter(); } get controller() { @@ -37,12 +37,16 @@ export default class SourceRoutes { } getRouter() { + router.use(SourceValidator.validate); + // Load the source from the database router.use(SourceLoader.load); // Parse the source into text (if necessary) router.use(SourceParser.getText); + router.use(SourceValidator.hasText); + // Summarize the text (if necessary) router.use(this.summarizer.summarize()); @@ -53,14 +57,26 @@ export default class SourceRoutes { router.post("/", this.controller.chat.bind(this.controller)); // Intro chat with the source (basically a summary) - router.post("/intro", this.controller.intro.bind(this.controller)); + router.post("/summarize", this.controller.summarize.bind(this.controller)); - // An endpoint to regenerate previous responses router.post( - "/regenerate", - this.controller.regenerate.bind(this.controller) + "/categorize", + this.controller.categorize.bind(this.controller) ); + // Get customized advice on how to read the Source + router.post("/tutor", this.controller.tutor.bind(this.controller)); + + router.post("/quiz", this.controller.quiz.bind(this.controller)); + + router.post("/topics", this.controller.topics.bind(this.controller)); + + // An endpoint to regenerate previous responses + // router.post( + // "/regenerate", + // this.controller.regenerate.bind(this.controller) + // ); + return router; } } diff --git a/src/kc_electron/src/local/utils/text.utils.ts b/src/kc_electron/src/local/utils/text.utils.ts index 36cbc836..0ea3fa33 100755 --- a/src/kc_electron/src/local/utils/text.utils.ts +++ b/src/kc_electron/src/local/utils/text.utils.ts @@ -63,12 +63,18 @@ export default class TextUtils { return chunks; } + static limit(text: string, limit: number): string { + if (text.length > limit) { + text = text.substring(0, limit); + } + return text; + } + static clean(text: string): string { text = text.replace(/(https?:\/\/[^\s]+)/g, ""); // Remove URLs text = text.replace(/(\r\n|\n|\r)/gm, ""); // Remove newlines text = text.replace(/(\t)/gm, ""); // Remove tabs text = text.replace(/(\s{2,})/g, " "); // Remove any double spaces - text = text.replace(/[^a-zA-Z0-9 ]/g, ""); // Remove any non-alphanumeric characters // Remove any non-whitespace substrings longer than 32 characters // (this is to prevent the API from returning an error) diff --git a/src/kc_electron/src/local/utils/tokenizer.utils.ts b/src/kc_electron/src/local/utils/tokenizer.utils.ts index be971dec..5acea207 100755 --- a/src/kc_electron/src/local/utils/tokenizer.utils.ts +++ b/src/kc_electron/src/local/utils/tokenizer.utils.ts @@ -27,8 +27,7 @@ import { ChatModel, SupportedChatModels, } from "../../../../kc_shared/models/chat.model"; -import { Completions } from "openai/resources/chat"; -import CreateChatCompletionRequestMessage = Completions.CreateChatCompletionRequestMessage; +import { ChatCompletionMessageParam } from "openai/resources/chat/completions"; const settingsService = require("../../app/services/settings.service"); @@ -74,19 +73,44 @@ export default class TokenizerUtils { } } + shortenText(text: string, maxTokens: number): string { + const tokenized = this.tiktoken.encode(text); + if (tokenized.length <= maxTokens) { + return text; + } + const model = this.verifiedModel(); + + // If the current model has a lower token limit, use that instead + maxTokens = Math.min(maxTokens, model.token_limit); + + const shortened = tokenized.slice(0, maxTokens); + const decoded = this.tiktoken.decode(shortened); + + // Decoded is an array of Unit8Array, we need to convert that back into a string + let shortenedText = ""; + decoded.forEach((unit8) => { + shortenedText += String.fromCharCode.apply(null, [unit8]); + }); + + return shortenedText; + } + /** * Given a token limit, prune the messages to fit within the limit. * Fails if the last message is longer than the limit. * Removes non-system messages first, then removes system messages if necessary. */ - limitTokens( - messages: CreateChatCompletionRequestMessage[], - max_tokens: number - ) { + limitTokens(messages: ChatCompletionMessageParam[], max_tokens = 0) { if (messages.length === 0) { return messages; } + // If max_tokens is 0, use the model's parameters + if (max_tokens === 0) { + const model = this.verifiedModel(); + max_tokens = model.token_limit - model.max_tokens - 512; + } + // If the messages are already within the token limit, return them. let tokenCount = this.countMessageTokens(messages); if (tokenCount < max_tokens) { @@ -130,12 +154,19 @@ export default class TokenizerUtils { return messages; } - deduplicate(messages: CreateChatCompletionRequestMessage[]) { - // Remove duplicate messages using a hash map based on message content. - const unique: CreateChatCompletionRequestMessage[] = []; + /** + * Remove duplicate messages from a list of messages. + * @param messages + */ + deduplicate(messages: ChatCompletionMessageParam[]) { + const unique: ChatCompletionMessageParam[] = []; const hash: { [key: string]: boolean } = {}; messages.forEach((message) => { - if (message.content && !hash[message.content]) { + if ( + message.content && + typeof message.content === "string" && + !hash[message.content] + ) { hash[message.content] = true; unique.push(message); } @@ -144,24 +175,33 @@ export default class TokenizerUtils { return unique; } + /** + * Count the number of tokens in a text for the current model. + * @param text + */ countTokens(text: string) { return this.tiktoken.encode(text).length + 5; } - countMessageTokens(messages: CreateChatCompletionRequestMessage[]): number { + /** + * Count the number of tokens in a list of messages for the current model. + * @param messages + */ + countMessageTokens(messages: ChatCompletionMessageParam[]): number { let tokenCount = 0; - messages.forEach((message: CreateChatCompletionRequestMessage) => { - if (message.content) { + messages.forEach((message: ChatCompletionMessageParam) => { + if (message.content && typeof message.content === "string") { tokenCount += this.tiktoken.encode(message.content).length + 5; } }); return tokenCount + (messages.length > 1 ? 3 : 0); } + /** + * Given a text, chunk it into smaller pieces that fit within the token limit. + * @param text + */ chunkLimitText(text: string): string[] { - /** - * Given a text, chunk it into pieces that are within the token limit. - */ const model = this.verifiedModel(); const maxTokens = model.token_limit - model.max_tokens - 512; @@ -190,12 +230,17 @@ export default class TokenizerUtils { return chunkedText; } - limitText(text: string): string { + /** + * Given a text, limit it to the token limit of the model. + * @param text The text to limit + * @param padding The number of tokens to pad the limit with + */ + limitText(text: string, padding = 350): string { const model = this.verifiedModel(); - // TODO: this 512 should be equal to the number of tokens taken by the rest of the messages - let maxTokens = model.token_limit - model.max_tokens - 512; - - maxTokens = Math.max(maxTokens, 0); + const maxTokens = Math.max( + model.token_limit - model.max_tokens - padding, + 0 + ); let tokenized = this.tiktoken.encode(text); if (tokenized.length > maxTokens) { @@ -204,7 +249,6 @@ export default class TokenizerUtils { ); tokenized = tokenized.slice(0, maxTokens); const decoded = this.tiktoken.decode(tokenized); - // Decoded is an array of Unit8Array, we need to convert that back into a string text = ""; decoded.forEach((unit8) => { @@ -215,6 +259,10 @@ export default class TokenizerUtils { return text; } + /** + * Get the path to the tiktoken wasm file. This is used to initialize the tokenizer. + * @private + */ private getWasmPath() { // TODO: Figure out how to do this in a cleaner way const possiblePaths = [ @@ -239,6 +287,9 @@ export default class TokenizerUtils { throw new Error("Could not find tiktoken wasm file."); } + /** + * Get the model object for the current model. Throws an error if the model is not found. + */ private verifiedModel(): ChatModel { const model = SupportedChatModels.find( (model) => model.name === this.model diff --git a/src/kc_shared/constants/supported.file.types.ts b/src/kc_shared/constants/supported.file.types.ts new file mode 100644 index 00000000..c12960b8 --- /dev/null +++ b/src/kc_shared/constants/supported.file.types.ts @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const codeMarkdownMap = { + ".c": "c", + ".cpp": "cpp", + ".cs": "csharp", + ".css": "css", + ".curl": "curl", + ".d": "d", + ".diff": "diff", + ".el": "", + ".erl": "erlang", + ".go": "go", + ".graphql": "graphql", + ".h": "c", + ".hpp": "cpp", + ".htm": "html", + ".html": "html", + ".java": "java", + ".jl": "julia", + ".js": "javascript", + ".json": "json", + ".jsx": "jsx", + ".kt": "kotlin", + ".liquid": "liquid", + ".m": "objc", + ".md": "markdown", + ".ml": "ocaml", + ".php": "php", + ".pl": "perl", + ".py": "python", + ".r": "r", + ".rake": "ruby", + ".rb": "ruby", + ".rs": "rust", + ".scss": "css", + ".sh": "shell", + ".sql": "sql", + ".swift": "swift", + ".ts": "typescript", + ".xml": "xml", + ".yaml": "yaml", + ".yml": "yaml", + Dockerfile: "dockerfile", +}; diff --git a/src/kc_shared/models/chat.model.ts b/src/kc_shared/models/chat.model.ts index 7fecc858..9d311f91 100644 --- a/src/kc_shared/models/chat.model.ts +++ b/src/kc_shared/models/chat.model.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Rob Royce + * Copyright (c) 2023-2024 Rob Royce * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,14 @@ * limitations under the License. */ +export type ChatModelName = "gpt-3.5-turbo" | "gpt-4" | "gpt-4o"; + /** * Used for user settings and for the back-end chat controller. */ export interface ChatModel { /* The name of the model to use for the API call (see OpenAI API) */ - name: "gpt-3.5-turbo" | "gpt-4"; + name: ChatModelName; /* The name of the model to display to the user */ label: string; @@ -66,25 +68,14 @@ const GPT3p5Turbo: ChatModel = { name: "gpt-3.5-turbo", label: "GPT-3.5 Turbo", description: "GPT-3.5 Turbo is optimized for dialogue.", - token_limit: 4096, - input_kilo_cost: 0.0015, - output_kilo_cost: 0.002, + token_limit: 16384, + input_kilo_cost: 0.0005, + output_kilo_cost: 0.0015, max_tokens_upper_bound: 2048, - max_tokens: 512, // Default + max_tokens: 256, // Default ...OpenAIDefaults, }; -/*TODO: not available in tiktoken yet - const GPT3p5Turbo16k: ChatModel = { - name: "gpt-3.5-turbo-16k", - label: "GPT-3.5 Turbo 16k", - description: - "GPT-3.5 Turbo is optimized for dialogue. This model has a larger token limit than GPT-3.5 Turbo, so you can enter more text at once.", - tokenLimit: 16384, - inputCostPer1k: 0.003, - outputCostPer1k: 0.004, -};*/ - const GPT4: ChatModel = { name: "gpt-4", label: "GPT-4", @@ -94,8 +85,21 @@ const GPT4: ChatModel = { input_kilo_cost: 0.03, output_kilo_cost: 0.06, max_tokens_upper_bound: 4096, + max_tokens: 512, + ...OpenAIDefaults, +}; + +const GPT4o: ChatModel = { + name: "gpt-4o", + label: "GPT-4o", + description: + "OpenAI's most advanced, multimodal flagship model that’s cheaper and faster than GPT-4 Turbo.", + token_limit: 128000, + input_kilo_cost: 0.005, + output_kilo_cost: 0.015, + max_tokens_upper_bound: 8192, max_tokens: 1024, ...OpenAIDefaults, }; -export const SupportedChatModels: ChatModel[] = [GPT3p5Turbo, GPT4]; +export const SupportedChatModels: ChatModel[] = [GPT3p5Turbo, GPT4, GPT4o]; diff --git a/src/kc_shared/models/settings.model.ts b/src/kc_shared/models/settings.model.ts index 3cc4b1d8..1a3d7278 100755 --- a/src/kc_shared/models/settings.model.ts +++ b/src/kc_shared/models/settings.model.ts @@ -241,19 +241,19 @@ export class FileManagerSettingsModel { } export class ChatSettingsModel { - suggestions: { - enabled: boolean; - onInput: boolean; + suggestions?: { + enabled?: boolean; + onInput?: boolean; } = { - enabled: true, - onInput: true, + enabled: false, + onInput: false, }; - display: { - introductions: boolean; - sourceMessages: boolean; + display?: { + introductions?: boolean; + sourceMessages?: boolean; } = { - introductions: true, + introductions: false, sourceMessages: true, }; diff --git a/src/kc_shared/models/uuid.model.ts b/src/kc_shared/models/uuid.model.ts index c8a08d31..00fca6f4 100755 --- a/src/kc_shared/models/uuid.model.ts +++ b/src/kc_shared/models/uuid.model.ts @@ -1,21 +1,21 @@ -/** - Copyright 2022 Rob Royce - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ export type UuidModel = { - value: string -} + value: string; +}; -export interface UUID extends UuidModel {} +export type UUID = UuidModel; diff --git a/src/kc_shared/models/web.source.model.ts b/src/kc_shared/models/web.source.model.ts index 4e372862..19fdb915 100755 --- a/src/kc_shared/models/web.source.model.ts +++ b/src/kc_shared/models/web.source.model.ts @@ -1,59 +1,55 @@ -/** - Copyright 2022 Rob Royce - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +/* + * Copyright (c) 2024 Rob Royce + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ -import {WebSourceMarkup} from "./markup.model"; +import { WebSourceMarkup } from "./markup.model"; export interface WebSourceModel { - accessLink: string, - iconUrl?: string, - topics?: string[], - metadata?: WebsiteMetadataModel, - markup?: WebSourceMarkup, - flagged?: boolean, - rawText?: string, - thumbnail?: string, - title?: string, - description?: string + accessLink: string; + iconUrl?: string; + topics?: string[]; + metadata?: WebsiteMetadataModel; + markup?: WebSourceMarkup; + flagged?: boolean; + rawText?: string; + thumbnail?: string; + title?: string; + description?: string; } export interface WebsiteMetadataModel { - title?: string, - meta?: WebsiteMetaTagsModel[], - icon?: any, - article?: ArticleModel, - code?: CodeModel[], - links?: string[] + title?: string; + meta?: WebsiteMetaTagsModel[]; + icon?: any; + article?: ArticleModel; + code?: CodeModel[]; + links?: string[]; } export interface WebsiteMetaTagsModel { - key?: string | null, - value?: string | null, - property?: string | null + key?: string | null; + value?: string | null; + property?: string | null; } export interface WebsiteContentModel { - title?: string, - type?: string, - text?: string, - html?: string + title?: string; + type?: string; + text?: string; + html?: string; } -export interface ArticleModel extends WebsiteContentModel { +export type ArticleModel = WebsiteContentModel; -} - -export interface CodeModel extends WebsiteContentModel { - -} +export type CodeModel = WebsiteContentModel; diff --git a/yarn.lock b/yarn.lock index 397d2145..15e81b9d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2117,10 +2117,10 @@ __metadata: languageName: node linkType: hard -"@dqbd/tiktoken@npm:^1.0.7": - version: 1.0.7 - resolution: "@dqbd/tiktoken@npm:1.0.7" - checksum: 81049797dcffa101c57b68a78be19b11958edf6e6cfffab7ccd3ec2e5290e3b27bc25f1060a1c34e1cb94f22e0f46d9aafa925d383fb4559738b644aa01c0fb1 +"@dqbd/tiktoken@npm:^1.0.15": + version: 1.0.15 + resolution: "@dqbd/tiktoken@npm:1.0.15" + checksum: e33259bb3fcb35acf8a897b92e83968a19d9de5cd24fc36068637dee703fe4516db4a7df2adc75411c6567e69d22f9d86b9b41e0c31ac711d4afc8d126e2d05f languageName: node linkType: hard @@ -2776,37 +2776,37 @@ __metadata: languageName: node linkType: hard -"@types/crypto-js@npm:^4.1.3": - version: 4.1.3 - resolution: "@types/crypto-js@npm:4.1.3" - checksum: 4489854579c1b90b77da881fc980e3f05ab1e9c68b18ae580fedbb7b0fb40d531fda7bdc05f1b3bd5d4218c1d97c2f821bbac57eb3d7d9e505ce5a2c5b034166 +"@types/crypto-js@npm:^4.2.2": + version: 4.2.2 + resolution: "@types/crypto-js@npm:4.2.2" + checksum: 727daa0d2db35f0abefbab865c23213b6ee6a270e27e177939bbe4b70d1e84c2202d9fac4ea84859c4b4d49a4ee50f948f601327a39b69ec013288018ba07ca5 languageName: node linkType: hard -"@types/cytoscape-dagre@npm:^2.3.2": - version: 2.3.2 - resolution: "@types/cytoscape-dagre@npm:2.3.2" +"@types/cytoscape-dagre@npm:^2.3.3": + version: 2.3.3 + resolution: "@types/cytoscape-dagre@npm:2.3.3" dependencies: "@types/cytoscape": "*" - checksum: ab88b8757de903acf2dd4a480b8be984d7450b7081f15cc0e380e84572f95c60ce54751d3fbf466aa6d8c4621d33b99bde18f01af2b687be78f1c47d73f1628f + checksum: ea3cdd17ad1d857215593d631397789232e46738e6bfa4bdb75c153df61c508bbd1f98b0b0cb429d9e615f727b58bc8ab0eaa1174e7dfcc4a6c360eb3692d4ff languageName: node linkType: hard -"@types/cytoscape-fcose@npm:^2.2.3": - version: 2.2.3 - resolution: "@types/cytoscape-fcose@npm:2.2.3" +"@types/cytoscape-fcose@npm:^2.2.4": + version: 2.2.4 + resolution: "@types/cytoscape-fcose@npm:2.2.4" dependencies: "@types/cytoscape": "*" - checksum: a962bc74a59c1d6865f7c3bd4d8dbb02d2847505576489fac139bb891338116494998b485065df05f3bf88a4500f6a942bff6b6521cf5290631e22e14201f9e2 + checksum: 9eae14075db4ef984701446966f9668be0f01c12a00aeacc0f614d6dfff7b8106448c5619047852aebbe456da78acd3cca0a2a9b45726716fb3d9547e13c063b languageName: node linkType: hard -"@types/cytoscape-klay@npm:^3.1.3": - version: 3.1.3 - resolution: "@types/cytoscape-klay@npm:3.1.3" +"@types/cytoscape-klay@npm:^3.1.4": + version: 3.1.4 + resolution: "@types/cytoscape-klay@npm:3.1.4" dependencies: "@types/cytoscape": "*" - checksum: 750958eb7b0847649c724e1a2b62be3942669351bd96af2f80f6e110a0a1abe94b399587ffd99ff392a0355ba64e87916a60220aaf149b79b3cf5bd2b81a02ae + checksum: 78177a9d5f571f2b7970d793c85eb1430fc91c8bf444382de4c193f3104c756d35d6533b5b8230756aaac73111866e2086f71943c24834af1f0f1a9f5fd80006 languageName: node linkType: hard @@ -2817,10 +2817,10 @@ __metadata: languageName: node linkType: hard -"@types/cytoscape@npm:^3.19.13": - version: 3.19.13 - resolution: "@types/cytoscape@npm:3.19.13" - checksum: 9716a8979c65b070d746624749184b35141ea5bb3173cac7685dec3fd721320230865d3fc215a163cd0eebd5f9ec02d075072e8ba0a4159f5927b81b658496fd +"@types/cytoscape@npm:^3.19.16": + version: 3.19.16 + resolution: "@types/cytoscape@npm:3.19.16" + checksum: 065c4b39f05888e65324bc517b6ac85adbfc4eb186ec9c9d5ff959a4a9628c0f9a99b60349c03600fcd6d0569e5a54980a8f18aa2b9ce6d85156da96c3095328 languageName: node linkType: hard @@ -2905,24 +2905,24 @@ __metadata: languageName: node linkType: hard -"@types/express@npm:^4.17.20": - version: 4.17.20 - resolution: "@types/express@npm:4.17.20" +"@types/express@npm:^4.17.21": + version: 4.17.21 + resolution: "@types/express@npm:4.17.21" dependencies: "@types/body-parser": "*" "@types/express-serve-static-core": ^4.17.33 "@types/qs": "*" "@types/serve-static": "*" - checksum: bf8a97d283128e5129f9ccabbeef728ff3f0484465e0ae74a304bd0588fa6cb715ae68845650caba9a641944b7791ba125d02ddbd47a7e62aaefdd036570c6c5 + checksum: fb238298630370a7392c7abdc80f495ae6c716723e114705d7e3fb67e3850b3859bbfd29391463a3fb8c0b32051847935933d99e719c0478710f8098ee7091c5 languageName: node linkType: hard -"@types/follow-redirects@npm:^1.14.3": - version: 1.14.3 - resolution: "@types/follow-redirects@npm:1.14.3" +"@types/follow-redirects@npm:^1.14.4": + version: 1.14.4 + resolution: "@types/follow-redirects@npm:1.14.4" dependencies: "@types/node": "*" - checksum: d93d893de5b4a9e6c601550ce866a14d125be94def10c3b59e332f59488381015c1ea0a941b795288bdb2a93996c333721c33b1e5023d3785b4f6012ff166542 + checksum: 8d5e4a7efa3602d26a2e0a2ff8970d29ba900c13874b29038faebc69adc89b0c78b34cd1100fac638c686848a52916c0310fb27c6e5300c0e19830e7f4f63094 languageName: node linkType: hard @@ -2935,10 +2935,10 @@ __metadata: languageName: node linkType: hard -"@types/html-to-text@npm:^9.0.3": - version: 9.0.3 - resolution: "@types/html-to-text@npm:9.0.3" - checksum: 9ca1a3927f1c08738ea395e1b9cd91af36808adf2916972330a0f8d1512d75e2951c21d8e324d0eb3588498965f58b5929bfcc1c6ff1d113fadf3d10eb7584de +"@types/html-to-text@npm:^9.0.4": + version: 9.0.4 + resolution: "@types/html-to-text@npm:9.0.4" + checksum: 58ab5564080febdf7a5db99e713bdcf1be8b1ca31c40dae76f8278eba6b1cf62a0291082abe1e00537350aaf1c5d64f92ff37b718c1f880bf5e6174fceb41b78 languageName: node linkType: hard @@ -2965,10 +2965,10 @@ __metadata: languageName: node linkType: hard -"@types/lodash@npm:^4.14.200": - version: 4.14.200 - resolution: "@types/lodash@npm:4.14.200" - checksum: 6471f8bb5da692a6ecf03a8da4935bfbc341e67ee9bcb4f5730bfacff0c367232548f0a01e8ac5ea18c6fe78fb085d502494e33ccb47a7ee87cbdee03b47d00d +"@types/lodash@npm:^4.14.202": + version: 4.14.202 + resolution: "@types/lodash@npm:4.14.202" + checksum: a91acf3564a568c6f199912f3eb2c76c99c5a0d7e219394294213b3f2d54f672619f0fde4da22b29dc5d4c31457cd799acc2e5cb6bd90f9af04a1578483b6ff7 languageName: node linkType: hard @@ -2979,10 +2979,10 @@ __metadata: languageName: node linkType: hard -"@types/mime-types@npm:^2.1.3": - version: 2.1.3 - resolution: "@types/mime-types@npm:2.1.3" - checksum: 2b39ed5de6ee607b3738a30c54879b439a8344e6f86e50af3b275cfb7165ed4d13e7f9a54971f1de4f759dd68929351741cf31c1c42e380503e53be6970bbf82 +"@types/mime-types@npm:^2.1.4": + version: 2.1.4 + resolution: "@types/mime-types@npm:2.1.4" + checksum: f8c521c54ee0c0b9f90a65356a80b1413ed27ccdc94f5c7ebb3de5d63cedb559cd2610ea55b4100805c7349606a920d96e54f2d16b2f0afa6b7cd5253967ccc9 languageName: node linkType: hard @@ -3007,10 +3007,10 @@ __metadata: languageName: node linkType: hard -"@types/minimist@npm:^1.2.4": - version: 1.2.4 - resolution: "@types/minimist@npm:1.2.4" - checksum: d7912f9a466312cbc1333800272b9208178140ef4da2ccec3fa82231c8e67f57f84275b3c19109c4f68f1b7b057baeacc6b80af1de14b58b46e6b54233e44c6a +"@types/minimist@npm:^1.2.5": + version: 1.2.5 + resolution: "@types/minimist@npm:1.2.5" + checksum: 477047b606005058ab0263c4f58097136268007f320003c348794f74adedc3166ffc47c80ec3e94687787f2ab7f4e72c468223946e79892cf0fd9e25e9970a90 languageName: node linkType: hard @@ -3021,12 +3021,12 @@ __metadata: languageName: node linkType: hard -"@types/multer@npm:^1.4.9": - version: 1.4.9 - resolution: "@types/multer@npm:1.4.9" +"@types/multer@npm:^1.4.11": + version: 1.4.11 + resolution: "@types/multer@npm:1.4.11" dependencies: "@types/express": "*" - checksum: 2e565052174389bf3bfdf77c0a8c7e9d4a44a65b05d388d9c1336db833fc3efc8d312587d3107b64e1e168b57d00f7834e093db0060b5acec8a7a02cd05b3877 + checksum: 3d80b2acdfbc9f3e9027d4467e948925810b67e5622a3017f42f58a3598d34b25376890801e55d0c03973ccc34573abf5218af334e8292ec455832f4ade3e5f5 languageName: node linkType: hard @@ -3040,12 +3040,12 @@ __metadata: languageName: node linkType: hard -"@types/node-forge@npm:^1.3.8": - version: 1.3.8 - resolution: "@types/node-forge@npm:1.3.8" +"@types/node-forge@npm:^1.3.11": + version: 1.3.11 + resolution: "@types/node-forge@npm:1.3.11" dependencies: "@types/node": "*" - checksum: fa75d22ef65671732f31965ea28618c52a1b4ef0fa44fa35110ba0fa375e35563e056c21545cf1b23a4b0b6546333e8885db003957b0cd8a45fecef16000ff63 + checksum: 1e86bd55b92a492eaafd75f6d01f31e7d86a5cdadd0c6bcdc0b1df4103b7f99bb75b832efd5217c7ddda5c781095dc086a868e20b9de00f5a427ddad4c296cd5 languageName: node linkType: hard @@ -3094,13 +3094,13 @@ __metadata: languageName: node linkType: hard -"@types/plist@npm:^3.0.4": - version: 3.0.4 - resolution: "@types/plist@npm:3.0.4" +"@types/plist@npm:^3.0.5": + version: 3.0.5 + resolution: "@types/plist@npm:3.0.5" dependencies: "@types/node": "*" xmlbuilder: ">=11.0.1" - checksum: fa5b60c231ff53ce69fda589eb2e982784400d6db9669eafb8a7d2ec990be7233204eef782d156f5d7ea1890ca76a2087d5fef3f0577c89794525d771603f609 + checksum: 71417189c9bc0d0cb4595106cea7c7a8a7274f64d2e9c4dd558efd7993bcfdada58be6917189e3be7c455fe4e5557004658fd13bd12254eafed8c56e0868b59e languageName: node linkType: hard @@ -3186,10 +3186,10 @@ __metadata: languageName: node linkType: hard -"@types/uuid@npm:^9.0.6": - version: 9.0.6 - resolution: "@types/uuid@npm:9.0.6" - checksum: 739dcb2e620ff097fa916edeab455eb75640c4883a850784fdb15b32f67b719e05275c6d6419bb6da11350d26bd14ed05ba5e992ff228411cdd98cbc772d71ef +"@types/uuid@npm:^9.0.7": + version: 9.0.7 + resolution: "@types/uuid@npm:9.0.7" + checksum: c7321194aeba9ea173efd1e721403bdf4e7ae6945f8f8cdbc87c791f4b505ccf3dbc4a8883d90b394ef13b7c2dc778045792b05dbb23b3c746f8ea347804d448 languageName: node linkType: hard @@ -4283,14 +4283,14 @@ __metadata: languageName: node linkType: hard -"axios@npm:^1.6.0": - version: 1.6.1 - resolution: "axios@npm:1.6.1" +"axios@npm:^1.6.5": + version: 1.6.5 + resolution: "axios@npm:1.6.5" dependencies: - follow-redirects: ^1.15.0 + follow-redirects: ^1.15.4 form-data: ^4.0.0 proxy-from-env: ^1.1.0 - checksum: 573f03f59b7487d54551b16f5e155d1d130ad4864ed32d1da93d522b78a57123b34e3bde37f822a65ee297e79f1db840f9ad6514addff50d3cbf5caeed39e8dc + checksum: e28d67b2d9134cb4608c44d8068b0678cfdccc652742e619006f27264a30c7aba13b2cd19c6f1f52ae195b5232734925928fb192d5c85feea7edd2f273df206d languageName: node linkType: hard @@ -5474,10 +5474,10 @@ __metadata: languageName: node linkType: hard -"crypto-js@npm:^4.1.1": - version: 4.1.1 - resolution: "crypto-js@npm:4.1.1" - checksum: b3747c12ee3a7632fab3b3e171ea50f78b182545f0714f6d3e7e2858385f0f4101a15f2517e033802ce9d12ba50a391575ff4638c9de3dd9b2c4bc47768d5425 +"crypto-js@npm:^4.2.0": + version: 4.2.0 + resolution: "crypto-js@npm:4.2.0" + checksum: f051666dbc077c8324777f44fbd3aaea2986f198fe85092535130d17026c7c2ccf2d23ee5b29b36f7a4a07312db2fae23c9094b644cc35f7858b1b4fcaf27774 languageName: node linkType: hard @@ -5623,13 +5623,13 @@ __metadata: languageName: node linkType: hard -"cytoscape@npm:^3.26.0": - version: 3.26.0 - resolution: "cytoscape@npm:3.26.0" +"cytoscape@npm:^3.28.1": + version: 3.28.1 + resolution: "cytoscape@npm:3.28.1" dependencies: heap: ^0.2.6 lodash: ^4.17.21 - checksum: c6a3b2fbe99009cde6e5e2eb6ff2e0469b37f3e12b0828d676aa3c25a7ce7b6ed689b6765e9d3975d51c718a7870e358c639c9128352813b95eb342ef64a26b8 + checksum: d9fcb14e909931ebead0fffaf230be7302156166372b7f0d446fea96bc5dfed2c36024842a8d80cd8817d6120125422beaee4a1e9ece4ba53a9addc880116afd languageName: node linkType: hard @@ -6132,10 +6132,10 @@ __metadata: languageName: node linkType: hard -"dotenv@npm:^16.3.1": - version: 16.3.1 - resolution: "dotenv@npm:16.3.1" - checksum: 15d75e7279018f4bafd0ee9706593dd14455ddb71b3bcba9c52574460b7ccaf67d5cf8b2c08a5af1a9da6db36c956a04a1192b101ee102a3e0cf8817bbcf3dfd +"dotenv@npm:^16.3.2": + version: 16.3.2 + resolution: "dotenv@npm:16.3.2" + checksum: 917b27eeb654b95846484009326b1c52af7a7c25f7b09e2939ed49de8f98cb9895dcc04f13a39cdb078d247985e21147311ccb5bfbf2fd151afb20fa8f96de15 languageName: node linkType: hard @@ -7441,7 +7441,7 @@ __metadata: languageName: node linkType: hard -"follow-redirects@npm:^1.0.0, follow-redirects@npm:^1.15.0": +"follow-redirects@npm:^1.0.0": version: 1.15.2 resolution: "follow-redirects@npm:1.15.2" peerDependenciesMeta: @@ -7451,7 +7451,7 @@ __metadata: languageName: node linkType: hard -"follow-redirects@npm:^1.15.4": +"follow-redirects@npm:^1.15.4, follow-redirects@npm:^1.15.5": version: 1.15.5 resolution: "follow-redirects@npm:1.15.5" peerDependenciesMeta: @@ -9156,16 +9156,16 @@ __metadata: "@fullcalendar/interaction": ^5.11.5 "@fullcalendar/list": ^5.11.5 "@fullcalendar/timegrid": ^5.11.5 - "@types/cytoscape": ^3.19.13 - "@types/cytoscape-dagre": ^2.3.2 - "@types/cytoscape-fcose": ^2.2.3 - "@types/cytoscape-klay": ^3.1.3 + "@types/cytoscape": ^3.19.16 + "@types/cytoscape-dagre": ^2.3.3 + "@types/cytoscape-fcose": ^2.2.4 + "@types/cytoscape-klay": ^3.1.4 "@types/marked": ^4.3.2 "@typescript-eslint/eslint-plugin": 5.37.0 "@typescript-eslint/parser": 5.37.0 chart.js: ^3.9.1 chokidar: ^3.5.3 - cytoscape: ^3.26.0 + cytoscape: ^3.28.1 cytoscape-cola: ^2.5.1 cytoscape-dagre: ^2.5.0 cytoscape-fcose: ^2.2.0 @@ -9183,7 +9183,7 @@ __metadata: rxjs: ~7.5.7 tslib: ^2.6.2 typescript: <4.9.0 - web-worker: ^1.2.0 + web-worker: ^1.3.0 zone.js: ^0.11.8 languageName: unknown linkType: soft @@ -9192,22 +9192,22 @@ __metadata: version: 0.0.0-use.local resolution: "kc_electron@workspace:src/kc_electron" dependencies: - "@dqbd/tiktoken": ^1.0.7 + "@dqbd/tiktoken": ^1.0.15 "@types/cors": ^2.8.17 - "@types/crypto-js": ^4.1.3 - "@types/lodash": ^4.14.200 - "@types/mime-types": ^2.1.3 + "@types/crypto-js": ^4.2.2 + "@types/lodash": ^4.14.202 + "@types/mime-types": ^2.1.4 chokidar: ^3.5.3 cors: ^2.8.5 - crypto-js: ^4.1.1 - dotenv: ^16.3.1 + crypto-js: ^4.2.0 + dotenv: ^16.3.2 electron: ^26.3.0 electron-builder: ^24.6.4 electron-updater: ^6.1.4 express-rate-limit: ^6.11.2 lodash: ^4.17.21 mime-types: ^2.1.35 - openai: ^4.12.4 + openai: ^4.25.0 rxjs: ^7.8.1 ts-loader: ^9.5.0 typescript: <4.9.0 @@ -9258,26 +9258,26 @@ __metadata: "@types/async": ^3.2.24 "@types/ejs": ^3.1.4 "@types/eslint": ^8 - "@types/express": ^4.17.20 - "@types/follow-redirects": ^1.14.3 - "@types/html-to-text": ^9.0.3 - "@types/minimist": ^1.2.4 - "@types/multer": ^1.4.9 + "@types/express": ^4.17.21 + "@types/follow-redirects": ^1.14.4 + "@types/html-to-text": ^9.0.4 + "@types/minimist": ^1.2.5 + "@types/multer": ^1.4.11 "@types/node": ^17.0.45 - "@types/node-forge": ^1.3.8 - "@types/plist": ^3.0.4 + "@types/node-forge": ^1.3.11 + "@types/plist": ^3.0.5 "@types/prettier": ^2.7.3 - "@types/uuid": ^9.0.6 + "@types/uuid": ^9.0.7 async: ^3.2.5 - axios: ^1.6.0 - ejs: ^3.1.9 + axios: ^1.6.5 canvas: ^2.11.2 + ejs: ^3.1.9 electron-builder: ^24.6.4 electron-notarize: ^1.2.2 eslint: ^8.38.0 eslint-plugin-html: ^7.1.0 express: ^4.18.2 - follow-redirects: ^1.15.4 + follow-redirects: ^1.15.5 got: ^13.0.0 highlight.js: ^11.9.0 html-to-text: ^9.0.5 @@ -9289,10 +9289,10 @@ __metadata: node-loader: ^2.0.0 pdfjs-dist: ^3.11.174 plist: ^3.1.0 - postcss: ^8.4.31 + postcss: ^8.4.33 prettier: ^2.8.8 standard-version: ^9.5.0 - terser: ^5.22.0 + terser: ^5.27.0 url: ^0.11.3 worker-loader: ^3.0.8 languageName: unknown @@ -10070,6 +10070,15 @@ __metadata: languageName: node linkType: hard +"nanoid@npm:^3.3.7": + version: 3.3.7 + resolution: "nanoid@npm:3.3.7" + bin: + nanoid: bin/nanoid.cjs + checksum: d36c427e530713e4ac6567d488b489a36582ef89da1d6d4e3b87eded11eb10d7042a877958c6f104929809b2ab0bafa17652b076cdf84324aa75b30b722204f2 + languageName: node + linkType: hard + "natural-compare@npm:^1.4.0": version: 1.4.0 resolution: "natural-compare@npm:1.4.0" @@ -10608,9 +10617,9 @@ __metadata: languageName: node linkType: hard -"openai@npm:^4.12.4": - version: 4.12.4 - resolution: "openai@npm:4.12.4" +"openai@npm:^4.25.0": + version: 4.25.0 + resolution: "openai@npm:4.25.0" dependencies: "@types/node": ^18.11.18 "@types/node-fetch": ^2.6.4 @@ -10623,7 +10632,7 @@ __metadata: web-streams-polyfill: ^3.2.1 bin: openai: bin/cli - checksum: d0de3d4785387f5dae61545ae753c39bb8fde72770d55b81137d0d5a8ed4d0abd308f4aa8e0ee755a3f64a66a0200f91d7ba4d7c9db10001b50c052e3ed46e29 + checksum: 16c91f008472b0718c13de3a1a945ca744688528b024c3be7e6e0a8eeec7a7c5a52122a719f0a76a44b294daf204a27e3a682599f5d2d33dfabeb770a7a9be5e languageName: node linkType: hard @@ -11592,7 +11601,7 @@ __metadata: languageName: node linkType: hard -"postcss@npm:8.4.31, postcss@npm:^8.2.14, postcss@npm:^8.3.7, postcss@npm:^8.4.31, postcss@npm:^8.4.7, postcss@npm:^8.4.8": +"postcss@npm:8.4.31, postcss@npm:^8.2.14, postcss@npm:^8.3.7, postcss@npm:^8.4.7, postcss@npm:^8.4.8": version: 8.4.31 resolution: "postcss@npm:8.4.31" dependencies: @@ -11603,6 +11612,17 @@ __metadata: languageName: node linkType: hard +"postcss@npm:^8.4.33": + version: 8.4.33 + resolution: "postcss@npm:8.4.33" + dependencies: + nanoid: ^3.3.7 + picocolors: ^1.0.0 + source-map-js: ^1.0.2 + checksum: 6f98b2af4b76632a3de20c4f47bf0e984a1ce1a531cf11adcb0b1d63a6cbda0aae4165e578b66c32ca4879038e3eaad386a6be725a8fb4429c78e3c1ab858fe9 + languageName: node + linkType: hard + "preact@npm:~10.12.1": version: 10.12.1 resolution: "preact@npm:10.12.1" @@ -13235,9 +13255,9 @@ __metadata: languageName: node linkType: hard -"terser@npm:^5.22.0": - version: 5.22.0 - resolution: "terser@npm:5.22.0" +"terser@npm:^5.27.0": + version: 5.27.0 + resolution: "terser@npm:5.27.0" dependencies: "@jridgewell/source-map": ^0.3.3 acorn: ^8.8.2 @@ -13245,7 +13265,7 @@ __metadata: source-map-support: ~0.5.20 bin: terser: bin/terser - checksum: ee95981c54ebd381e0b7f5872c646e7a05543e53960f8e0c2f240863c368989d43a3ca80b7e9f691683c92ba199eb4b91d61785fef0b9ca4a887eb55866001f4 + checksum: c165052cfea061e8512e9b9ba42a098c2ff6382886ae122b040fd5b6153443070cc2dcb4862269f1669c09c716763e856125a355ff984aa72be525d6fffd8729 languageName: node linkType: hard @@ -13814,10 +13834,10 @@ __metadata: languageName: node linkType: hard -"web-worker@npm:^1.2.0": - version: 1.2.0 - resolution: "web-worker@npm:1.2.0" - checksum: 1bb28348ddcf9b2e7c62c5fd02e49a84098795856cd905456de957271bba288e9618941cf69d8960f0a7ae81f5dfb74b427c0634be47ec69e3e955c4ec5213be +"web-worker@npm:^1.3.0": + version: 1.3.0 + resolution: "web-worker@npm:1.3.0" + checksum: ed1f869aefd1d81a43d0fbfe7b315a65beb6d7d2486b378c436a7047eed4216be34b2e6afca738b6fa95d016326b765f5f816355db33267dbf43b2b8a1837c0c languageName: node linkType: hard