From 7b904067b18be0c8913dcee05c268ac0943a9c85 Mon Sep 17 00:00:00 2001 From: Umesh Madan Date: Thu, 30 Jan 2025 17:03:30 -0800 Subject: [PATCH] Refactoring --- ts/examples/chat/src/memory/common.ts | 29 +++ ts/examples/chat/src/memory/knowproMemory.ts | 211 +----------------- ts/examples/chat/src/memory/knowproPrinter.ts | 187 ++++++++++++++++ 3 files changed, 227 insertions(+), 200 deletions(-) create mode 100644 ts/examples/chat/src/memory/knowproPrinter.ts diff --git a/ts/examples/chat/src/memory/common.ts b/ts/examples/chat/src/memory/common.ts index 703636bfb..64941b197 100644 --- a/ts/examples/chat/src/memory/common.ts +++ b/ts/examples/chat/src/memory/common.ts @@ -172,6 +172,35 @@ export function argChunkSize(defaultValue?: number | undefined): ArgDef { }; } +export function recordFromArgs( + args: NamedArgs, + metadata?: CommandMetadata, +): Record { + const record: Record = {}; + const keys = Object.keys(args); + for (const key of keys) { + const value = args[key]; + if (typeof value !== "function") { + record[key] = value; + } + } + if (metadata !== undefined) { + if (metadata.args) { + removeKeysFromRecord(record, Object.keys(metadata.args)); + } + if (metadata.options) { + removeKeysFromRecord(record, Object.keys(metadata.options)); + } + } + return record; +} + +function removeKeysFromRecord(record: Record, keys: string[]) { + for (const key of keys) { + delete record[key]; + } +} + export function argToDate(value: string | undefined): Date | undefined { return value ? dateTime.stringToDate(value) : undefined; } diff --git a/ts/examples/chat/src/memory/knowproMemory.ts b/ts/examples/chat/src/memory/knowproMemory.ts index 0b50bb0b7..b4964c3d1 100644 --- a/ts/examples/chat/src/memory/knowproMemory.ts +++ b/ts/examples/chat/src/memory/knowproMemory.ts @@ -16,16 +16,17 @@ import { import { ChatContext } from "./chatMemory.js"; import { ChatModel } from "aiclient"; import fs from "fs"; -import { ChatPrinter } from "../chatPrinter.js"; import { addFileNameSuffixToPath, argDestFile, argSourceFile, parseFreeAndNamedArguments, + recordFromArgs, } from "./common.js"; import { ensureDir, readJsonFile, writeJsonFile } from "typeagent"; import path from "path"; import chalk from "chalk"; +import { KnowProPrinter } from "./knowproPrinter.js"; type KnowProContext = { knowledgeModel: ChatModel; @@ -178,9 +179,10 @@ export async function createKnowproCommands( if (!conversation) { return; } + const commandDef = searchTermsDef(); let [termArgs, namedArgs] = parseFreeAndNamedArguments( args, - searchTermsDef(), + commandDef, ); const terms = parseQueryTerms(termArgs); // Todo: De dupe if (conversation.semanticRefIndex && conversation.semanticRefs) { @@ -192,7 +194,7 @@ export async function createKnowproCommands( const matches = await kp.searchConversation( conversation, terms, - filterFromArgs(namedArgs), + filterFromArgs(namedArgs, commandDef), ); if (matches === undefined || matches.size === 0) { context.printer.writeLine("No matches"); @@ -210,23 +212,12 @@ export async function createKnowproCommands( } } - function filterFromArgs(namedArgs: NamedArgs) { - let filter: kp.SearchFilter = { type: namedArgs.ktype }; - let argCopy = { ...namedArgs }; - delete argCopy.maxToDisplay; - delete argCopy.ktype; - let keys = Object.keys(argCopy); - if (keys.length > 0) { - for (const key of keys) { - const value = argCopy[key]; - if (typeof value === "function") { - delete argCopy[key]; - } - } - if (Object.keys(argCopy).length > 0) { - filter.propertiesToMatch = argCopy; - } - } + function filterFromArgs(namedArgs: NamedArgs, metadata: CommandMetadata) { + let filter: kp.SearchFilter = { + type: namedArgs.ktype, + propertiesToMatch: recordFromArgs(namedArgs, metadata), + }; + return filter; return filter; } @@ -349,174 +340,6 @@ export async function createKnowproCommands( } } -class KnowProPrinter extends ChatPrinter { - constructor() { - super(); - } - - public writeEntity( - entity: knowLib.conversation.ConcreteEntity | undefined, - ) { - if (entity !== undefined) { - this.writeLine(entity.name.toUpperCase()); - this.writeList(entity.type, { type: "csv" }); - if (entity.facets) { - const facetList = entity.facets.map((f) => - knowLib.conversation.facetToString(f), - ); - this.writeList(facetList, { type: "ul" }); - } - } - return this; - } - - public writeAction(action: knowLib.conversation.Action | undefined) { - if (action !== undefined) { - this.writeLine(knowLib.conversation.actionToString(action)); - } - } - - public writeTopic(topic: kp.ITopic | undefined) { - if (topic !== undefined) { - this.writeLine(topic.text); - } - } - - public writeSemanticRef(semanticRef: kp.SemanticRef) { - switch (semanticRef.knowledgeType) { - default: - this.writeLine(semanticRef.knowledgeType); - break; - case "entity": - this.writeEntity( - semanticRef.knowledge as knowLib.conversation.ConcreteEntity, - ); - break; - case "action": - this.writeAction( - semanticRef.knowledge as knowLib.conversation.Action, - ); - break; - case "topic": - this.writeTopic(semanticRef.knowledge as kp.ITopic); - break; - } - return this; - } - - public writeSemanticRefs(refs: kp.SemanticRef[] | undefined) { - if (refs && refs.length > 0) { - for (const ref of refs) { - this.writeSemanticRef(ref); - this.writeLine(); - } - } - return this; - } - - public writeScoredSemanticRefs( - semanticRefMatches: kp.ScoredSemanticRef[], - semanticRefs: kp.SemanticRef[], - maxToDisplay: number, - ) { - this.writeLine( - `Displaying ${maxToDisplay} matches of total ${semanticRefMatches.length}`, - ); - for (let i = 0; i < maxToDisplay; ++i) { - const match = semanticRefMatches[i]; - const semanticRef = semanticRefs[match.semanticRefIndex]; - - this.writeInColor( - chalk.green, - `#${i + 1}: ${semanticRef.knowledgeType} [${match.score}]`, - ); - this.writeSemanticRef(semanticRef); - this.writeLine(); - } - } - - public writeSearchResult( - conversation: kp.IConversation, - result: kp.SearchResult | undefined, - maxToDisplay: number, - ) { - if (result) { - this.writeListInColor(chalk.cyanBright, result.termMatches, { - title: "Matched terms", - type: "ol", - }); - maxToDisplay = Math.min( - result.semanticRefMatches.length, - maxToDisplay, - ); - this.writeScoredSemanticRefs( - result.semanticRefMatches, - conversation.semanticRefs!, - maxToDisplay, - ); - } - } - - public writeSearchResults( - conversation: kp.IConversation, - results: Map, - maxToDisplay: number, - ) { - // Do entities before actions... - this.writeResult(conversation, "entity", results, maxToDisplay); - this.writeResult(conversation, "action", results, maxToDisplay); - this.writeResult(conversation, "topic", results, maxToDisplay); - this.writeResult(conversation, "tag", results, maxToDisplay); - } - - private writeResult( - conversation: kp.IConversation, - type: kp.KnowledgeType, - results: Map, - maxToDisplay: number, - ) { - const result = results.get(type); - if (result !== undefined) { - this.writeTitle(type.toUpperCase()); - this.writeSearchResult(conversation, result, maxToDisplay); - } - } - - public writeConversationInfo(conversation: kp.IConversation) { - this.writeTitle(conversation.nameTag); - this.writeLine(`${conversation.messages.length} messages`); - return this; - } - - public writePodcastInfo(podcast: kp.Podcast) { - this.writeConversationInfo(podcast); - this.writeList(getPodcastParticipants(podcast), { - type: "csv", - title: "Participants", - }); - } - - public writeIndexingResults( - results: kp.ConversationIndexingResult, - verbose = false, - ) { - if (results.failedMessages.length > 0) { - this.writeError( - `Errors for ${results.failedMessages.length} messages`, - ); - if (verbose) { - for (const failedMessage of results.failedMessages) { - this.writeInColor( - chalk.cyan, - failedMessage.message.textChunks[0], - ); - this.writeError(failedMessage.error); - } - } - } - } -} - export function filterSemanticRefsByType( semanticRefs: kp.SemanticRef[] | undefined, type: string, @@ -532,18 +355,6 @@ export function filterSemanticRefsByType( return matches; } -export function getPodcastParticipants(podcast: kp.Podcast) { - const participants = new Set(); - for (let message of podcast.messages) { - const meta = message.metadata; - if (meta.speaker) { - participants.add(meta.speaker); - } - meta.listeners.forEach((l) => participants.add(l)); - } - return [...participants.values()]; -} - export function parseQueryTerms(args: string[]): kp.QueryTerm[] { const queryTerms: kp.QueryTerm[] = []; for (const arg of args) { diff --git a/ts/examples/chat/src/memory/knowproPrinter.ts b/ts/examples/chat/src/memory/knowproPrinter.ts new file mode 100644 index 000000000..1ee689af5 --- /dev/null +++ b/ts/examples/chat/src/memory/knowproPrinter.ts @@ -0,0 +1,187 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import * as kp from "knowpro"; +import * as knowLib from "knowledge-processor"; +import { ChatPrinter } from "../chatPrinter.js"; +import chalk from "chalk"; + +export class KnowProPrinter extends ChatPrinter { + constructor() { + super(); + } + + public writeEntity( + entity: knowLib.conversation.ConcreteEntity | undefined, + ) { + if (entity !== undefined) { + this.writeLine(entity.name.toUpperCase()); + this.writeList(entity.type, { type: "csv" }); + if (entity.facets) { + const facetList = entity.facets.map((f) => + knowLib.conversation.facetToString(f), + ); + this.writeList(facetList, { type: "ul" }); + } + } + return this; + } + + public writeAction(action: knowLib.conversation.Action | undefined) { + if (action !== undefined) { + this.writeLine(knowLib.conversation.actionToString(action)); + } + } + + public writeTopic(topic: kp.ITopic | undefined) { + if (topic !== undefined) { + this.writeLine(topic.text); + } + } + + public writeSemanticRef(semanticRef: kp.SemanticRef) { + switch (semanticRef.knowledgeType) { + default: + this.writeLine(semanticRef.knowledgeType); + break; + case "entity": + this.writeEntity( + semanticRef.knowledge as knowLib.conversation.ConcreteEntity, + ); + break; + case "action": + this.writeAction( + semanticRef.knowledge as knowLib.conversation.Action, + ); + break; + case "topic": + this.writeTopic(semanticRef.knowledge as kp.ITopic); + break; + } + return this; + } + + public writeSemanticRefs(refs: kp.SemanticRef[] | undefined) { + if (refs && refs.length > 0) { + for (const ref of refs) { + this.writeSemanticRef(ref); + this.writeLine(); + } + } + return this; + } + + public writeScoredSemanticRefs( + semanticRefMatches: kp.ScoredSemanticRef[], + semanticRefs: kp.SemanticRef[], + maxToDisplay: number, + ) { + this.writeLine( + `Displaying ${maxToDisplay} matches of total ${semanticRefMatches.length}`, + ); + for (let i = 0; i < maxToDisplay; ++i) { + const match = semanticRefMatches[i]; + const semanticRef = semanticRefs[match.semanticRefIndex]; + + this.writeInColor( + chalk.green, + `#${i + 1}: ${semanticRef.knowledgeType} [${match.score}]`, + ); + this.writeSemanticRef(semanticRef); + this.writeLine(); + } + } + + public writeSearchResult( + conversation: kp.IConversation, + result: kp.SearchResult | undefined, + maxToDisplay: number, + ) { + if (result) { + this.writeListInColor(chalk.cyanBright, result.termMatches, { + title: "Matched terms", + type: "ol", + }); + maxToDisplay = Math.min( + result.semanticRefMatches.length, + maxToDisplay, + ); + this.writeScoredSemanticRefs( + result.semanticRefMatches, + conversation.semanticRefs!, + maxToDisplay, + ); + } + } + + public writeSearchResults( + conversation: kp.IConversation, + results: Map, + maxToDisplay: number, + ) { + // Do entities before actions... + this.writeResult(conversation, "entity", results, maxToDisplay); + this.writeResult(conversation, "action", results, maxToDisplay); + this.writeResult(conversation, "topic", results, maxToDisplay); + this.writeResult(conversation, "tag", results, maxToDisplay); + } + + private writeResult( + conversation: kp.IConversation, + type: kp.KnowledgeType, + results: Map, + maxToDisplay: number, + ) { + const result = results.get(type); + if (result !== undefined) { + this.writeTitle(type.toUpperCase()); + this.writeSearchResult(conversation, result, maxToDisplay); + } + } + + public writeConversationInfo(conversation: kp.IConversation) { + this.writeTitle(conversation.nameTag); + this.writeLine(`${conversation.messages.length} messages`); + return this; + } + + public writePodcastInfo(podcast: kp.Podcast) { + this.writeConversationInfo(podcast); + this.writeList(getPodcastParticipants(podcast), { + type: "csv", + title: "Participants", + }); + } + + public writeIndexingResults( + results: kp.ConversationIndexingResult, + verbose = false, + ) { + if (results.failedMessages.length > 0) { + this.writeError( + `Errors for ${results.failedMessages.length} messages`, + ); + if (verbose) { + for (const failedMessage of results.failedMessages) { + this.writeInColor( + chalk.cyan, + failedMessage.message.textChunks[0], + ); + this.writeError(failedMessage.error); + } + } + } + } +} + +function getPodcastParticipants(podcast: kp.Podcast) { + const participants = new Set(); + for (let message of podcast.messages) { + const meta = message.metadata; + if (meta.speaker) { + participants.add(meta.speaker); + } + meta.listeners.forEach((l) => participants.add(l)); + } + return [...participants.values()]; +}