Skip to content

Commit

Permalink
Query builder
Browse files Browse the repository at this point in the history
  • Loading branch information
@umeshma
umeshma committed Feb 4, 2025
1 parent 7173714 commit 88ae942
Show file tree
Hide file tree
Showing 4 changed files with 84 additions and 61 deletions.
2 changes: 1 addition & 1 deletion ts/packages/knowPro/src/dataFormat.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ export type Term = {

export interface ITermToRelatedTermsIndex {
lookupTerm(termText: string): Term[] | undefined;
lookupTermsFuzzy(termText: string): Promise<Term[] | undefined>;
lookupTermFuzzy(termText: string): Promise<Term[] | undefined>;
serialize(): ITermsToRelatedTermsIndexData;
deserialize(data?: ITermsToRelatedTermsIndexData): void;
}
Expand Down
70 changes: 39 additions & 31 deletions ts/packages/knowPro/src/query.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,16 +293,16 @@ export class QueryOpExpr<T = void> implements IQueryOpExpr<T> {
}
}

export class SelectTopNExpr<T extends MatchAccumulator>
implements IQueryOpExpr<T>
{
export class SelectTopNExpr<T extends MatchAccumulator> extends QueryOpExpr<T> {
constructor(
public sourceExpr: IQueryOpExpr<T>,
public maxMatches: number | undefined = undefined,
public minHitCount: number | undefined = undefined,
) {}
) {
super();
}

public eval(context: QueryEvalContext): T {
public override eval(context: QueryEvalContext): T {
const matches = this.sourceExpr.eval(context);
matches.selectTopNScoring(this.maxMatches, this.minHitCount);
return matches;
Expand All @@ -316,7 +316,7 @@ export class MatchTermsExpr extends QueryOpExpr<SemanticRefAccumulator> {
super();
}

public eval(context: QueryEvalContext): SemanticRefAccumulator {
public override eval(context: QueryEvalContext): SemanticRefAccumulator {
const matchAccumulator: SemanticRefAccumulator =
new SemanticRefAccumulator();
const index = context.conversation.semanticRefIndex;
Expand All @@ -337,7 +337,7 @@ export class MatchSearchTermExpr extends QueryOpExpr<SemanticRefAccumulator> {
super();
}

public eval(context: QueryEvalContext): SemanticRefAccumulator {
public override eval(context: QueryEvalContext): SemanticRefAccumulator {
const matchAccumulator = new SemanticRefAccumulator();
const semanticRefIndex = context.conversation.semanticRefIndex;
if (semanticRefIndex) {
Expand All @@ -358,7 +358,9 @@ export class MatchQualifiedSearchTermExpr extends QueryOpExpr<
super();
}

public eval(context: QueryEvalContext): SemanticRefAccumulator | undefined {
public override eval(
context: QueryEvalContext,
): SemanticRefAccumulator | undefined {
if (!context.conversation.propertyToSemanticRefIndex) {
return undefined;
}
Expand Down Expand Up @@ -412,12 +414,14 @@ export class MatchQualifiedSearchTermExpr extends QueryOpExpr<
}
}

export class GroupByKnowledgeTypeExpr
implements IQueryOpExpr<Map<KnowledgeType, SemanticRefAccumulator>>
{
constructor(public matches: IQueryOpExpr<SemanticRefAccumulator>) {}
export class GroupByKnowledgeTypeExpr extends QueryOpExpr<
Map<KnowledgeType, SemanticRefAccumulator>
> {
constructor(public matches: IQueryOpExpr<SemanticRefAccumulator>) {
super();
}

public eval(
public override eval(
context: QueryEvalContext,
): Map<KnowledgeType, SemanticRefAccumulator> {
const semanticRefMatches = this.matches.eval(context);
Expand All @@ -427,18 +431,20 @@ export class GroupByKnowledgeTypeExpr
}
}

export class SelectTopNKnowledgeGroupExpr
implements IQueryOpExpr<Map<KnowledgeType, SemanticRefAccumulator>>
{
export class SelectTopNKnowledgeGroupExpr extends QueryOpExpr<
Map<KnowledgeType, SemanticRefAccumulator>
> {
constructor(
public sourceExpr: IQueryOpExpr<
Map<KnowledgeType, SemanticRefAccumulator>
>,
public maxMatches: number | undefined = undefined,
public minHitCount: number | undefined = undefined,
) {}
) {
super();
}

public eval(
public override eval(
context: QueryEvalContext,
): Map<KnowledgeType, SemanticRefAccumulator> {
const groupsAccumulators = this.sourceExpr.eval(context);
Expand All @@ -449,15 +455,15 @@ export class SelectTopNKnowledgeGroupExpr
}
}

export class WhereSemanticRefExpr
implements IQueryOpExpr<SemanticRefAccumulator>
{
export class WhereSemanticRefExpr extends QueryOpExpr<SemanticRefAccumulator> {
constructor(
public sourceExpr: IQueryOpExpr<SemanticRefAccumulator>,
public predicates: IQuerySemanticRefPredicate[],
) {}
) {
super();
}

public eval(context: QueryEvalContext): SemanticRefAccumulator {
public override eval(context: QueryEvalContext): SemanticRefAccumulator {
const accumulator = this.sourceExpr.eval(context);
const filtered = new SemanticRefAccumulator(
accumulator.searchTermMatches,
Expand Down Expand Up @@ -505,17 +511,19 @@ export class PropertyMatchPredicate implements IQuerySemanticRefPredicate {
}
}

export class ScopeExpr implements IQueryOpExpr<SemanticRefAccumulator> {
export class ScopeExpr extends QueryOpExpr<SemanticRefAccumulator> {
constructor(
public sourceExpr: IQueryOpExpr<SemanticRefAccumulator>,
// Predicates that look at matched semantic refs to determine what is in scope
public predicates: IQuerySemanticRefPredicate[],
public timeScopeExpr:
| IQueryOpExpr<TimestampedTextRange[]>
| undefined = undefined,
) {}
) {
super();
}

public eval(context: QueryEvalContext): SemanticRefAccumulator {
public override eval(context: QueryEvalContext): SemanticRefAccumulator {
let accumulator = this.sourceExpr.eval(context);
// Scope => text ranges in scope
const scope = new TextRangeCollection();
Expand Down Expand Up @@ -564,12 +572,12 @@ export class ScopeExpr implements IQueryOpExpr<SemanticRefAccumulator> {
}
}

export class TimestampScopeExpr
implements IQueryOpExpr<TimestampedTextRange[]>
{
constructor(public dateRange: DateRange) {}
export class TimestampScopeExpr extends QueryOpExpr<TimestampedTextRange[]> {
constructor(public dateRange: DateRange) {
super();
}

public eval(context: QueryEvalContext): TimestampedTextRange[] {
public override eval(context: QueryEvalContext): TimestampedTextRange[] {
const index = context.conversation.timestampIndex;
let ranges: TimestampedTextRange[] | undefined;
if (index !== undefined) {
Expand Down
20 changes: 19 additions & 1 deletion ts/packages/knowPro/src/relatedTermsIndex.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
ITermsToRelatedTermsIndexData,
} from "./dataFormat.js";
import { createEmbeddingCache } from "knowledge-processor";
import { SearchTerm } from "./search.js";

export class TermToRelatedTermsMap {
public map: collections.MultiMap<string, Term> = new collections.MultiMap();
Expand Down Expand Up @@ -69,6 +70,23 @@ export class TermToRelatedTermsMap {
}
}

export async function resolveRelatedTerms(
relatedTermsIndex: ITermToRelatedTermsIndex,
searchTerms: SearchTerm[],
): Promise<void> {
// Resolve any hardcoded mappings
for (const searchTerm of searchTerms) {
const termText = searchTerm.term.text;
if (!searchTerm.relatedTerms || searchTerm.relatedTerms.length === 0) {
searchTerm.relatedTerms = relatedTermsIndex.lookupTerm(termText);
}
if (!searchTerm.relatedTerms || searchTerm.relatedTerms.length === 0) {
searchTerm.relatedTerms =
await relatedTermsIndex.lookupTermFuzzy(termText);
}
}
}

export type TermsToRelatedTermIndexSettings = {
embeddingIndexSettings: TextEmbeddingIndexSettings;
};
Expand All @@ -85,7 +103,7 @@ export class TermToRelatedTermsIndex implements ITermToRelatedTermsIndex {
return this.termAliases.lookupTerm(termText);
}

public lookupTermsFuzzy(termText: string): Promise<Term[] | undefined> {
public lookupTermFuzzy(termText: string): Promise<Term[] | undefined> {
if (this.termEmbeddingsIndex) {
return this.termEmbeddingsIndex.lookupTermsFuzzy(termText);
}
Expand Down
53 changes: 25 additions & 28 deletions ts/packages/knowPro/src/search.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
} from "./dataFormat.js";
import { PropertyNames } from "./propertyIndex.js";
import * as q from "./query.js";
import { resolveRelatedTerms } from "./relatedTermsIndex.js";

export type SearchTerm = {
term: Term;
Expand Down Expand Up @@ -95,6 +96,8 @@ export async function searchConversation(
}

class SearchQueryBuilder {
private allSearchTerms: SearchTerm[] = [];

constructor(public conversation: IConversation) {}

public compile(
Expand All @@ -103,11 +106,10 @@ class SearchQueryBuilder {
filter?: SearchFilter,
maxMatches?: number,
) {
this.prepareTerms(terms, filter);

let select = this.compileSelect(terms, propertyTerms, filter);
let selectExpr = this.compileSelect(terms, propertyTerms, filter);
this.prepareSearchTerms(this.allSearchTerms);
const query = new q.SelectTopNKnowledgeGroupExpr(
new q.GroupByKnowledgeTypeExpr(select),
new q.GroupByKnowledgeTypeExpr(selectExpr),
maxMatches,
);
return query;
Expand Down Expand Up @@ -142,6 +144,7 @@ class SearchQueryBuilder {
const matchExpressions: q.MatchSearchTermExpr[] = [];
for (const searchTerm of searchTerms) {
matchExpressions.push(new q.MatchSearchTermExpr(searchTerm));
this.allSearchTerms.push(searchTerm);
}
return matchExpressions;
}
Expand All @@ -154,29 +157,32 @@ class SearchQueryBuilder {
for (const propertyName of Object.keys(properties)) {
const propertyValue = properties[propertyName];
let matchExpr: q.MatchQualifiedSearchTermExpr | undefined;
let searchTerm: QualifiedSearchTerm | undefined;
let qualifiedTerm: QualifiedSearchTerm | undefined;
switch (propertyName) {
default:
searchTerm = {
qualifiedTerm = {
type: "facet",
facetName: createSearchTerm(propertyName),
facetValue: createSearchTerm(propertyValue),
};
this.allSearchTerms.push(qualifiedTerm.facetName);
this.allSearchTerms.push(qualifiedTerm.facetValue);
break;
case PropertyNames.EntityName:
case PropertyNames.EntityType:
case PropertyNames.Verb:
case PropertyNames.Subject:
case PropertyNames.Object:
case PropertyNames.IndirectObject:
searchTerm = {
qualifiedTerm = {
type: "property",
propertyName: propertyName as KnowledgePropertyNames,
propertyValue: createSearchTerm(propertyValue),
};
this.allSearchTerms.push(qualifiedTerm.propertyValue);
break;
}
matchExpr = new q.MatchQualifiedSearchTermExpr(searchTerm);
matchExpr = new q.MatchQualifiedSearchTermExpr(qualifiedTerm);
matchExpressions.push(matchExpr);
}
return matchExpressions;
Expand Down Expand Up @@ -205,26 +211,17 @@ class SearchQueryBuilder {
return predicates;
}

private prepareTerms(
queryTerms: SearchTerm[],
filter?: SearchFilter,
): void {
const termText = new Set<string>();
termText.add("*");
let i = 0;
// Prepare terms and remove duplicates
while (i < queryTerms.length) {
const queryTerm = queryTerms[i];
this.prepareTerm(queryTerm.term);
if (termText.has(queryTerm.term.text)) {
// Duplicate
queryTerms.splice(i, 1);
} else {
if (queryTerm.relatedTerms !== undefined) {
queryTerm.relatedTerms.forEach((t) => this.prepareTerm(t));
}
termText.add(queryTerm.term.text);
++i;
private async prepareSearchTerms(searchTerms: SearchTerm[]): Promise<void> {
if (this.conversation.relatedTermsIndex) {
await resolveRelatedTerms(
this.conversation.relatedTermsIndex,
searchTerms,
);
}
for (const searchTerm of searchTerms) {
this.prepareTerm(searchTerm.term);
if (searchTerm.relatedTerms) {
searchTerm.relatedTerms.forEach((st) => this.prepareTerm(st));
}
}
}
Expand Down

0 comments on commit 88ae942

Please sign in to comment.