Skip to content

Commit

Permalink
property search
Browse files Browse the repository at this point in the history
  • Loading branch information
@umeshma
umeshma committed Feb 4, 2025
1 parent 88ae942 commit e553cab
Show file tree
Hide file tree
Showing 3 changed files with 145 additions and 80 deletions.
10 changes: 8 additions & 2 deletions ts/packages/knowPro/src/collections.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,10 @@ export class SemanticRefAccumulator extends MatchAccumulator<SemanticRefIndex> {

public addSearchTermMatch(
searchTerm: Term,
semanticRefs: ScoredSemanticRef[] | undefined,
semanticRefs:
| ScoredSemanticRef[]
| IterableIterator<ScoredSemanticRef>
| undefined,
scoreBoost?: number,
) {
if (semanticRefs) {
Expand All @@ -197,7 +200,10 @@ export class SemanticRefAccumulator extends MatchAccumulator<SemanticRefIndex> {
public addRelatedTermMatch(
searchTerm: Term,
relatedTerm: Term,
semanticRefs: ScoredSemanticRef[] | undefined,
semanticRefs:
| ScoredSemanticRef[]
| IterableIterator<ScoredSemanticRef>
| undefined,
scoreBoost?: number,
) {
if (semanticRefs) {
Expand Down
132 changes: 92 additions & 40 deletions ts/packages/knowPro/src/query.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import {
IPropertyToSemanticRefIndex,
ITermToSemanticRefIndex,
KnowledgeType,
ScoredSemanticRef,
SemanticRef,
SemanticRefIndex,
Term,
Expand Down Expand Up @@ -148,16 +149,19 @@ export function textRangeForMessage(
};
}

export function lookupSearchTermInIndex(
export function lookupSearchTermInSemanticRefIndex(
semanticRefIndex: ITermToSemanticRefIndex,
searchTerm: SearchTerm,
predicate?: (scoredRef: ScoredSemanticRef) => boolean,
matchAccumulator?: SemanticRefAccumulator,
): SemanticRefAccumulator {
matchAccumulator ??= new SemanticRefAccumulator();
// Lookup search term
matchAccumulator.addSearchTermMatch(
searchTerm.term,
semanticRefIndex.lookupTerm(searchTerm.term.text),
predicate
? lookupAndFilter(semanticRefIndex, searchTerm.term.text, predicate)
: semanticRefIndex.lookupTerm(searchTerm.term.text),
);
// And any related terms
if (searchTerm.relatedTerms && searchTerm.relatedTerms.length > 0) {
Expand All @@ -167,12 +171,33 @@ export function lookupSearchTermInIndex(
matchAccumulator.addRelatedTermMatch(
searchTerm.term,
relatedTerm,
semanticRefIndex.lookupTerm(relatedTerm.text),
predicate
? lookupAndFilter(
semanticRefIndex,
relatedTerm.text,
predicate,
)
: semanticRefIndex.lookupTerm(relatedTerm.text),
relatedTerm.score,
);
}
}
return matchAccumulator;

function* lookupAndFilter(
semanticRefIndex: ITermToSemanticRefIndex,
text: string,
predicate: (scoredRef: ScoredSemanticRef) => boolean,
) {
const scoredRefs = semanticRefIndex.lookupTerm(text);
if (scoredRefs) {
for (const scoredRef of scoredRefs) {
if (predicate(scoredRef)) {
yield scoredRef;
}
}
}
}
}

export function lookupSearchTermInPropertyIndex(
Expand Down Expand Up @@ -277,6 +302,14 @@ export class QueryEvalContext {
}
}

public get semanticRefIndex() {
return this.conversation.semanticRefIndex!;
}

public get propertyIndex() {
return this.conversation.propertyToSemanticRefIndex;
}

public getSemanticRef(semanticRefIndex: SemanticRefIndex): SemanticRef {
return this.conversation.semanticRefs![semanticRefIndex];
}
Expand Down Expand Up @@ -338,23 +371,17 @@ export class MatchSearchTermExpr extends QueryOpExpr<SemanticRefAccumulator> {
}

public override eval(context: QueryEvalContext): SemanticRefAccumulator {
const matchAccumulator = new SemanticRefAccumulator();
const semanticRefIndex = context.conversation.semanticRefIndex;
if (semanticRefIndex) {
lookupSearchTermInIndex(
semanticRefIndex,
this.searchTerm,
matchAccumulator,
);
}
return matchAccumulator;
return lookupSearchTermInSemanticRefIndex(
context.semanticRefIndex,
this.searchTerm,
);
}
}

export class MatchQualifiedSearchTermExpr extends QueryOpExpr<
SemanticRefAccumulator | undefined
> {
constructor(public searchTerm: QualifiedSearchTerm) {
constructor(public qualifiedSearchTerm: QualifiedSearchTerm) {
super();
}

Expand All @@ -365,52 +392,77 @@ export class MatchQualifiedSearchTermExpr extends QueryOpExpr<
return undefined;
}
let matches: SemanticRefAccumulator | undefined;
if (this.searchTerm.type === "property") {
matches = this.matchProperty(context, this.searchTerm);
if (this.qualifiedSearchTerm.type === "property") {
matches = this.matchProperty(context, this.qualifiedSearchTerm);
} else {
matches = this.matchFacet(context, this.searchTerm);
matches = this.matchFacet(context, this.qualifiedSearchTerm);
}
return matches;
}

private matchProperty(
context: QueryEvalContext,
searchTerm: PropertySearchTerm,
propertySearchTerm: PropertySearchTerm,
): SemanticRefAccumulator | undefined {
const propertyIndex = context.conversation.propertyToSemanticRefIndex!;
return lookupSearchTermInPropertyIndex(
propertyIndex,
searchTerm.propertyName,
if (propertySearchTerm.propertyName === "tag") {
return this.matchTag(context, propertySearchTerm);
}
const propertyIndex = context.propertyIndex;
if (propertyIndex) {
return lookupSearchTermInPropertyIndex(
propertyIndex,
propertySearchTerm.propertyName,
propertySearchTerm.propertyValue,
);
}
return undefined;
}

private matchTag(
context: QueryEvalContext,
searchTerm: PropertySearchTerm,
) {
return lookupSearchTermInSemanticRefIndex(
context.semanticRefIndex,
searchTerm.propertyValue,
(scoredRef) => {
return (
context.getSemanticRef(scoredRef.semanticRefIndex)
.knowledgeType === "tag"
);
},
);
}

private matchFacet(
context: QueryEvalContext,
facetSearchTerm: FacetSearchTerm,
): SemanticRefAccumulator | undefined {
const propertyIndex = context.conversation.propertyToSemanticRefIndex!;
let facetMatches = lookupSearchTermInPropertyIndex(
propertyIndex,
PropertyNames.FacetName,
facetSearchTerm.facetName,
);
if (
facetMatches.size > 0 &&
facetSearchTerm.facetValue &&
!isSearchTermWildcard(facetSearchTerm.facetValue)
) {
let valueMatches = lookupSearchTermInPropertyIndex(
const propertyIndex = context.propertyIndex;
if (propertyIndex) {
let facetMatches = lookupSearchTermInPropertyIndex(
propertyIndex,
PropertyNames.FacetValue,
facetSearchTerm.facetValue,
PropertyNames.FacetName,
facetSearchTerm.facetName,
);
if (valueMatches.size > 0) {
facetMatches = facetMatches.intersect(valueMatches);
if (
facetMatches.size > 0 &&
facetSearchTerm.facetValue &&
!isSearchTermWildcard(facetSearchTerm.facetValue)
) {
let valueMatches = lookupSearchTermInPropertyIndex(
propertyIndex,
PropertyNames.FacetValue,
facetSearchTerm.facetValue,
);
if (valueMatches.size > 0) {
facetMatches = facetMatches.intersect(valueMatches);
}
}
}

return facetMatches;
return facetMatches;
}
return undefined;
}
}

Expand Down
83 changes: 45 additions & 38 deletions ts/packages/knowPro/src/search.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import {
ScoredSemanticRef,
Term,
} from "./dataFormat.js";
import { PropertyNames } from "./propertyIndex.js";
import * as q from "./query.js";
import { resolveRelatedTerms } from "./relatedTermsIndex.js";

Expand All @@ -32,30 +31,26 @@ export function createSearchTerm(text: string, score?: number): SearchTerm {

export type QualifiedSearchTerm = PropertySearchTerm | FacetSearchTerm;

export type KnowledgePropertyNames =
export type KnowledgePropertyName =
| "name"
| "type"
| "verb"
| "subject"
| "object"
| "indirectObject";
| "indirectObject"
| "tag";

export interface PropertySearchTerm {
export type PropertySearchTerm = {
type: "property";
propertyName: KnowledgePropertyNames;
propertyName: KnowledgePropertyName;
propertyValue: SearchTerm;
}
};

export interface FacetSearchTerm {
export type FacetSearchTerm = {
type: "facet";
facetName: SearchTerm;
facetValue: SearchTerm;
}

export interface TagSearchTerm {
type: "tag";
tagValue: SearchTerm;
}
};

export type SearchResult = {
termMatches: Set<string>;
Expand Down Expand Up @@ -157,31 +152,9 @@ class SearchQueryBuilder {
for (const propertyName of Object.keys(properties)) {
const propertyValue = properties[propertyName];
let matchExpr: q.MatchQualifiedSearchTermExpr | undefined;
let qualifiedTerm: QualifiedSearchTerm | undefined;
switch (propertyName) {
default:
qualifiedTerm = {
type: "facet",
facetName: createSearchTerm(propertyName),
facetValue: createSearchTerm(propertyValue),
};
this.allSearchTerms.push(qualifiedTerm.facetName);
this.allSearchTerms.push(qualifiedTerm.facetValue);
break;
case PropertyNames.EntityName:
case PropertyNames.EntityType:
case PropertyNames.Verb:
case PropertyNames.Subject:
case PropertyNames.Object:
case PropertyNames.IndirectObject:
qualifiedTerm = {
type: "property",
propertyName: propertyName as KnowledgePropertyNames,
propertyValue: createSearchTerm(propertyValue),
};
this.allSearchTerms.push(qualifiedTerm.propertyValue);
break;
}
const [qualifiedTerm, searchTermsCreated] =
qualifiedSearchTermFromKeyValue(propertyName, propertyValue);
this.allSearchTerms.push(...searchTermsCreated);
matchExpr = new q.MatchQualifiedSearchTermExpr(qualifiedTerm);
matchExpressions.push(matchExpr);
}
Expand Down Expand Up @@ -231,6 +204,40 @@ class SearchQueryBuilder {
}
}

export function qualifiedSearchTermFromKeyValue(
key: string,
value: string,
): [QualifiedSearchTerm, SearchTerm[]] {
let qualifiedSearchTerm: QualifiedSearchTerm | undefined;
const searchTermsCreated: SearchTerm[] = [];
switch (key) {
default:
qualifiedSearchTerm = {
type: "facet",
facetName: createSearchTerm(key),
facetValue: createSearchTerm(value),
};
searchTermsCreated.push(qualifiedSearchTerm.facetName);
searchTermsCreated.push(qualifiedSearchTerm.facetValue);
break;
case "name":
case "type":
case "verb":
case "subject":
case "object":
case "indirectObject":
case "tag":
qualifiedSearchTerm = {
type: "property",
propertyName: key as KnowledgePropertyName,
propertyValue: createSearchTerm(value),
};
searchTermsCreated.push(qualifiedSearchTerm.propertyValue);
break;
}
return [qualifiedSearchTerm, searchTermsCreated];
}

function toGroupedSearchResults(
evalResults: Map<KnowledgeType, SemanticRefAccumulator>,
) {
Expand Down

0 comments on commit e553cab

Please sign in to comment.