Skip to content

Commit

Permalink
fix #9 add benchmark; compare with match-sorter & fuse.js
Browse files Browse the repository at this point in the history
  • Loading branch information
thundermiracle committed Feb 27, 2020
1 parent 85e42b0 commit 7e814bb
Show file tree
Hide file tree
Showing 5 changed files with 207 additions and 7 deletions.
40 changes: 38 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,43 @@

## Description

filter and slice data array.
Simple but fast data filter.

## Benchmark

### _100 Records_(ms)

_*The results are little different in partial search._

| | match-sorter (4.0.2) | fuse.js (3.4.6) | filter-data (0.1.1) |
| :--- | :--: | :-: | :--: |
| match all, 1 key | 3.272ms | 3.207ms | <span style="color: green">1.172ms</span> |
| no match, 1 key | <span style="color: green">1.523ms</span> | 6.629ms | 1.706ms |
| match partial, 1 key | 2.483ms | 4.117ms | <span style="color: green">1.130ms</span> |
| match all, 2 keys | 3.907ms | 3.655ms | <span style="color: green">1.466ms</span> |
| no match, 2 keys | 1.921ms | 8.209ms | <span style="color: green">1.471ms</span> |
| match partial, 2 keys | 1.958ms | 5.830ms | <span style="color: green">1.394ms</span> |
| match all, 1 key, slice(0,10) | 3.324ms | 3.630ms | <span style="color: green">1.524ms</span> |
| no match, 1 key, slice(0,10) | 1.964ms | 4.944ms | <span style="color: green">1.162ms</span> |
| match partial, 1 key, slice(0,10) | 3.011ms | 4.838ms | <span style="color: green">1.863ms</span> |
| input empty | <span style="color: green">0.006ms</span> | 0.056ms | 0.109ms |

### _10000 Records_(ms)

_*The results are little different in partial search._

| | match-sorter (4.0.2) | fuse.js (3.4.6) | filter-data (0.1.1) |
| :--- | :--: | :-: | :--: |
| match all, 1 key | 111.281ms | 68.690ms | <span style="color: green">14.884ms</span> |
| no match, 1 key | 69.560ms | 63.553ms | <span style="color: green">12.245ms</span> |
| match partial, 1 key | 84.315ms | 83.006ms | <span style="color: green">17.690ms</span> |
| match all, 2 keys | 189.732ms | 86.417ms | <span style="color: green">32.801ms</span> |
| no match, 2 keys | 87.112ms | 91.501ms | <span style="color: green">13.870ms</span> |
| match partial, 2 keys | 92.896ms | 123.191ms | <span style="color: green">44.515ms</span> |
| match all, 1 key, slice(0,10) | 137.180ms | 81.718ms | <span style="color: green">0.204ms</span> |
| no match, 1 key, slice(0,10) | 68.765ms | 63.469ms | <span style="color: green">18.511ms</span> |
| match partial, 1 key, slice(0,10) | 82.715ms | 91.082ms | <span style="color: green">0.393ms</span> |
| input empty | <span style="color: green">0.007ms</span> | 4.665ms | 0.375ms |

## Installation

Expand Down Expand Up @@ -127,7 +163,7 @@ npm install --save filter-data
|:---|:-------------:|:---------:|:--------------|:-----------|
| 1 | data || | array of object for filtering |
| 2 | searchConditions || | array of searchCondition; ```{ key: 'search column', value: 'search value', type: 'search type' }``` |
| 3 | options | | ```{ caseSensitive: false, includeNull: false, offset: undefined, limit: undefined }``` | |
| 3 | options | | ```{ caseSensitive: false, includeNull: false, offset: undefined, limit: undefined }``` | includeNull: include data even `key` is not exist or `value` is null |

### SearchType

Expand Down
91 changes: 91 additions & 0 deletions benchmark/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
import matchSorter from 'match-sorter';
import Fuse from 'fuse.js';
import { filterData, SearchType } from '../src';

interface Dummy {
value: string;
label: string;
group: string;
}

function makeDummy(count: number): Dummy[] {
return Array.from({ length: count }).map((_, ind) => ({
value: `value_${ind}`,
label: `label_${ind}`,
group: ind % 2 === 0 ? 'group1' : 'group2',
}));
}

function bench(
count: number,
inputValue: string,
keys: string[],
prefixStr = '',
sliceCount?: number,
): void {
const data = makeDummy(count);

console.time(`[${prefixStr}]matchSorter`);
let msResult = matchSorter(data, inputValue, { keys });
if (sliceCount) msResult = msResult.slice(0, sliceCount);
console.timeEnd(`[${prefixStr}]matchSorter`);

console.time(`[${prefixStr}]fuse.js`);
const fuse = new Fuse(data, { keys });
let fuseResult = fuse.search(inputValue);
if (sliceCount) fuseResult = fuseResult.slice(0, sliceCount);
console.timeEnd(`[${prefixStr}]fuse.js`);

console.time(`[${prefixStr}]filter-data`);
const searchConditions = keys.map(key => ({
key,
value: inputValue,
type: SearchType.LK,
}));
const fdResult = filterData(data, searchConditions, {
includeNull: true,
offset: 0,
limit: sliceCount,
});
console.timeEnd(`[${prefixStr}]filter-data`);

console.log(msResult.length, fuseResult.length, fdResult.length);
}

function benchArr(
counts: number[],
inputValue: string,
keys: string[],
prefixStr: string,
sliceCount?: number,
): void {
counts.forEach(count => {
bench(count, inputValue, keys, `${prefixStr}--${count}`, sliceCount);
});
console.log('');
}

benchArr([100, 10000], 'lue', ['value'], 'AllMatch--Key:value');

benchArr([100, 10000], 'nono', ['value'], 'NoMatch--Key:value');

benchArr([100, 10000], '10', ['value'], 'PartialMatch--Key:value');

benchArr([100, 10000], 'l', ['value', 'label'], 'AllMatch--Key:value,label');

benchArr([100, 10000], 'nono', ['value', 'label'], 'NoMatch--Key:value,label');

benchArr(
[100, 10000],
'10',
['value', 'label'],
'PartialMatch--Key:value,label',
);

benchArr([100, 10000], 'lue', ['value'], 'Slice10-AllMatch--Key:value', 10);

benchArr([100, 10000], 'nono', ['value'], 'Slice10-NoMatch--Key:value', 10);

benchArr([100, 10000], '10', ['value'], 'Slice10-PartialMatch--Key:value', 10);

benchArr([100, 10000], '', ['value'], 'InputEmpty-NoMatch--Key:value');
20 changes: 16 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,8 +1,15 @@
{
"name": "filter-data",
"version": "0.1.0",
"description": "filter, sort data by transducer for datatable",
"keywords": [],
"version": "0.1.1",
"description": "simple, fast data filter",
"keywords": [
"match-sorter",
"data filter",
"filter data",
"search data",
"datatable",
"fast search"
],
"author": "ThunderMiracle",
"license": "MIT",
"main": "lib/index.js",
Expand All @@ -26,6 +33,7 @@
"type-check": "tsc --noEmit --project tsconfig.eslint.json",
"type-check:watch": "yarn type-check -- --watch",
"test": "yarn type-check && yarn eslint && yarn jest && codecov",
"bench": "ts-node ./benchmark/index.ts",
"build:js:module": "tsc",
"build:js:es": "tsc --project tsconfig.es.json",
"build:js": "yarn build:js:module && yarn build:js:es",
Expand All @@ -39,9 +47,10 @@
"ramda": "0.26.1"
},
"devDependencies": {
"@babel/types": "^7.8.3",
"@babel/core": "^7.8.4",
"@babel/types": "^7.8.3",
"@types/jest": "^25.1.3",
"@types/match-sorter": "^4.0.0",
"@types/node": "^13.7.6",
"@types/ramda": "^0.26.42",
"@typescript-eslint/eslint-plugin": "^2.21.0",
Expand All @@ -56,11 +65,14 @@
"eslint-plugin-jest": "^23.8.0",
"eslint-plugin-jsx-a11y": "^6.2.3",
"eslint-plugin-prettier": "^3.1.2",
"fuse.js": "^3.4.6",
"jest": "^25.1.0",
"match-sorter": "^4.0.2",
"prettier": "^1.19.1",
"rimraf": "^3.0.2",
"ts-jest": "^25.2.1",
"ts-loader": "^6.2.1",
"ts-node": "^8.6.2",
"typescript": "^3.8.2",
"webpack": "^4.41.6",
"webpack-cli": "^3.3.11"
Expand Down
2 changes: 1 addition & 1 deletion tsconfig.eslint.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{
"extends": "./tsconfig.json",
"include": ["src/**/*", "__test__/**/*", "./*.ts"]
"include": ["src/**/*", "__test__/**/*", "benchmark/**/*", "./*.ts"]
}
61 changes: 61 additions & 0 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,13 @@
dependencies:
regenerator-runtime "^0.13.2"

"@babel/runtime@^7.5.5":
version "7.8.4"
resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.8.4.tgz#d79f5a2040f7caa24d53e563aad49cbc05581308"
integrity sha512-neAp3zt80trRVBI1x0azq6c57aNBqYZH8KhMm3TaB7wEI5Q4A2SHfBHE8w9gOhI/lrqxtEbXZgQIrHP+wvSGwQ==
dependencies:
regenerator-runtime "^0.13.2"

"@babel/template@^7.7.4":
version "7.7.4"
resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.7.4.tgz#428a7d9eecffe27deac0a98e23bf8e3675d2a77b"
Expand Down Expand Up @@ -496,6 +503,11 @@
version "0.0.29"
resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"

"@types/match-sorter@^4.0.0":
version "4.0.0"
resolved "https://registry.yarnpkg.com/@types/match-sorter/-/match-sorter-4.0.0.tgz#8a7286019d4e328c09422bb2af2403a94b7038fd"
integrity sha512-JK7HNHXZA7i/nEp6fbNAxoX/1j1ysZXmv2/nlkt2UpX1LiUWKLtyt/dMmDTlMPR6t6PkwMmIr2W2AAyu6oELNw==

"@types/node@^13.7.6":
version "13.7.6"
resolved "https://registry.yarnpkg.com/@types/node/-/node-13.7.6.tgz#cb734a7c191472ae6a2b3a502b4dfffcea974113"
Expand Down Expand Up @@ -833,6 +845,11 @@ are-we-there-yet@~1.1.2:
delegates "^1.0.0"
readable-stream "^2.0.6"

arg@^4.1.0:
version "4.1.3"
resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089"
integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==

argparse@^1.0.7:
version "1.0.10"
resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
Expand Down Expand Up @@ -1615,6 +1632,11 @@ diff-sequences@^25.1.0:
version "25.1.0"
resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-25.1.0.tgz#fd29a46f1c913fd66c22645dc75bffbe43051f32"

diff@^4.0.1:
version "4.0.2"
resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==

diffie-hellman@^5.0.0:
version "5.0.3"
resolved "https://registry.yarnpkg.com/diffie-hellman/-/diffie-hellman-5.0.3.tgz#40e8ee98f55a2149607146921c63e1ae5f3d2875"
Expand Down Expand Up @@ -2253,6 +2275,11 @@ functional-red-black-tree@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327"

fuse.js@^3.4.6:
version "3.4.6"
resolved "https://registry.yarnpkg.com/fuse.js/-/fuse.js-3.4.6.tgz#545c3411fed88bf2e27c457cab6e73e7af697a45"
integrity sha512-H6aJY4UpLFwxj1+5nAvufom5b2BT2v45P1MkPvdGIK8fWjQx/7o6tTT1+ALV0yawQvbmvCF0ufl2et8eJ7v7Cg==

gauge@~2.7.3:
version "2.7.4"
resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7"
Expand Down Expand Up @@ -3405,6 +3432,11 @@ [email protected]:
version "1.3.5"
resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.5.tgz#efe4e81f6db28cadd605c70f29c831b58ef776c8"

make-error@^1.1.1:
version "1.3.6"
resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2"
integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==

[email protected]:
version "1.0.11"
resolved "https://registry.yarnpkg.com/makeerror/-/makeerror-1.0.11.tgz#e01a5c9109f2af79660e4e8b9587790184f5a96c"
Expand All @@ -3431,6 +3463,14 @@ map-visit@^1.0.0:
dependencies:
object-visit "^1.0.0"

match-sorter@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/match-sorter/-/match-sorter-4.0.2.tgz#1b451e8bc7112e5a41ebe3a977b7e19f167c3215"
integrity sha512-5EcCLEmPgfvq2hg1DAgAG7zqqS9bnZmRXzLR3md0xRi3Q1oGnnze6HuY+4bDRtm+X2lTsVZL8oG9FOkALFT4vw==
dependencies:
"@babel/runtime" "^7.5.5"
remove-accents "0.4.2"

md5.js@^1.3.4:
version "1.3.5"
resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f"
Expand Down Expand Up @@ -4300,6 +4340,11 @@ regexpp@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.0.0.tgz#dd63982ee3300e67b41c1956f850aa680d9d330e"

[email protected]:
version "0.4.2"
resolved "https://registry.yarnpkg.com/remove-accents/-/remove-accents-0.4.2.tgz#0a43d3aaae1e80db919e07ae254b285d9e1c7bb5"
integrity sha1-CkPTqq4egNuRngeuJUsoXZ4ce7U=

remove-trailing-separator@^1.0.1:
version "1.1.0"
resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef"
Expand Down Expand Up @@ -5122,6 +5167,17 @@ ts-loader@^6.2.1:
micromatch "^4.0.0"
semver "^6.0.0"

ts-node@^8.6.2:
version "8.6.2"
resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-8.6.2.tgz#7419a01391a818fbafa6f826a33c1a13e9464e35"
integrity sha512-4mZEbofxGqLL2RImpe3zMJukvEvcO1XP8bj8ozBPySdCUXEcU5cIRwR0aM3R+VoZq7iXc8N86NC0FspGRqP4gg==
dependencies:
arg "^4.1.0"
diff "^4.0.1"
make-error "^1.1.1"
source-map-support "^0.5.6"
yn "3.1.1"

ts-toolbelt@^6.1.11:
version "6.3.3"
resolved "https://registry.yarnpkg.com/ts-toolbelt/-/ts-toolbelt-6.3.3.tgz#973a409c94ba129382facd67b35333de7c09ff86"
Expand Down Expand Up @@ -5543,3 +5599,8 @@ yargs@^15.0.0:
which-module "^2.0.0"
y18n "^4.0.0"
yargs-parser "^16.1.0"

[email protected]:
version "3.1.1"
resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"
integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==

0 comments on commit 7e814bb

Please sign in to comment.