dev: Work towards showing compound word seperators (#2386)

he goal is to improve suggestions. * Support showing word compounds. * Adjust linting * refactor: move walker under suggestions. * Show separators in suggestions
streetsidesoftware · Feb 2, 2022 · 377f9d0 · 377f9d0
1 parent e5b7ed5
commit 377f9d0
Show file tree

Hide file tree

Showing 21 changed files with 130 additions and 48 deletions.
diff --git a/.eslintrc.js b/.eslintrc.js
@@ -38,8 +38,7 @@ const config = {
             parser: '@typescript-eslint/parser',
             plugins: ['@typescript-eslint'],
             rules: {
-                'no-unused-vars': 0, // off - caught by the compiler
-                '@typescript-eslint/no-unused-vars': ['warn', { argsIgnorePattern: '^_', varsIgnorePattern: '^_' }],
+                '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_', varsIgnorePattern: '^_' }],
                 'node/no-missing-import': [
                     'error',
                     {

diff --git a/packages/cspell-eslint-plugin/src/index.test.ts b/packages/cspell-eslint-plugin/src/index.test.ts
@@ -35,7 +35,7 @@ ruleTester.run('cspell', rule.rules.cspell, {
         // 'async function* values(iter) { yield* iter; }',
         // 'var foo = true',
         // 'const x = `It is now time to add everything up: \\` ${y} + ${x}`',
-        // sampleCodeJS(),
+        sampleCodeJS(),
         sampleTs(),
     ],
     invalid: [],

diff --git a/packages/cspell-trie-lib/src/lib/suggestions/genSuggestionsOptions.ts b/packages/cspell-trie-lib/src/lib/suggestions/genSuggestionsOptions.ts
@@ -1,5 +1,5 @@
 import { WeightMap } from '..';
-import { CompoundWordsMethod } from '../walker';
+import { CompoundWordsMethod } from './walker';
 
 export interface GenSuggestionOptionsStrict {
     /**
@@ -20,10 +20,10 @@ export interface GenSuggestionOptionsStrict {
     changeLimit: number;
 
     /**
-     * Include the `+` compound character when returning results.
-     * @default false
+     * Inserts a compound character between compounded word segments.
+     * @default ""
      */
-    includeCompoundChar?: boolean;
+    compoundSeparator?: string;
 }
 
 export type GenSuggestionOptions = Partial<GenSuggestionOptionsStrict>;
@@ -63,7 +63,6 @@ export const defaultGenSuggestionOptions: GenSuggestionOptionsStrict = {
     compoundMethod: CompoundWordsMethod.NONE,
     ignoreCase: true,
     changeLimit: 5,
-    includeCompoundChar: false,
 };
 
 export const defaultSuggestionOptions: SuggestionOptionsStrict = {
@@ -89,7 +88,7 @@ const keyMapOfGenSuggestionOptionsStrict: KeyMapOfGenSuggestionOptionsStrict = {
     changeLimit: 'changeLimit',
     compoundMethod: 'compoundMethod',
     ignoreCase: 'ignoreCase',
-    includeCompoundChar: 'includeCompoundChar',
+    compoundSeparator: 'compoundSeparator',
 } as const;
 
 const keyMapOfSuggestionOptionsStrict: KeyMapOfSuggestionOptionsStrict = {

diff --git a/packages/cspell-trie-lib/src/lib/suggestions/suggest-en-a-star.test.ts b/packages/cspell-trie-lib/src/lib/suggestions/suggest-en-a-star.test.ts
@@ -8,7 +8,7 @@ import {
     SuggestionCollector,
 } from './suggestCollector';
 import { createTimer } from '../utils/timer';
-import { CompoundWordsMethod } from '../walker';
+import { CompoundWordsMethod } from './walker';
 import { clean } from '../trie-util';
 
 function getTrie() {

diff --git a/packages/cspell-trie-lib/src/lib/suggestions/suggest-en.test.ts b/packages/cspell-trie-lib/src/lib/suggestions/suggest-en.test.ts
@@ -2,7 +2,7 @@ import { readRawDictionaryFile, readTrie } from '../../test/dictionaries.test.he
 import { genCompoundableSuggestions, suggest } from './suggest';
 import { suggestionCollector, SuggestionCollectorOptions, SuggestionResult } from './suggestCollector';
 import { createTimer } from '../utils/timer';
-import { CompoundWordsMethod } from '../walker';
+import { CompoundWordsMethod } from './walker';
 import { clean } from '../trie-util';
 import { DictionaryInformation } from '../models/DictionaryInformation';
 import { mapDictionaryInformationToWeightMap, WeightMap } from '..';

diff --git a/packages/cspell-trie-lib/src/lib/suggestions/suggest.legacy.test.ts b/packages/cspell-trie-lib/src/lib/suggestions/suggest.legacy.test.ts
@@ -3,7 +3,7 @@ import { SuggestionResult } from './suggestCollector';
 import { Trie } from '../trie';
 import { TrieNode } from '../TrieNode';
 import { isWordTerminationNode } from '../trie-util';
-import { walker } from '../walker';
+import { walker } from './walker';
 
 describe('Validate Suggest', () => {
     test('Tests suggestions against Legacy Suggestion generator', () => {

diff --git a/packages/cspell-trie-lib/src/lib/suggestions/suggest.test.ts b/packages/cspell-trie-lib/src/lib/suggestions/suggest.test.ts
@@ -2,7 +2,7 @@ import { GenSuggestionOptions, SuggestionOptions } from './genSuggestionsOptions
 import { parseDictionary } from '../SimpleDictionaryParser';
 import { Trie } from '../trie';
 import { cleanCopy } from '../utils/util';
-import * as Walker from '../walker';
+import * as Walker from './walker';
 import { genCompoundableSuggestions, genSuggestions, suggest } from './suggest';
 import {
     compSuggestionResults,
@@ -255,6 +255,36 @@ describe('Validate Suggest', () => {
         const r = trie.suggestWithCost(word, { numSuggestions, ignoreCase, changeLimit });
         expect(r).toEqual(expected);
     });
+
+    test.each`
+        word              | ignoreCase   | numSuggestions | changeLimit | expected
+        ${'runningtree'}  | ${undefined} | ${2}           | ${3}        | ${[sr('running•tree', 0), sr('Running•tree', 1)]}
+        ${'Runningpod'}   | ${undefined} | ${4}           | ${1}        | ${[sr('Running•pod', 0), sr('running•pod', 1), sr('Running•Pod', 1), sr('running•Pod', 2)]}
+        ${'Runningpod'}   | ${false}     | ${4}           | ${1}        | ${[sr('Running•Pod', 1)]}
+        ${'runningpod'}   | ${undefined} | ${4}           | ${1}        | ${[sr('running•pod', 0), sr('running•Pod', 1), sr('Running•pod', 1), sr('Running•Pod', 2)]}
+        ${'runningpod'}   | ${false}     | ${4}           | ${1}        | ${[sr('Running•Pod', 2)]}
+        ${'walkingpod'}   | ${undefined} | ${2}           | ${3}        | ${[sr('walking•pod', 0), sr('walking•Pod', 1)]}
+        ${'walkingstick'} | ${undefined} | ${2}           | ${3}        | ${[sr('walking•stick', 0), sr('talking•stick', 99)]}
+        ${'walkingtree'}  | ${undefined} | ${2}           | ${4}        | ${[sr('talking•tree', 99), sr('walking•stick', 359)]}
+        ${'talkingtrick'} | ${undefined} | ${2}           | ${4}        | ${[sr('talking•stick', 183), sr('talking•tree', 268)]}
+        ${'running'}      | ${undefined} | ${2}           | ${3}        | ${[sr('running', 0), sr('Running', 1)]}
+        ${'free'}         | ${undefined} | ${2}           | ${2}        | ${[sr('tree', 99)]}
+        ${'stock'}        | ${undefined} | ${2}           | ${2}        | ${[sr('stick', 97)]}
+    `('suggestWithCost and separator $word', ({ word, ignoreCase, numSuggestions, changeLimit, expected }) => {
+        const trie = parseDictionary(`
+            walk
+            Running*
+            walking*
+            *stick
+            talking*
+            *tree
+            +Pod
+            !walkingtree
+            !walking+
+        `);
+        const r = trie.suggestWithCost(word, { numSuggestions, ignoreCase, changeLimit, compoundSeparator: '•' });
+        expect(r).toEqual(expected);
+    });
 });
 
 function numSugs(numSuggestions: number): SuggestionOptions {

diff --git a/packages/cspell-trie-lib/src/lib/suggestions/suggest.ts b/packages/cspell-trie-lib/src/lib/suggestions/suggest.ts
@@ -9,7 +9,7 @@ import {
 } from './suggestCollector';
 import { TrieRoot } from '../TrieNode';
 import { clean, isWordTerminationNode } from '../trie-util';
-import { CompoundWordsMethod, hintedWalker, JOIN_SEPARATOR, WORD_SEPARATOR } from '../walker';
+import { CompoundWordsMethod, hintedWalker, JOIN_SEPARATOR, WORD_SEPARATOR } from './walker';
 
 const baseCost = 100;
 const swapCost = 75;
@@ -105,7 +105,7 @@ export function* genCompoundableSuggestions(
     stack[0] = { a, b };
 
     const hint = word;
-    const iWalk = hintedWalker(root, ignoreCase, hint, compoundMethod);
+    const iWalk = hintedWalker(root, ignoreCase, hint, compoundMethod, options.compoundSeparator);
     let goDeeper = true;
     for (let r = iWalk.next({ goDeeper }); !stopNow && !r.done; r = iWalk.next({ goDeeper })) {
         const { text, node, depth } = r.value;

diff --git a/packages/cspell-trie-lib/src/lib/suggestions/suggestAStar.test.ts b/packages/cspell-trie-lib/src/lib/suggestions/suggestAStar.test.ts
@@ -3,7 +3,7 @@ import { parseDictionary } from '../SimpleDictionaryParser';
 import * as Sug from './suggestAStar';
 import { SuggestionCollector, suggestionCollector, SuggestionCollectorOptions } from './suggestCollector';
 import { Trie } from '../trie';
-import { CompoundWordsMethod } from '../walker';
+import { CompoundWordsMethod } from './walker';
 import { clean } from '../trie-util';
 
 const defaultOptions: SuggestionCollectorOptions = {

diff --git a/packages/cspell-trie-lib/src/lib/suggestions/suggestAStar.ts b/packages/cspell-trie-lib/src/lib/suggestions/suggestAStar.ts
@@ -1,5 +1,5 @@
 import { TrieRoot, TrieNode } from '../TrieNode';
-import { CompoundWordsMethod, JOIN_SEPARATOR, WORD_SEPARATOR } from '../walker';
+import { CompoundWordsMethod, JOIN_SEPARATOR, WORD_SEPARATOR } from './walker';
 import { SuggestionGenerator, suggestionCollector, SuggestionResult } from './suggestCollector';
 import { PairingHeap } from '../utils/PairingHeap';
 import { visualLetterMaskMap } from './orthography';

diff --git a/packages/cspell-trie-lib/src/lib/suggestions/suggestCollector.ts b/packages/cspell-trie-lib/src/lib/suggestions/suggestCollector.ts
@@ -1,6 +1,6 @@
 import { editDistanceWeighted, WeightMap } from '..';
 import { createTimer } from '../utils/timer';
-import { JOIN_SEPARATOR, WORD_SEPARATOR } from '../walker';
+import { JOIN_SEPARATOR, WORD_SEPARATOR } from './walker';
 
 const defaultMaxNumberSuggestions = 10;
 

diff --git a/...e-lib/src/lib/walker/hintedWalker.test.ts → ...b/suggestions/walker/hintedWalker.test.ts b/...e-lib/src/lib/walker/hintedWalker.test.ts → ...b/suggestions/walker/hintedWalker.test.ts
@@ -1,7 +1,7 @@
 import type { YieldResult } from './walkerTypes';
 import { hintedWalker } from './hintedWalker';
-import { orderTrie, createTriFromList } from '../trie-util';
-import { parseLinesToDictionary } from '../SimpleDictionaryParser';
+import { orderTrie, createTriFromList } from '../../trie-util';
+import { parseLinesToDictionary } from '../../SimpleDictionaryParser';
 
 describe('Validate Util Functions', () => {
     test('Hinted Walker', () => {
@@ -20,13 +20,42 @@ describe('Validate Util Functions', () => {
             }
             goDeeper = text.length < 4;
         }
-        expect(result).toEqual('joy jowl talk lift walk'.split(' '));
+        expect(result).toEqual(s('joy jowl talk lift walk'));
     });
 
-    test('Hinted Walker compounds', () => {
-        const dict = ['A*', '+a*', '*b*', '+c'];
+    test.each`
+        word        | expected
+        ${'joty'}   | ${s('joy jowl talk lift walk')}
+        ${'talked'} | ${s('talk lift jowl joy walk')}
+    `('Hinted Walker with strange word list: "$word"', ({ word, expected }) => {
+        const root = createTriFromList([...sampleWords, 'joy++', 'talk++']);
+        orderTrie(root);
+        // cspell:ignore joty
+        // prefer letters j, o, t, y before the others.
+        const i = hintedWalker(root, false, word, undefined);
+        let goDeeper = true;
+        let ir: IteratorResult<YieldResult>;
+        const result: string[] = [];
+        while (!(ir = i.next({ goDeeper })).done) {
+            const { text, node } = ir.value;
+            if (node.f) {
+                result.push(text);
+            }
+            goDeeper = text.length < 4;
+        }
+        expect(result).toEqual(expected);
+    });
+
+    test.each`
+        dict                          | sep    | depth | expected
+        ${[]}                         | ${''}  | ${2}  | ${[]}
+        ${['A*', '+a*', '*b*', '+c']} | ${''}  | ${2}  | ${['A', 'Aa', 'Ab', 'Ac', 'b', 'ba', 'bb', 'bc']}
+        ${['A*', '+a*', '*b*', '+c']} | ${'+'} | ${2}  | ${['A', 'A+a', 'A+b', 'A+c', 'b', 'b+a', 'b+b', 'b+c']}
+        ${['A+', '+a*', '+b']}        | ${'•'} | ${3}  | ${['A•a', 'A•a•a', 'A•a•b', 'A•b']}
+        ${['A+', '+b+', '+C']}        | ${'•'} | ${5}  | ${['A•C', 'A•b•C', 'A•b•b•C', 'A•b•b•b•C']}
+    `('Hinted Walker compounds $dict', ({ dict, sep, depth, expected }) => {
         const trie = parseLinesToDictionary(dict, { stripCaseAndAccents: true });
-        const i = hintedWalker(trie.root, false, 'a', undefined);
+        const i = hintedWalker(trie.root, false, 'a', undefined, sep);
         let goDeeper = true;
         let ir: IteratorResult<YieldResult>;
         const result: string[] = [];
@@ -35,9 +64,9 @@ describe('Validate Util Functions', () => {
             if (node.f) {
                 result.push(text);
             }
-            goDeeper = text.length < 2;
+            goDeeper = text.split(sep).join('').length < depth;
         }
-        expect(result).toEqual(['A', 'Aa', 'Ab', 'Ac', 'b', 'ba', 'bb', 'bc']);
+        expect(result).toEqual(expected);
     });
 
     test('Hinted Walker compounds ignoreCase', () => {
@@ -58,6 +87,10 @@ describe('Validate Util Functions', () => {
     });
 });
 
+function s(text: string, splitOn = ' '): string[] {
+    return text.split(splitOn);
+}
+
 const sampleWords = [
     'walk',
     'walked',

diff --git a/...l-trie-lib/src/lib/walker/hintedWalker.ts → ...rc/lib/suggestions/walker/hintedWalker.ts b/...l-trie-lib/src/lib/walker/hintedWalker.ts → ...rc/lib/suggestions/walker/hintedWalker.ts
@@ -1,5 +1,5 @@
-import { isDefined } from '../trie-util';
-import { TrieNode, TrieRoot } from '../TrieNode';
+import { isDefined } from '../../trie-util';
+import { TrieNode, TrieRoot } from '../../TrieNode';
 import { CompoundWordsMethod, JOIN_SEPARATOR, WORD_SEPARATOR, YieldResult } from './walkerTypes';
 
 /**
@@ -11,16 +11,26 @@ import { CompoundWordsMethod, JOIN_SEPARATOR, WORD_SEPARATOR, YieldResult } from
 // [Symbol.iterator]: () => HintedWalkerIterator;
 export type HintedWalkerIterator = Generator<YieldResult, void, Hinting | undefined>;
 
+export function hintedWalker(
+    root: TrieRoot,
+    ignoreCase: boolean,
+    hint: string,
+    compoundingMethod: CompoundWordsMethod | undefined,
+    emitWordSeparator?: string
+): HintedWalkerIterator {
+    return hintedWalkerNext(root, ignoreCase, hint, compoundingMethod, emitWordSeparator);
+}
+
 /**
  * Walks the Trie and yields a value at each node.
  * next(goDeeper: boolean):
  */
-
-export function* hintedWalker(
+function* hintedWalkerNext(
     root: TrieRoot,
     ignoreCase: boolean,
     hint: string,
-    compoundingMethod: CompoundWordsMethod | undefined
+    compoundingMethod: CompoundWordsMethod | undefined,
+    emitWordSeparator = ''
 ): HintedWalkerIterator {
     const _compoundingMethod = compoundingMethod ?? CompoundWordsMethod.NONE;
 
@@ -40,6 +50,7 @@ export function* hintedWalker(
 
     const roots = rawRoots.map(filterRoot);
     const compoundRoots = rawRoots.map((r) => r.c?.get(compoundCharacter)).filter(isDefined);
+    const setOfCompoundRoots = new Set(compoundRoots);
     const rootsForCompoundMethods = roots.concat(compoundRoots);
 
     const compoundMethodRoots: { [index: number]: readonly (readonly [string, TrieNode])[] } = {
@@ -82,15 +93,18 @@ export function* hintedWalker(
                     node,
                     hintOffset: hintOffset + 1,
                 }));
-            if (c.has(compoundCharacter)) {
+            if (c.has(compoundCharacter) && !setOfCompoundRoots.has(n)) {
                 for (const compoundRoot of compoundRoots) {
-                    yield* children(compoundRoot, hintOffset);
+                    for (const child of children(compoundRoot, hintOffset)) {
+                        const { letter, node, hintOffset } = child;
+                        yield { letter: emitWordSeparator + letter, node, hintOffset };
+                    }
                 }
             }
         }
         if (n.f) {
             yield* [...compoundMethodRoots[_compoundingMethod]].map(([letter, node]) => ({
-                letter,
+                letter: emitWordSeparator + letter,
                 node,
                 hintOffset,
             }));
@@ -100,22 +114,21 @@ export function* hintedWalker(
     for (const root of roots) {
         let depth = 0;
         const stack: Stack = [];
-        let baseText = '';
+        const stackText: string[] = [''];
         stack[depth] = children(root, depth);
         let ir: IteratorResult<StackItemEntry, StackItemEntry>;
         while (depth >= 0) {
             while (!(ir = stack[depth].next()).done) {
                 const { letter: char, node, hintOffset } = ir.value;
-                const text = baseText + char;
+                const text = stackText[depth] + char;
                 const hinting = (yield { text, node, depth }) as Hinting;
                 if (hinting && hinting.goDeeper) {
                     depth++;
-                    baseText = text;
+                    stackText[depth] = text;
                     stack[depth] = children(node, hintOffset);
                 }
             }
             depth -= 1;
-            baseText = baseText.slice(0, -1);
         }
     }
 }
@@ -124,10 +137,14 @@ export interface Hinting {
     goDeeper: boolean;
 }
 
-export function existMap(values: string[]): Record<string, true> {
+function existMap(values: string[]): Record<string, true> {
     const m: Record<string, true> = Object.create(null);
     for (const v of values) {
         m[v] = true;
     }
     return m;
 }
+
+export const __testing__ = {
+    hintedWalkerNext,
+};
diff --git a/packages/cspell-trie-lib/src/lib/suggestions/walker/index.ts b/packages/cspell-trie-lib/src/lib/suggestions/walker/index.ts
@@ -0,0 +1,4 @@
+export * from './walker';
+export { hintedWalker } from './hintedWalker';
+export type { HintedWalkerIterator, Hinting } from './hintedWalker';
+export * from './walkerTypes';
diff --git a/...ll-trie-lib/src/lib/walker/walker.test.ts → ...src/lib/suggestions/walker/walker.test.ts b/...ll-trie-lib/src/lib/walker/walker.test.ts → ...src/lib/suggestions/walker/walker.test.ts
@@ -1,6 +1,6 @@
 import { walker } from './walker';
 import type { YieldResult } from './walkerTypes';
-import { orderTrie, createTriFromList } from '../trie-util';
+import { orderTrie, createTriFromList } from '../../trie-util';
 
 describe('Validate Util Functions', () => {
     test('Tests Walker', () => {

diff --git a/.../cspell-trie-lib/src/lib/walker/walker.ts → ...-lib/src/lib/suggestions/walker/walker.ts b/.../cspell-trie-lib/src/lib/walker/walker.ts → ...-lib/src/lib/suggestions/walker/walker.ts
@@ -1,4 +1,4 @@
-import { TrieNode, ChildMap } from '../TrieNode';
+import { TrieNode, ChildMap } from '../../TrieNode';
 import { CompoundWordsMethod, WalkerIterator, WORD_SEPARATOR, JOIN_SEPARATOR } from './walkerTypes';
 
 /**

diff --git a/...ll-trie-lib/src/lib/walker/walkerTypes.ts → ...src/lib/suggestions/walker/walkerTypes.ts b/...ll-trie-lib/src/lib/walker/walkerTypes.ts → ...src/lib/suggestions/walker/walkerTypes.ts
@@ -1,4 +1,4 @@
-import { TrieNode } from '../TrieNode';
+import { TrieNode } from '../../TrieNode';
 
 export const JOIN_SEPARATOR = '+';
 export const WORD_SEPARATOR = ' ';