How to use the @nlpjs/core/src.Tokenizer function in @nlpjs/core
To help you get started, we’ve selected a few @nlpjs/core examples, based on popular ways it is used in public projects.
Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
hubtype / botonic / packages / botonic-plugin-contentful / src / nlp / tokens.ts View on Github 
private trim(arr: string[]): string[] {
while (arr[arr.length - 1] === '') {
arr.pop()
}
while (arr[0] === '') {
arr.shift()
}
return arr
}
}
const tokenizers: { [locale: string]: Tokenizer } = {
es: new TokenizerEs(),
en: new TokenizerEn(),
ca: new TokenizerCa(),
pl: new Tokenizer(),
pt: new TokenizerPt(),
}
export function tokenizerPerLocale(locale: Locale): Tokenizer {
return tokenizers[locale]
}
export const DEFAULT_SEPARATORS = ';,./()!?" '
export const DEFAULT_SEPARATORS_REGEX = new RegExp(
'[' + DEFAULT_SEPARATORS + ']',
'g'
)
export const DEFAULT_NOT_SEPARATORS_REGEX = new RegExp(
'[^' + DEFAULT_SEPARATORS + ']',
'g'
)
