How to use string-punctuation-tokenizer - 4 common examples
To help you get started, we’ve selected a few string-punctuation-tokenizer examples, based on popular ways it is used in public projects.
Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
unfoldingWord / translationCore / src / js / helpers / VerseObjectHelpers.js View on Github 
stringHelpers.tokenizeWithPunctuation(_verseObject.text).map( text => {
let verseObject;
if (stringHelpers.word.test(text)) { // if the text has word characters, its a word object
const wordIndex = verseObjects.length - nonWordVerseObjectCount;
let occurrence = stringHelpers.occurrenceInString(_verseObjectsWithTextString, wordIndex, text);
const occurrences = stringHelpers.occurrencesInString(_verseObjectsWithTextString, text);
if (occurrence > occurrences) occurrence = occurrences;
verseObject = {
tag: "w",
type: "word",
text,
occurrence,
occurrences
};
} else { // the text does not have word characters
nonWordVerseObjectCount ++;
verseObject = {
type: "text",
text: text
};
}stringHelpers.tokenizeWithPunctuation(_verseObject.text).map( text => {
let verseObject;
if (stringHelpers.word.test(text)) { // if the text has word characters, its a word object
const wordIndex = verseObjects.length - nonWordVerseObjectCount;
let occurrence = stringHelpers.occurrenceInString(_verseObjectsWithTextString, wordIndex, text);
const occurrences = stringHelpers.occurrencesInString(_verseObjectsWithTextString, text);
if (occurrence > occurrences) occurrence = occurrences;
verseObject = {
tag: "w",
type: "word",
text,
occurrence,
occurrences
};
} else { // the text does not have word characters
nonWordVerseObjectCount ++;
verseObject = {
type: "text",
text: text
};
}
verseObjects.push(verseObject);_verseObjects.forEach(_verseObject => {
if (_verseObject.text) {
stringHelpers.tokenizeWithPunctuation(_verseObject.text).map( text => {
let verseObject;
if (stringHelpers.word.test(text)) { // if the text has word characters, its a word object
const wordIndex = verseObjects.length - nonWordVerseObjectCount;
let occurrence = stringHelpers.occurrenceInString(_verseObjectsWithTextString, wordIndex, text);
const occurrences = stringHelpers.occurrencesInString(_verseObjectsWithTextString, text);
if (occurrence > occurrences) occurrence = occurrences;
verseObject = {
tag: "w",
type: "word",
text,
occurrence,
occurrences
};
} else { // the text does not have word characters
nonWordVerseObjectCount ++;
verseObject = {stringHelpers.tokenizeWithPunctuation(_verseObject.text).map( text => {
let verseObject;
if (stringHelpers.word.test(text)) { // if the text has word characters, its a word object
const wordIndex = verseObjects.length - nonWordVerseObjectCount;
let occurrence = stringHelpers.occurrenceInString(_verseObjectsWithTextString, wordIndex, text);
const occurrences = stringHelpers.occurrencesInString(_verseObjectsWithTextString, text);
if (occurrence > occurrences) occurrence = occurrences;
verseObject = {
tag: "w",
type: "word",
text,
occurrence,
occurrences
};
} else { // the text does not have word characters
nonWordVerseObjectCount ++;
verseObject = {
type: "text",
text: textstring-punctuation-tokenizer
Small library that provides functions to tokenize a string into an array of words with or without punctuation
