Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
this.terminate('finished browsing on root node');
}
return;
} else {
this.repeatingCrawlingCount = 0;
}
} catch (e) {
this.terminate(e.stack);
}
// check is new
let isNew = true;
for (let index in this.crawlingBuffer) {
if (this.crawlingBuffer[index] &&
this.crawlingBuffer[index].text &&
stringSimilarity.compareTwoStrings(node.text, this.crawlingBuffer[index].text) >= 0.90) {
isNew = false;
node = this.crawlingBuffer[index];
this.currentNode = this.crawlingBuffer[index];
console.log('similar page exists');
break;
}
}
// if new, load actions to current node
if (isNew) {
// if reaching max depth, avoid crawling for current page
if (this.currentNode && this.currentNode.depth >= this.config.depth) {
console.log('exceeding max depth, triggering back');
this.back();
return;
}
function bestCat(label, showRating = false, log = false) {
const lbl = label.toLowerCase()
if (lbl in SIM_EXCEPTIONS) {
const target = SIM_EXCEPTIONS[lbl]
return showRating ? {target, rating: 1} : target
}
const match = {...strSim.findBestMatch(lbl, CATEGORIES).bestMatch, ref: 'M'}
const seMatch = {...strSim.findBestMatch(lbl, SE).bestMatch, ref: 'SE'}
const nclMatch = {
...strSim.findBestMatch(lbl, NON_CATEGORY_LABELS).bestMatch,
ref: 'NC',
}
const clMatch = {...strSim.findBestMatch(lbl, CL).bestMatch, ref: 'CL'}
const scores = [match, seMatch, clMatch, nclMatch].sort(
(a, b) => b.rating - a.rating,
)
if (log) {
//eslint-disable-next-line no-console
console.log('scores=', scores)
}
let idx = 0
while (scores[idx++].rating < MATCH_THRESHOLD && idx < scores.length) {
/* */
}
const related = this.isDefined(option, 'options');
if (related) {
const details = this.readOption(related);
Object.assign(options, details);
}
if (!related && !definedSubcommand) {
// Unknown Option
const availableOptions = [];
this.details.options.forEach((opt) => {
availableOptions.push(...opt.usage);
});
const suggestOption = stringSimilarity.findBestMatch(
option,
availableOptions,
);
console.log(`The option "${option}" is unknown.`); // eslint-disable-line
if (suggestOption.bestMatch.rating >= 0.5) {
console.log(' Did you mean the following one?\n'); // eslint-disable-line
const suggestion = this.details.options.filter((item) => {
for (const flag of item.usage) {
if (flag === suggestOption.bestMatch.target) {
return true;
}
}
const getClosest = (value, list) => {
const matches = stringSimilarity.findBestMatch(value, list);
// only return the best match if its greater than .5 in similarity
return matches.bestMatch.rating >= 0.5 ? matches.bestMatch.target : '';
};
texts.forEach((text, i) => {
const similarity = compareTwoStrings(text, target);
if (similarity > bestSimilarity) {
idx = i;
bestSimilarity = similarity;
}
});
let term = suggestion.term;
if (suggestion.sub_type) { term += ` (${suggestion.sub_type})`; }
return value === term;
}) || suggestions[0];
if (topSuggestion) {
let term = topSuggestion.term;
if (topSuggestion.sub_type) {
term += ` (${topSuggestion.sub_type})`;
topSuggestion.term = term;
}
if (topSuggestion.term_type === "_locations") {
let locParts = topSuggestion.term.split(", ");
value = value.split(", ")[0];
if (locParts.length) { term = locParts[0]; }
}
let similarity = Similarity.compareTwoStrings(value, term);
if (similarity > this.SIMILARITY_THRESHOLD) {
return this.gotoSearch(event, topSuggestion);
}
}
}
return this.gotoSearch(event, {
"term_type": "_all",
"term": value
});
}
function bestCat(label, showRating = false, log = false) {
const lbl = label.toLowerCase()
if (lbl in SIM_EXCEPTIONS) {
const target = SIM_EXCEPTIONS[lbl]
return showRating ? {target, rating: 1} : target
}
const match = {...strSim.findBestMatch(lbl, CATEGORIES).bestMatch, ref: 'M'}
const seMatch = {...strSim.findBestMatch(lbl, SE).bestMatch, ref: 'SE'}
const nclMatch = {
...strSim.findBestMatch(lbl, NON_CATEGORY_LABELS).bestMatch,
ref: 'NC',
}
const clMatch = {...strSim.findBestMatch(lbl, CL).bestMatch, ref: 'CL'}
const scores = [match, seMatch, clMatch, nclMatch].sort(
(a, b) => b.rating - a.rating,
)
if (log) {
//eslint-disable-next-line no-console
console.log('scores=', scores)
}
let idx = 0
while (scores[idx++].rating < MATCH_THRESHOLD && idx < scores.length) {
const thisTitle = t.content.title;
const titleSimilarity = stringSimilarity.compareTwoStrings(
incomingTitle,
thisTitle
);
debug(`Title similarity score for spam check: ${titleSimilarity}`);
if (titleSimilarity > 0.8) return true;
if (thread.content.body) {
const incomingBody = threadBodyToPlainText(thread.content.body);
const thisBody = threadBodyToPlainText(t.content.body);
if (incomingBody.length === 0 || thisBody.length === 0) return false;
const bodySimilarity = stringSimilarity.compareTwoStrings(
incomingBody,
thisBody
);
debug(`Body similarity score for spam check: ${bodySimilarity}`);
if (bodySimilarity > 0.8) return true;
}
return false;
});
(script, index) =>
stringSimilarity.compareTwoStrings(
script.contents.replace(/[0-9]/g, ''),
job.scripts[index].contents.replace(/[0-9]/g, ''),
) < similarIndication,
),
const sortBestMatch = query => (a, b) => {
const matchA = compareTwoStrings(query, a.name)
const matchB = compareTwoStrings(query, b.name)
return matchB - matchA
}