Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
this.terminate('finished browsing on root node');
}
return;
} else {
this.repeatingCrawlingCount = 0;
}
} catch (e) {
this.terminate(e.stack);
}
// check is new
let isNew = true;
for (let index in this.crawlingBuffer) {
if (this.crawlingBuffer[index] &&
this.crawlingBuffer[index].text &&
stringSimilarity.compareTwoStrings(node.text, this.crawlingBuffer[index].text) >= 0.90) {
isNew = false;
node = this.crawlingBuffer[index];
this.currentNode = this.crawlingBuffer[index];
console.log('similar page exists');
break;
}
}
// if new, load actions to current node
if (isNew) {
// if reaching max depth, avoid crawling for current page
if (this.currentNode && this.currentNode.depth >= this.config.depth) {
console.log('exceeding max depth, triggering back');
this.back();
return;
}
texts.forEach((text, i) => {
const similarity = compareTwoStrings(text, target);
if (similarity > bestSimilarity) {
idx = i;
bestSimilarity = similarity;
}
});
let term = suggestion.term;
if (suggestion.sub_type) { term += ` (${suggestion.sub_type})`; }
return value === term;
}) || suggestions[0];
if (topSuggestion) {
let term = topSuggestion.term;
if (topSuggestion.sub_type) {
term += ` (${topSuggestion.sub_type})`;
topSuggestion.term = term;
}
if (topSuggestion.term_type === "_locations") {
let locParts = topSuggestion.term.split(", ");
value = value.split(", ")[0];
if (locParts.length) { term = locParts[0]; }
}
let similarity = Similarity.compareTwoStrings(value, term);
if (similarity > this.SIMILARITY_THRESHOLD) {
return this.gotoSearch(event, topSuggestion);
}
}
}
return this.gotoSearch(event, {
"term_type": "_all",
"term": value
});
}
const thisTitle = t.content.title;
const titleSimilarity = stringSimilarity.compareTwoStrings(
incomingTitle,
thisTitle
);
debug(`Title similarity score for spam check: ${titleSimilarity}`);
if (titleSimilarity > 0.8) return true;
if (thread.content.body) {
const incomingBody = threadBodyToPlainText(thread.content.body);
const thisBody = threadBodyToPlainText(t.content.body);
if (incomingBody.length === 0 || thisBody.length === 0) return false;
const bodySimilarity = stringSimilarity.compareTwoStrings(
incomingBody,
thisBody
);
debug(`Body similarity score for spam check: ${bodySimilarity}`);
if (bodySimilarity > 0.8) return true;
}
return false;
});
(script, index) =>
stringSimilarity.compareTwoStrings(
script.contents.replace(/[0-9]/g, ''),
job.scripts[index].contents.replace(/[0-9]/g, ''),
) < similarIndication,
),
const sortBestMatch = query => (a, b) => {
const matchA = compareTwoStrings(query, a.name)
const matchB = compareTwoStrings(query, b.name)
return matchB - matchA
}
.map((i, index) => {
const { name: resultName, singer: resultArtist } = i;
return {
match:
compareTwoStrings(name, resultName) *
compareTwoStrings(artist, resultArtist),
index
};
})
.sort((i, j) => j.match - i.match)[0].index;
.map((i, index) => {
const { name: resultName, singer: resultArtist } = i;
return {
match:
compareTwoStrings(name, resultName) *
compareTwoStrings(artist, resultArtist),
index
};
})
.sort((i, j) => j.match - i.match)[0].index;