Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
/*!
* levels
* Copyright(c) 2013 Eugene Ware
* MIT Licensed
*/
/**
* Module dependencies.
*/
var natural = require('natural')
, metaphone = natural.Metaphone.process
, stem = natural.PorterStemmer.stem
, stopwords = natural.stopwords
, sublevel = require('level-sublevel')
, _ = require('underscore')
, async = require('async')
, byteup = require('byteup')
, noop = function(){};
/* Add bytewise encoding */
byteup();
/**
* Library version.
*/
exports.version = '0.0.3';
/**
/*!
* reds
* Copyright(c) 2011 TJ Holowaychuk
* cn-search
* Copyright(c) 2013 Sxyizhiren <786647787@qq.com>
* MIT Licensed
*/
/**
* Module dependencies.
*/
var natural = require('natural');
var metaphone = natural.Metaphone.process;
var stem = natural.PorterStemmer.stem;
var stopwords = natural.stopwords;
var cnstopwords = require('./cnstopWords');
// default chinese segment
var Segment = require('segment').Segment;
var segment = new Segment();
segment.useDefault();
/**
* Chinese Segment
* @type {*}
*/
var segmentSync = function(str){
var words=segment.doSegment(str);
var result=[];
for(var i= 0,len=words.length;i
function stripStopWords(words, options) {
var ret = [];
options = options || {};
var stopwords = options.stopwords || natural.stopwords;
if (words) {
for (var i = 0, len = words.length; i < len; ++i) {
if (~stopwords.indexOf(words[i])) continue;
ret.push(words[i]);
}
}
return ret;
}
/*!
* reds
* Copyright(c) 2011 TJ Holowaychuk
* MIT Licensed
*/
/**
* Module dependencies.
*/
var natural = require('natural');
var metaphone = natural.Metaphone.process;
var stem = natural.PorterStemmer.stem;
var stopwords = natural.stopwords;
var redis = require('redis');
function noop(){};
/**
* Library version.
*/
exports.version = '1.0.0';
/**
* Expose `Search`.
*/
exports.Search = Search;
/**
var fs = require('fs')
, levelup = require('levelup')
, natural = require('natural');
//Calibration values
//precalculate index metadata to save time ewith every query
//Arbitrary defaults
var maxStringFieldLength = 300; //this has a really big effect on performance for some reason...
var maxMetadataFieldLength = 20;
//only use this global for fast reads when searching- NEVER WRITE TO IT
var indexMetaDataGlobal = readMetaDataGlobal();
var reverseIndex = levelup('./norchindex')
, stopwords = require('natural').stopwords
, TfIdf = require('natural').TfIdf;
exports.indexData = function(callback) {
callback(readMetaDataGlobal());
}
exports.indexPeek = function(start, stop, callback) {
var dump = '';
reverseIndex.createReadStream({
start:start + "~",
end:stop + "~~"})
.on('data', function(data) {
dump += data.key + '<br>'
var stopwords = require('natural').stopwords;
var async = require('async');
var scontext = require('search-context');
var indexSize = 0;
exports.search = function (reverseIndex, docFreqIndex, q, callback) {
docFreqIndex.get('forage.totalDocs', function (err, value) {
indexSize = value;
});
//this must be set to true for a query to be carried out
var canSearch = true,
//tq = Transformed Query
tq = Object.create(q),
k,
indexKeys,
self.keywords = function() {
var keywords = natural.stopwords.filter(function(i) {
return i.length > 1 || i === "a" || i === "i";
});
keywords.push("?");
keywords.push("when");
return keywords;
}();
function Search(key, options) {
var options = options || {};
this.key = key;
this.stopwords=options.stopwords ? options.stopwords : false;
this.stopwords=options.naturalstop ? natural.stopwords : false;
}