Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
// Hack to make iconv load the encodings module, otherwise jest crashes. Compare
// https://github.com/sidorares/node-mysql2/issues/489
require('iconv-lite').encodingExists('cesu8');
// load dotenv
require('dotenv').config();
function getSupportedEncoding (encoding) {
if (encoding === 'ISO-8859-8-I') encoding = 'ISO-8859-8'
if (iconv.encodingExists(encoding)) return encoding
return 'utf8' // default
}
function _getCharset(str, isMeta) {
var charset;
if ((isMeta ? META_CHARSET_RE : CHARSET_RE).test(str)) {
charset = RegExp.$1;
if (!iconv.encodingExists(charset)) {
charset = null;
}
}
return charset;
}
function _getCharset(str, isMeta) {
var charset;
if ((isMeta ? META_CHARSET_RE : CHARSET_RE).test(str)) {
charset = RegExp.$1;
if (!iconv.encodingExists(charset)) {
charset = null;
}
}
return charset;
}
if(err){
cb(err);
}else if(!bodyContent){
cb(null,result);
}else{
var record = doc[0],
resHeader = record['resHeader'] || {};
try{
var headerStr = JSON.stringify(resHeader),
charsetMatch = headerStr.match(/charset="?([a-zA-Z0-9\-]+)"?/),
imageMatch = resHeader && resHeader["content-type"];
if(charsetMatch && charsetMatch.length){
var currentCharset = charsetMatch[1].toLowerCase();
if(currentCharset != "utf-8" && iconv.encodingExists(currentCharset)){
bodyContent = iconv.decode(bodyContent, currentCharset);
}
result.type = "text";
result.content = bodyContent.toString();
}else if(imageMatch && /image/i.test(imageMatch)){
result.type = "image";
result.mime = imageMatch;
result.content = bodyContent;
}else{
result.content = bodyContent.toString();
}
}catch(e){}
cb(null,result);
}
function autoDecodeCharset(data){
if(data){
var buffer = new Buffer(data),
charset = chardet.detect(buffer);
console.log(('Data charset is '+charset ).magenta.bold);
try {
data = buffer.toString(charset);
} catch (e) {
if(Iconv.encodingExists(charset)){
data = Iconv.decode(buffer,charset);
}
}
return data;
}
}//end autoDecodeCharset
/**
export function encode(string, encoding = 'utf8') {
if (encoding === 'utf8') return Buffer.from(string, 'utf8')
if (!iconv.encodingExists(encoding))
throw new Error(`Encoding does not exist: "${encoding}"`)
return iconv.encode(string, encoding)
}
function normalizeCharset(charset) {
charset = charset
&& charset.replace(/[- ]/g, "").toLowerCase()
|| "utf8";
assert.ok(
iconv.encodingExists(charset),
"Unrecognized charset: " + charset
);
return charset;
}
function decode(content, encoding) {
if (encoding !== 'UTF-8') {
logger.Debug('Decoding with: ' + encoding);
var iconv = require('iconv-lite');
if(iconv.encodingExists(encoding))
var buffer = iconv.decode(content, encoding);
else if (iconv.encodingExists(encoding.replace('CP', 'win')))
var buffer = iconv.decode(content, encoding.replace('CP', 'win'));
else
buffer = content;
} else {
buffer = content
}
return buffer.toString('UTF-8');
};
function _parser(filename) {
var fileBuffer = grunt.file.read(filename, {
encoding: null
});
var detected = jschardet.detect(fileBuffer);
var fileContent;
if (iconv.encodingExists(detected.encoding)) {
fileContent = iconv.decode(fileBuffer, detected.encoding);
grunt.log.debug(filename + ' - encoding detected: ' + detected.encoding);
} else {
fileContent = iconv.decode(fileBuffer, 'utf8');
grunt.log.debug(filename + ' - encoding not detected, used utf-8 instead');
}
csv.parse(fileContent, options, function(error, output) {
if (error) {
_cb(error);
} else {
lines = lines.concat(output);
_cb();
}
});
}