Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
getTags(url) {
Metascraper
.scrapeUrl(url)
.then(metadata => {
const hasTags = !_.isEmpty(metadata);
this.setState({ tags: metadata, hasTags });
})
.catch(console.info);
}
scrape: async (source, { url }) => {
const { body: html } = await got(url);
const metadata = await Metascraper({ html, url });
return {
...metadata,
id: url,
};
return Metascraper.scrapeUrl(url, getRules(new URL(url)))
.then(getImages)
.then(metadata => ({
...metadata,
id: url,
}))
.catch(() => ({}))
}
},
router.post('/', (req, res, next) => {
if (!req.body.url) {
return res.status(400).json({
type: 'error',
error_code: 400,
error_message: 'Invalid request. Missing url',
});
}
const timer = logger.time('extract.post').namespace(req.body.url);
Metascraper.scrapeUrl(req.body.url).then(
data => {
const payload = {
url: data.url || req.body.url || '',
title: data.title || 'Unable to scrape title.',
content:
data.description ||
"Error: Unable to scrape description from the provided url. You'll have to do this on your own.",
author: data.publisher || 'Unable to scrape author.',
image: data.image || '',
};
cache.put(req.body.url, payload, TWENTY_FOUR_HOURS);
logger.log(Object.assign({}, { type: 'info' }, payload));
res.status(200).json(payload);
},
e => {
timer.log();
function onRequestDataExtraction(message, reply) {
logger.log(message);
const timer = logger.time('extract.post').namespace(message);
const cachedResult = cache.get(message.url);
if (cachedResult) {
return reply(cachedResult);
}
Metascraper.scrapeUrl(message.url)
.then(data => {
timer.log();
const payload = {
url: data.url || message.url,
title: data.title || 'Unable to scrape title.',
content: data.description ||
"Error: Unable to scrape description from the provided url. You'll have to do this on your own.",
author: data.publisher || 'Unable to scrape author.',
image: data.image || '',
};
cache.put(message.url, payload, TWENTY_FOUR_HOURS);
logger.log(Object.assign({}, { type: 'info' }, payload));
reply(payload);
})
.catch(e => {
timer.log();