Use utils.get instead of axios directly

Author: JexSrs

src/components/AxiosConfig.ts

@@ -2,13 +2,9 @@ import {ParserClass} from "../ParserClass";
 import Instructions from "../../../components/instructions";
 import Job from "../../../components/job";
 import Article from "../../../components/articles";
-import https from "https";
-import axios, {AxiosRequestConfig, AxiosResponse} from "axios";
+import {AxiosResponse} from "axios";
 import cheerio from "cheerio";
 import Utils from "../Utils";
-import {AxiosConfig} from "../../../components/AxiosConfig";
-
-const httpsAgent = new https.Agent({rejectUnauthorized: false})
 
 interface ArticleImage {
     [key: string]: any
@@ -19,7 +15,7 @@ export class HTMLParser extends ParserClass {
     static async parse2(instructions: Instructions, utils: Utils): Promise<Article[]> {
         let parsedArticles: Article[] = [];
 
-        await HTMLParser.request(utils.url, instructions.getSource().timeout, instructions)
+        await HTMLParser.request(utils)
             .then((response: AxiosResponse) => {
                 const cheerioLoad: cheerio.Root = cheerio.load(instructions.textDecoder.decode(response.data));
 
@@ -109,26 +105,20 @@ export class HTMLParser extends ParserClass {
         return parsedArticles
     }
 
-    private static async request(url: string, timeout: number, instructions: Instructions): Promise<AxiosResponse> {
+    private static async request(utils: Utils): Promise<AxiosResponse> {
         return new Promise((resolve, reject) => {
-
-            let config: AxiosConfig = {
-                method: 'get',
-                url,
-                timeout,
+            utils.get(utils.url, {
+                timeout: utils.instructions.getSource().timeout,
                 responseType: 'arraybuffer',
+                // @ts-ignore
                 responseEncoding: 'binary'
-            }
-
-            if (instructions["ignoreCertificates"]) config.httpsAgent = httpsAgent
-            axios((config as AxiosRequestConfig)).then((result: AxiosResponse) => {
-                resolve(result)
+            }).then((result: AxiosResponse) => {
+                resolve(result);
             }).catch((e: any) => {
                 let message = `HTMLParserException error during request, original error ${e.message}`
-                reject(new Error(message))
-            })
-
-        })
+                reject(new Error(message));
+            });
+        });
     }
 
     /**

src/modules/parsers/drivers/HTMLParser.ts

@@ -5,7 +5,6 @@ import Article from "../../../../components/articles";
 import axios, {AxiosRequestConfig} from "axios";
 import Utils from "../../Utils";
 import https from "https";
-import {AxiosConfig} from "../../../../components/AxiosConfig";
 
 const httpsAgent = new https.Agent({rejectUnauthorized: false})
 
@@ -90,16 +89,16 @@ export class WordpressV2Parser extends ParserClass {
         let categories: any
             , posts: any[];
 
-        let config: AxiosConfig & AxiosRequestConfig = {
-            timeout: instructions.getSource().timeout,
+        let config = {
+            timeout: utils.instructions.getSource().timeout,
             responseType: 'arraybuffer',
+            // @ts-ignore
             responseEncoding: 'binary'
         };
-        if (instructions["ignoreCertificates"]) config.httpsAgent = httpsAgent;
 
         try {
-            categories = JSON.parse(instructions.textDecoder.decode((await axios.get(categoriesUrl, config))?.data))
-            posts = JSON.parse(instructions.textDecoder.decode((await axios.get(postsUrl, config))?.data))
+            categories = JSON.parse(instructions.textDecoder.decode((await utils.get(categoriesUrl, config as any))?.data))
+            posts = JSON.parse(instructions.textDecoder.decode((await utils.get(postsUrl, config as any))?.data))
         } catch (e: any) {
             throw new Error(`WordpressParserException job failed for ${instructions.getSource().name}, original error: ${e.message}`);
         }

src/modules/parsers/drivers/wordpress/WordpressV2Parser.ts

@@ -31,6 +31,7 @@ export default class Worker {
             utils.aliases = pair.aliases;
             utils.isScrapeAfterError = job.attempts !== 0;
             utils.amount = job.getInstructions().amount;
+            utils.instructions = job.getInstructions();
 
             const newArticles = await parser.parse(job, utils);
             results.push({

src/modules/workers/index.ts

@@ -46,33 +46,28 @@ try {
     let errors = []
     const saffron = new Saffron();
 
-    try {
-        await saffron.initialize(config)
-        await saffron.start()
+    saffron.on("workers.articles.found", (articles, src) => {
+        console.log('articles.found', src, articles.length)
+        // console.log(util.inspect(articles, {showHidden: false, depth: null, colors: true}));
+    });
 
-        saffron.on("workers.articles.found", (articles, src) => {
-            console.log('articles.found')
-            console.log(src, articles.length)
-            console.log(util.inspect(articles, {showHidden: false, depth: null, colors: true}));
-        });
+    saffron.on('workers.parsers.error', (e) => errors.push(e));
 
-        // saffron.on("workers.articles.new", (articles, src) => {
-        //     console.log('articles.new')
-        //     console.log(src, articles.length);
-        //     console.log(util.inspect(articles, {showHidden: false, depth: null, colors: true}));
-        // });
+    saffron.on("middleware.before", (articles) => {
+        // console.log('middleware.before')
+        // console.log(util.inspect(articles, {showHidden: false, depth: null, colors: true}));
+    });
 
-        saffron.on("middleware.before", (articles) => {
-            // console.log('middleware.before')
-            // console.log(util.inspect(articles, {showHidden: false, depth: null, colors: true}));
-        });
+    saffron.on("middleware.after", (articles) => {
+        // console.log('middleware.after')
+        // console.log(util.inspect(articles, {showHidden: false, depth: null, colors: true}));
+    });
 
-        saffron.on("middleware.after", (articles) => {
-            // console.log('middleware.after')
-            // console.log(util.inspect(articles, {showHidden: false, depth: null, colors: true}));
-        });
+    try {
+        await saffron.initialize(config);
+        await saffron.start();
     } catch (error) {
-        errors.push(error)
+        errors.push(error);
     }
 
     if (process.env.NODE_ENV === "testing") setTimeout(() => {