Skip to content

Commit d07ee2b

Browse files
JexSrsJexSrs
committedJul 5, 2022
Exported types and changed source error messages
·
v6.2.1v3.0.0
1 parent 28e770a commit d07ee2b

File tree

10 files changed

+84
-105
lines changed

10 files changed

+84
-105
lines changed
 

‎src/components/article.ts

+8-3
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ export type Attachment = {
77
text?: string;
88
};
99

10+
export type Category = {
11+
name: string;
12+
links: string[];
13+
};
14+
1015
export default class Article {
1116
declare id: string;
1217
declare title: string;
@@ -19,8 +24,8 @@ export default class Article {
1924
id: string;
2025
name: string;
2126
};
22-
declare attachments: object[];
23-
declare categories: object[];
27+
declare attachments: Attachment[];
28+
declare categories: Category[];
2429
declare thumbnail: string;
2530

2631
constructor() {
@@ -93,7 +98,7 @@ export default class Article {
9398
this.categories.push({name, links});
9499
}
95100

96-
public pushCategories(categories: { name: string, links: string[] }[]) {
101+
public pushCategories(categories: Category[]) {
97102
if (typeof this.categories === 'undefined')
98103
this.categories = [];
99104

‎src/components/config.ts

+6-6
Original file line numberDiff line numberDiff line change
@@ -34,16 +34,18 @@ export type ConfigType = {
3434
distributed: false;
3535
} | {
3636
distributed: true;
37+
useHTTP: false;
38+
3739
serverAddress: string;
3840
serverPort?: number;
3941
authToken: string;
40-
useHTTP: false;
4142
} | {
4243
distributed: true;
44+
useHTTP: true;
45+
4346
serverAddress: string;
4447
serverPort?: number;
4548
authToken: string;
46-
useHTTP: true;
4749
key: any;
4850
cert: any;
4951
};
@@ -81,10 +83,8 @@ export default class Config {
8183
},
8284
grid: {
8385
distributed: false,
84-
// For default values on false
85-
// @ts-ignore
86-
serverPort: 3000,
87-
useHTTP: false
86+
useHTTP: false,
87+
serverPort: 3000
8888
},
8989
misc: {
9090
log: "all"

‎src/components/instructions.ts

+7-12
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import Source from "./source";
22
import {ParserType} from "../middleware/ParserType";
3-
import randomId from "../middleware/randomId"
3+
4+
export type InstructionUrl = {
5+
url: string;
6+
aliases: string[];
7+
};
48

59
/**
610
* The instructions class is used mainly by parsers.
@@ -9,13 +13,8 @@ import randomId from "../middleware/randomId"
913
* a web page content.
1014
*/
1115
export default class Instructions {
12-
declare id: string;
13-
1416
declare source: { id: string; };
15-
declare url: {
16-
url: string;
17-
aliases: string[];
18-
}[];
17+
declare url: InstructionUrl[];
1918
declare parserType: ParserType;
2019
declare endPoint: string;
2120
declare amount: number;
@@ -26,15 +25,11 @@ export default class Instructions {
2625
declare ignoreCertificates: boolean;
2726
declare extraFields: string[];
2827

29-
constructor() {
30-
this.id = randomId("ins");
31-
}
32-
3328
/**
3429
* Return the source that variable source refers to.
3530
* @return Source
3631
*/
3732
getSource(): Source {
38-
return Source.getSourceFrom(this.source.id);
33+
return Source.getSourceFrom(this);
3934
}
4035
}

‎src/components/job.ts

+2-8
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ export default class Job {
1717
id: string;
1818
};
1919

20-
constructor() {
20+
private constructor() {
2121
this.id = randomId("job");
2222

2323
this.attempts = 0;
@@ -33,12 +33,9 @@ export default class Job {
3333
static createJob(sourceId: string, workerId: string, interval: number): Job {
3434
let job = new Job();
3535
job.source = {id: sourceId};
36-
// nextRetry = The time the job finished (just now) + interval + randomTIme
3736
job.untilRetry = interval + this.getRandomTime(sourceId);
3837
job.worker = {id: workerId};
3938
job.status = JobStatus.PENDING;
40-
job.attempts = 0;
41-
job.emitAttempts = 0;
4239
return job;
4340
}
4441

@@ -54,10 +51,7 @@ export default class Job {
5451
const low = 0;
5552

5653
const random = Math.floor(Math.random() * (high - low) + low) * 1000;
57-
if (Math.random() >= 0.5)
58-
return random;
59-
else
60-
return -random;
54+
return Math.random() >= 0.5 ? random : -random;
6155
}
6256

6357
/**

‎src/components/payload.entity.ts

-4
This file was deleted.

‎src/components/source.ts

+52-59
Original file line numberDiff line numberDiff line change
@@ -20,65 +20,61 @@ export default class Source {
2020
declare extra: any;
2121
private declare id: string;
2222

23-
constructor() {
24-
}
25-
2623
/**
2724
* Parse and store a source file contents to an array in memory
2825
* @param source the source file
2926
*/
3027
static fileToSource(source: any): Source {
31-
source.filename = source.filename ? source.filename : source.name ? source.name : '[unknown filename]';
28+
source.filename = source.filename ? source.filename : source.name ? source.name : 'unknown filename';
3229

3330
let ret = new Source();
3431

3532
// Source name
36-
if (!source.name || source.name.length < 3)
37-
throw new Error(`SourceException: ${source.filename}: name: is not valid, must be type string with a least 3 characters.`);
33+
if (source.name == null || source.name.length < 3)
34+
throw new Error(`SourceException: [${source.filename}] Field name is not valid, requirements(type = string, length >= 3).`);
3835
ret.name = source.name;
3936

4037
// Source tableName
41-
if (source.tableName && (typeof source.tableName !== 'string' || source.tableName.length < 3))
42-
throw new Error(`SourceException: ${source.filename}: tableName: is not valid, must be type string with a least 3 characters.`);
43-
if (["saffron", "config", "workers"].includes(source.tableName))
44-
throw new Error(`SourceException: ${source.filename}: tableName: is blacklisted.`);
38+
if (source.tableName != null && (typeof source.tableName !== 'string' || source.tableName.length < 3 || ["saffron", "config", "workers"].includes(source.tableName)))
39+
throw new Error(`SourceException: [${source.filename}] Field tableName is not valid, requirements(type = string, length >= 3, != saffron, != workers, != config).`);
4540
ret.tableName = source.tableName;
4641

47-
if (source.interval && (typeof source.interval != 'number' || source.interval < 0))
48-
throw new Error(`SourceException: ${source.filename}: interval: is not valid, must be a positive number.`);
49-
ret.interval = source.interval;
42+
if (source.interval != null && (typeof source.interval != 'number' || source.interval < 0))
43+
throw new Error(`SourceException: [${source.filename}] Field interval is not valid, requirements(type = number, positive or zero).`);
44+
ret.interval = source.interval ? source.interval : Config.getOption(ConfigOptions.SCHEDULER_JOB_INT);
45+
46+
if (source.retryInterval != null && (typeof source.retryInterval != 'number' || source.retryInterval < 0))
47+
throw new Error(`SourceException: [${source.filename}] Field retryInterval is not valid, requirements(type = number, positive or zero).`);
48+
ret.retryInterval = source.retryInterval ? source.retryInterval : Config.getOption(ConfigOptions.SCHEDULER_JOB_INT) / 2;
5049

51-
if (source.retryInterval && (typeof source.retryInterval != 'number' || source.retryInterval < 0))
52-
throw new Error(`SourceException: ${source.filename}: retryInterval: is not valid, must be a positive number.`);
53-
ret.retryInterval = source.retryInterval;
50+
if (source.timeout != null && (typeof source.timeout != 'number' || source.timeout < 0))
51+
throw new Error(`SourceException: [${source.filename}] Field timeout is not valid, requirements(type = number, positive or zero).`);
52+
ret.timeout = source.timeout ? source.timeout : Config.getOption(ConfigOptions.REQUEST_TIMEOUT);
5453

55-
if (source.timeout && (typeof source.timeout != 'number' || source.timeout < 0))
56-
throw new Error(`SourceException: ${source.filename}: timeout: is not valid, must be a positive number.`);
57-
ret.timeout = source.timeout ? source.timeout : Config.getOption(ConfigOptions.REQUEST_TIMEOUT)
54+
ret.extra = source.extra;
5855

59-
// If it is not one time scrape:
60-
ret.instructions = new Instructions()
61-
ret.instructions.source = {id: ret.getId()}
56+
const instructions = new Instructions();
57+
ret.instructions = instructions;
58+
instructions.source = {id: ret.getId()}
6259

63-
if (source.amount && (typeof source.amount != 'number' || source.amount <= 0))
64-
throw new Error(`SourceException: ${source.filename}: amount: is not valid, must be a positive number.`);
65-
ret.instructions.amount = source.amount ? source.amount : Config.getOption(ConfigOptions.ARTICLE_AMOUNT)
60+
if (source.amount != null && (typeof source.amount != 'number' || source.amount <= 0))
61+
throw new Error(`SourceException: [${source.filename}] Field amount is not valid, requirements(type = number, positive).`);
62+
instructions.amount = source.amount ? source.amount : Config.getOption(ConfigOptions.ARTICLE_AMOUNT);
6663

67-
if (typeof source.ignoreCertificates !== 'undefined' && typeof source.ignoreCertificates !== 'boolean')
68-
throw new Error(`SourceException: ${source.filename}: ignoreCertificates: is not valid, must be boolean.`);
69-
ret.instructions.ignoreCertificates = source.ignoreCertificates ? source.ignoreCertificates : false;
64+
if (source.ignoreCertificates != null && typeof source.ignoreCertificates !== 'boolean')
65+
throw new Error(`SourceException: [${source.filename}] Field ignoreCertificates is not valid, requirements(type = boolean).`);
66+
instructions.ignoreCertificates = source.ignoreCertificates ? source.ignoreCertificates : false;
7067

71-
if (typeof source.encoding !== 'undefined' && typeof source.encoding !== 'string')
72-
throw new Error(`SourceException: ${source.filename}: encoding: is not valid, must be string value.`);
73-
ret.instructions.textDecoder = source.encoding ? new TextDecoder(`${source.encoding}`) : new TextDecoder();
68+
if (source.encoding != null && typeof source.encoding !== 'string')
69+
throw new Error(`SourceException: [${source.filename}] Field encoding is not valid requirements(type = string).`);
70+
instructions.textDecoder = source.encoding ? new TextDecoder(`${source.encoding}`) : new TextDecoder();
7471

75-
ret.extra = source.extra;
7672

77-
ret.instructions.url = [];
73+
instructions.url = [];
7874
if (typeof source.url === 'string') {
7975
if (source.url.length == 0)
80-
throw new Error(`SourceException: ${source.filename}: url: is not valid, url cannot be empty.`);
81-
ret.instructions.url.push({url: source.url, aliases: []});
76+
throw new Error(`SourceException: [${source.filename}] Field url is not valid, requirements(type = string, not empty).`);
77+
instructions.url.push({url: source.url, aliases: []});
8278
} else if (Array.isArray(source.url)) {
8379
for (const pair of source.url) {
8480
if (Array.isArray(pair) && pair.length >= 2) {
@@ -87,65 +83,62 @@ export default class Source {
8783

8884
aliases.forEach(alias => {
8985
if (typeof alias !== 'string' || alias.trim() === '')
90-
throw new Error(`SourceException: ${source.filename}: url: is not valid, invalid alias '${alias}'.`);
86+
throw new Error(`SourceException: [${source.filename}] At field url, field alias is not valid, requirements(type = string, not empty, not whitespace).`);
9187
});
9288

9389
if (typeof url !== 'string' || url.trim() === '')
94-
throw new Error(`SourceException: ${source.filename}: url: is not valid, invalid url '${url}'.`);
90+
throw new Error(`SourceException: [${source.filename}] At field url, field url is not valid, requirements(type = string, not empty not whitespace).`);
9591

96-
ret.instructions.url.push({url, aliases});
92+
instructions.url.push({url, aliases});
9793
} else if (typeof pair === 'string' || ((Array.isArray(pair) && pair.length == 1))) {
9894
let url: any = pair;
9995
if (Array.isArray(pair)) url = pair[0];
10096

101-
ret.instructions.url.push({url, aliases: []});
102-
} else throw new Error(`SourceException: ${source.filename}: url: is not valid, error during parsing pair: ${pair}.`);
97+
instructions.url.push({url, aliases: []});
98+
} else
99+
throw new Error(`SourceException: [${source.filename}] Field url is not valid, requirements(type = string[] | string[][]).`);
103100
}
104101
} else
105-
throw new Error(`SourceException: ${source.filename}: url: is not valid, must be a string type or an array.`)
102+
throw new Error(`SourceException: [${source.filename}] Field url is not valid, requirements(type = string | string[] | string[][]).`);
106103

107-
let parserType = ParserType.getFromString(source.type)
104+
let parserType = ParserType.getFromString(source.type);
108105
if (parserType === ParserType.UNKNOWN)
109-
throw new Error(`SourceException: ${source.filename}: type: is not valid.`);
110-
ret.instructions.parserType = parserType;
106+
throw new Error(`SourceException: [${source.filename}] Field type is not valid, requirements(equals html, rss, dynamic, wordpress-v1, wordpress-v2).`);
107+
instructions.parserType = parserType;
111108

112109
try {
113110
ParserLoader.validateScrapeOptions(parserType, source.scrape);
114111
} catch (e: any) {
115-
throw new Error(`SourceException: ${source.filename}: invalid scrape method, parser error: ${e.message}`);
112+
throw new Error(`SourceException: [${source.filename}] Field scrape is not valid, parser error: ${e.message}`);
116113
}
117114

118115
ParserLoader.assignScrapeInstructions(parserType, ret.instructions, source);
119116

120-
return ret
117+
return ret;
121118
}
122119

123120
static pushSource(source: Source) {
124121
this._sources.push(source);
125122
}
126123

127124
/**
128-
* Return a copy array of the sources
125+
* Return an array of the parsed sources.
129126
*/
130127
static getSources(): Source[] {
131-
return this._sources
128+
return this._sources;
132129
}
133130

134131
/**
135132
* Return the source class based on job, article or source id
136133
* @param from
137134
*/
138-
static getSourceFrom(from: Job | Article | string): Source {
139-
if (from instanceof Job)
140-
return this._sources.find((source: Source) => {
141-
return source.getId() === from.source?.id
142-
})!!
143-
else if (from instanceof Article)
144-
return this._sources.find((source: Source) => {
145-
return source.getId() === from.source?.id
146-
})!!
147-
148-
return this._sources.find((source: Source) => source.getId() === from)!!
135+
static getSourceFrom(from: Job | Article | Instructions | string): Source {
136+
let id = '';
137+
if (from instanceof Job || from instanceof Article || from instanceof Instructions)
138+
id = from.source?.id;
139+
else id = from;
140+
141+
return this._sources.find(source => source.getId() === id)!;
149142
}
150143

151144
/**

‎src/index.ts

+1-2
Original file line numberDiff line numberDiff line change
@@ -113,8 +113,7 @@ export default class Saffron {
113113
let source: Source = await Source.fileToSource(sourceJson);
114114
source.instructions.getSource = (): Source => source;
115115

116-
let job = new Job();
117-
job.source = {id: source.getId()};
116+
let job = Job.createJob(source.getId(), '', 0);
118117
job.getSource = (): Source => source;
119118
job.getInstructions = (): Instructions => source.instructions;
120119

‎src/modules/grid/index.ts

+6-5
Original file line numberDiff line numberDiff line change
@@ -248,18 +248,19 @@ export default class Grid {
248248

249249
let getExtPair = Extensions.getInstance().startCount();
250250
let pair: any = {};
251-
try {
252-
while ((pair = getExtPair()) != null) {
251+
252+
while ((pair = getExtPair()) != null) {
253+
try {
253254
if (pair.event === 'article.format') {
254255
for (const i in articles)
255256
articles[i] = await pair.callback(articles[i]);
256257
} else if (pair.event === 'articles') {
257258
articles = await pair.callback(articles);
258259
}
260+
} catch (e) {
261+
Events.emit("middleware.error", pair.event, e);
262+
return;
259263
}
260-
} catch (e) {
261-
Events.emit("middleware.error", pair.event, e);
262-
return;
263264
}
264265

265266
Events.emit("middleware.after", articles);

‎src/modules/parsers/Utils.ts

+1-2
Original file line numberDiff line numberDiff line change
@@ -323,8 +323,7 @@ export default class Utils {
323323
let source: Source = await Source.fileToSource(sourceJson);
324324
source.instructions.getSource = (): Source => source;
325325

326-
let job = new Job();
327-
job.source = {id: source.getId()};
326+
let job = Job.createJob(source.getId(), '', 0);
328327
job.getSource = (): Source => source;
329328
job.getInstructions = (): Instructions => source.instructions;
330329

‎src/modules/scheduler/index.ts

+1-4
Original file line numberDiff line numberDiff line change
@@ -37,11 +37,8 @@ export default class Scheduler {
3737
* @return Return the issued job id
3838
*/
3939
issueJobForSource(source: Source, lasWorkerId: string = "", interval: number = -1): void {
40-
if (interval == -1)
41-
interval = source.interval ? source.interval : Config.getOption(ConfigOptions.SCHEDULER_JOB_INT);
42-
4340
let worker = Worker.electWorker(lasWorkerId);
44-
let nJob = Job.createJob(source.getId(), worker, interval);
41+
let nJob = Job.createJob(source.getId(), worker, interval !== -1 ? interval : source.interval);
4542

4643
this.jobsStorage.push(nJob);
4744
Events.emit("scheduler.job.new", nJob);

0 commit comments

Comments
 (0)
Please sign in to comment.