Skip to content

Commit 32f2d6a

Browse files
vpluschenpluschenalexander-fenster
authoredJan 24, 2023
feat(cli): generate static files at the granularity of proto messages (#1840)
* feat: add message filter for cli * feat: add test * fix: update comment * fix: update error message * fix: remove test file * fix: lint, jsdoc comments, return values Co-authored-by: pluschen <pluschen@tencent.com> Co-authored-by: Alexander Fenster <fenster@google.com>
·
protobufjs-v7.4.0protobufjs-cli-v1.1.0
1 parent ea7b9a6 commit 32f2d6a

File tree

7 files changed

+225
-1
lines changed

7 files changed

+225
-1
lines changed
 

‎cli/pbjs.js

+17-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ exports.main = function main(args, callback) {
4040
"force-long": "strict-long",
4141
"force-message": "strict-message"
4242
},
43-
string: [ "target", "out", "path", "wrap", "dependency", "root", "lint" ],
43+
string: [ "target", "out", "path", "wrap", "dependency", "root", "lint", "filter" ],
4444
boolean: [ "create", "encode", "decode", "verify", "convert", "delimited", "typeurl", "beautify", "comments", "service", "es6", "sparse", "keep-case", "alt-comment", "force-long", "force-number", "force-enum-string", "force-message", "null-defaults" ],
4545
default: {
4646
target: "json",
@@ -98,6 +98,9 @@ exports.main = function main(args, callback) {
9898
"",
9999
" -p, --path Adds a directory to the include path.",
100100
"",
101+
" --filter Set up a filter to configure only those messages you need and their dependencies to compile, this will effectively reduce the final file size",
102+
" Set A json file path, Example of file content: {\"messageNames\":[\"mypackage.messageName1\", \"messageName2\"] } ",
103+
"",
101104
" -o, --out Saves to a file instead of writing to stdout.",
102105
"",
103106
" --sparse Exports only those types referenced from a main file (experimental).",
@@ -308,7 +311,20 @@ exports.main = function main(args, callback) {
308311
root.resolveAll();
309312
}
310313

314+
function filterMessage() {
315+
if (argv.filter) {
316+
// This is a piece of degradable logic
317+
try {
318+
const needMessage = JSON.parse(fs.readFileSync(argv.filter));
319+
util.filterMessage(root, needMessage);
320+
} catch (error) {
321+
process.stderr.write(`The filter not work, please check whether the file is correct: ${error.message}\n`);
322+
}
323+
}
324+
}
325+
311326
function callTarget() {
327+
filterMessage();
312328
target(root, argv, function targetCallback(err, output) {
313329
if (err) {
314330
if (callback)

‎cli/util.js

+116
Original file line numberDiff line numberDiff line change
@@ -125,3 +125,119 @@ exports.pad = function(str, len, l) {
125125
return str;
126126
};
127127

128+
129+
/**
130+
* DFS to get all message dependencies, cache in filterMap.
131+
* @param {Root} root The protobuf root instance
132+
* @param {Message} message The message need to process.
133+
* @param {Map} filterMap The result of message you need and their dependencies.
134+
* @param {Map} flatMap A flag to record whether the message was searched.
135+
* @returns {undefined} Does not return a value
136+
*/
137+
function dfsFilterMessageDependencies(root, message, filterMap, flatMap) {
138+
if (message instanceof protobuf.Type) {
139+
if (flatMap.get(`${message.fullName}`)) return;
140+
flatMap.set(`${message.fullName}`, true);
141+
for (var field of message.fieldsArray) {
142+
if (field.resolvedType) {
143+
// a nested message
144+
if (field.resolvedType.parent.name === message.name) {
145+
var nestedMessage = message.nested[field.resolvedType.name];
146+
dfsFilterMessageDependencies(root, nestedMessage, filterMap, flatMap);
147+
continue;
148+
}
149+
var packageName = field.resolvedType.parent.name;
150+
var typeName = field.resolvedType.name;
151+
var fullName = packageName ? `${packageName}.${typeName}` : typeName;
152+
doFilterMessage(root, { messageNames: [fullName] }, filterMap, flatMap, packageName);
153+
}
154+
}
155+
}
156+
}
157+
158+
/**
159+
* DFS to get all message you need and their dependencies, cache in filterMap.
160+
* @param {Root} root The protobuf root instance
161+
* @param {object} needMessageConfig Need message config:
162+
* @param {string[]} needMessageConfig.messageNames The message names array in the root namespace you need to gen. example: [msg1, msg2]
163+
* @param {Map} filterMap The result of message you need and their dependencies.
164+
* @param {Map} flatMap A flag to record whether the message was searched.
165+
* @param {string} currentPackageName Current package name
166+
* @returns {undefined} Does not return a value
167+
*/
168+
function doFilterMessage(root, needMessageConfig, filterMap, flatMap, currentPackageName) {
169+
var needMessageNames = needMessageConfig.messageNames;
170+
171+
for (var messageFullName of needMessageNames) {
172+
var nameSplit = messageFullName.split(".");
173+
var packageName = "";
174+
var messageName = "";
175+
if (nameSplit.length > 1) {
176+
packageName = nameSplit[0];
177+
messageName = nameSplit[1];
178+
} else {
179+
messageName = nameSplit[0];
180+
}
181+
182+
// in Namespace
183+
if (packageName) {
184+
var ns = root.nested[packageName];
185+
if (!ns || !(ns instanceof protobuf.Namespace)) {
186+
throw new Error(`package not foud ${currentPackageName}.${messageName}`);
187+
}
188+
189+
doFilterMessage(root, { messageNames: [messageName] }, filterMap, flatMap, packageName);
190+
} else {
191+
var message = root.nested[messageName];
192+
193+
if (currentPackageName) {
194+
message = root.nested[currentPackageName].nested[messageName];
195+
}
196+
197+
if (!message) {
198+
throw new Error(`message not foud ${currentPackageName}.${messageName}`);
199+
}
200+
201+
var set = filterMap.get(currentPackageName);
202+
if (!filterMap.has(currentPackageName)) {
203+
set = new Set();
204+
filterMap.set(currentPackageName, set);
205+
}
206+
207+
set.add(messageName);
208+
209+
// dfs to find all dependencies
210+
dfsFilterMessageDependencies(root, message, filterMap, flatMap, currentPackageName);
211+
}
212+
}
213+
}
214+
215+
/**
216+
* filter the message you need and their dependencies, all others will be delete from root.
217+
* @param {Root} root Root the protobuf root instance
218+
* @param {object} needMessageConfig Need message config:
219+
* @param {string[]} needMessageConfig.messageNames Tthe message names array in the root namespace you need to gen. example: [msg1, msg2]
220+
* @returns {boolean} True if a message should present in the generated files
221+
*/
222+
exports.filterMessage = function (root, needMessageConfig) {
223+
var filterMap = new Map();
224+
var flatMap = new Map();
225+
doFilterMessage(root, needMessageConfig, filterMap, flatMap, "");
226+
root._nestedArray = root._nestedArray.filter(ns => {
227+
if (ns instanceof protobuf.Type || ns instanceof protobuf.Enum) {
228+
return filterMap.get("").has(ns.name);
229+
} else if (ns instanceof protobuf.Namespace) {
230+
if (!filterMap.has(ns.name)) {
231+
return false;
232+
}
233+
ns._nestedArray = ns._nestedArray.filter(nns => {
234+
const nnsSet = filterMap.get(ns.name);
235+
return nnsSet.has(nns.name);
236+
});
237+
238+
return true;
239+
}
240+
return true;
241+
});
242+
};
243+

‎package.json

+8
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,14 @@
1111
"engines": {
1212
"node": ">=12.0.0"
1313
},
14+
"eslintConfig": {
15+
"env": {
16+
"es6": true
17+
},
18+
"parserOptions": {
19+
"ecmaVersion": 6
20+
}
21+
},
1422
"keywords": [
1523
"protobuf",
1624
"protocol-buffers",

‎tests/cli.js

+52
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ var tape = require("tape");
44
var path = require("path");
55
var Module = require("module");
66
var protobuf = require("..");
7+
var fs = require("fs");
78

89
function cliTest(test, testFunc) {
910
// pbjs does not seem to work with Node v4, so skip this test if we're running on it
@@ -162,3 +163,54 @@ tape.test("with null-defaults, absent optional fields have null values", functio
162163
});
163164
});
164165
});
166+
167+
168+
tape.test("pbjs generates static code with message filter", function (test) {
169+
cliTest(test, function () {
170+
var root = protobuf.loadSync("tests/data/cli/test-filter.proto");
171+
root.resolveAll();
172+
173+
var staticTarget = require("../cli/targets/static");
174+
var util = require("../cli/util");
175+
176+
const needMessageConfig = JSON.parse(fs.readFileSync("tests/data/cli/filter.json"));
177+
178+
util.filterMessage(root, needMessageConfig);
179+
180+
staticTarget(root, {
181+
create: true,
182+
decode: true,
183+
encode: true,
184+
convert: true,
185+
"null-defaults": true,
186+
}, function (err, jsCode) {
187+
test.error(err, 'static code generation worked');
188+
189+
// jsCode is the generated code; we'll eval it
190+
// (since this is what we normally does with the code, right?)
191+
// This is a test code. Do not use this in production.
192+
var $protobuf = protobuf;
193+
eval(jsCode);
194+
195+
console.log(protobuf.roots);
196+
197+
var NeedMessage1 = protobuf.roots.default.filtertest.NeedMessage1;
198+
var NeedMessage2 = protobuf.roots.default.filtertest.NeedMessage2;
199+
var DependentMessage1 = protobuf.roots.default.filtertest.DependentMessage1;
200+
var DependentMessageFromImport = protobuf.roots.default.DependentMessageFromImport;
201+
202+
var NotNeedMessageInRootFile = protobuf.roots.default.filtertest.NotNeedMessageInRootFile;
203+
var NotNeedMessageInImportFile = protobuf.roots.default.NotNeedMessageInImportFile;
204+
205+
test.ok(NeedMessage1, "NeedMessage1 is loaded");
206+
test.ok(NeedMessage2, "NeedMessage2 is loaded");
207+
test.ok(DependentMessage1, "DependentMessage1 is loaded");
208+
test.ok(DependentMessageFromImport, "DependentMessageFromImport is loaded");
209+
210+
test.notOk(NotNeedMessageInImportFile, "NotNeedMessageInImportFile is not loaded");
211+
test.notOk(NotNeedMessageInRootFile, "NotNeedMessageInRootFile is not loaded");
212+
213+
test.end();
214+
});
215+
});
216+
});

‎tests/data/cli/filter.json

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"messageNames": ["filtertest.NeedMessage1", "filtertest.NeedMessage2"]
3+
}

‎tests/data/cli/test-filter-import.proto

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
message DependentMessageFromImport {
3+
optional int32 test1 = 1;
4+
}
5+
6+
message NotNeedMessageInImportFile {
7+
optional int32 test1 = 1;
8+
}

‎tests/data/cli/test-filter.proto

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package filtertest;
2+
import "./test-filter-import.proto";
3+
4+
message NeedMessage1 {
5+
optional uint32 test1 = 1;
6+
optional NeedMessage2 needMessage2 = 2;
7+
optional DependentMessage1 dependentMessage1 = 3;
8+
optional DependentMessageFromImport dependentMessage2 = 4;
9+
}
10+
11+
message NeedMessage2 {
12+
optional uint32 test1 = 1;
13+
}
14+
15+
message DependentMessage1 {
16+
optional uint32 test1 = 1;
17+
}
18+
19+
message NotNeedMessageInRootFile {
20+
optional uint32 test1 = 1;
21+
}

0 commit comments

Comments
 (0)
Please sign in to comment.