Skip to content

Commit fdbc801

Browse files
ebozdumanebozduman
authoredNov 5, 2020
Replaces the old xml parser with a new, faster one (#861)
·
8.0.57.0.17
1 parent ffd3fdb commit fdbc801

File tree

4 files changed

+118
-82
lines changed

4 files changed

+118
-82
lines changed
 

‎package.json

+4-4
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,13 @@
4141
"querystring": "0.2.0",
4242
"through2": "^3.0.1",
4343
"xml": "^1.0.0",
44-
"xml2js": "^0.4.15"
44+
"fast-xml-parser": "^3.17.4"
4545
},
4646
"devDependencies": {
47-
"@babel/core": "^7.5.5",
48-
"@babel/preset-env": "^7.5.5",
47+
"@babel/core": "^7.11.0",
48+
"@babel/preset-env": "^7.11.0",
4949
"babelify": "^10.0.0",
50-
"browserify": "^16.3.0",
50+
"browserify": "^16.5.1",
5151
"chai": "^4.2.0",
5252
"eslint": "^6.1.0",
5353
"gulp": "^4.0.2",

‎src/main/minio.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import Https from 'https'
2121
import Stream from 'stream'
2222
import BlockStream2 from 'block-stream2'
2323
import Xml from 'xml'
24-
import xml2js from 'xml2js'
24+
import fxp from 'fast-xml-parser'
2525
import async from 'async'
2626
import querystring from 'querystring'
2727
import mkdirp from 'mkdirp'
@@ -2066,7 +2066,7 @@ export class Client {
20662066
}
20672067
var method = 'PUT'
20682068
var query = 'notification'
2069-
var builder = new xml2js.Builder({rootName:'NotificationConfiguration', renderOpts:{'pretty':false}, headless:true})
2069+
var builder = new fxp.Builder({rootName:'NotificationConfiguration', renderOpts:{'pretty':false}, headless:true})
20702070
var payload = builder.buildObject(config)
20712071
this.makeRequest({method, bucketName, query}, payload, 200, '', false, cb)
20722072
}

‎src/main/xml-parsers.js

+111-76
Original file line numberDiff line numberDiff line change
@@ -14,49 +14,35 @@
1414
* limitations under the License.
1515
*/
1616

17-
import xml2js from 'xml2js'
17+
import fxp from 'fast-xml-parser'
1818
import _ from 'lodash'
1919
import * as errors from './errors.js'
2020

21-
var options = { // options passed to xml2js parser
22-
explicitRoot: false, // return the root node in the resulting object?
23-
ignoreAttrs: true, // ignore attributes, only create text nodes
24-
}
25-
2621
var parseXml = (xml) => {
2722
var result = null
28-
var error = null
29-
30-
var parser = new xml2js.Parser(options)
31-
parser.parseString(xml, function (e, r) {
32-
error = e
33-
result = r
34-
})
35-
36-
if (error) {
37-
throw new Error('XML parse error')
23+
result = fxp.parse(xml)
24+
if (result.Error) {
25+
throw result.Error
3826
}
27+
3928
return result
4029
}
4130

4231
// Parse XML and return information as Javascript types
4332

4433
// parse error XML response
4534
export function parseError(xml, headerInfo) {
46-
var xmlError = {}
47-
var xmlobj = parseXml(xml)
48-
var message
49-
_.each(xmlobj, (n, key) => {
50-
if (key === 'Message') {
51-
message = xmlobj[key][0]
52-
return
53-
}
54-
xmlError[key.toLowerCase()] = xmlobj[key][0]
55-
})
56-
var e = new errors.S3Error(message)
57-
_.each(xmlError, (value, key) => {
58-
e[key] = value
35+
var xmlErr = {}
36+
var xmlObj = fxp.parse(xml)
37+
if (xmlObj.Error) {
38+
xmlErr = xmlObj.Error
39+
}
40+
41+
var e = new errors.S3Error()
42+
_.each(xmlErr, (value, key) => {
43+
e[key.toLowerCase()] = value
5944
})
45+
6046
_.each(headerInfo, (value, key) => {
6147
e[key] = value
6248
})
@@ -69,11 +55,16 @@ export function parseCopyObject(xml) {
6955
etag: "",
7056
lastModified: ""
7157
}
58+
7259
var xmlobj = parseXml(xml)
73-
if (xmlobj.ETag) result.etag = xmlobj.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
60+
if (!xmlobj.CopyObjectResult) {
61+
throw new errors.InvalidXMLError('Missing tag: "CopyObjectResult"')
62+
}
63+
xmlobj = xmlobj.CopyObjectResult
64+
if (xmlobj.ETag) result.etag = xmlobj.ETag.replace(/^"/g, '').replace(/"$/g, '')
7465
.replace(/^"/g, '').replace(/"$/g, '')
75-
.replace(/^"/g, '').replace(/^"$/g, '')
76-
if (xmlobj.LastModified) result.lastModified = new Date(xmlobj.LastModified[0])
66+
.replace(/^"/g, '').replace(/"$/g, '')
67+
if (xmlobj.LastModified) result.lastModified = new Date(xmlobj.LastModified)
7768

7869
return result
7970
}
@@ -85,32 +76,54 @@ export function parseListMultipart(xml) {
8576
prefixes: [],
8677
isTruncated: false
8778
}
88-
var xmlobj = parseXml(xml)
89-
if (xmlobj.IsTruncated && xmlobj.IsTruncated[0] === 'true') result.isTruncated = true
90-
if (xmlobj.NextKeyMarker) result.nextKeyMarker = xmlobj.NextKeyMarker[0]
79+
80+
var xmlobj = parseXml(xml)
81+
82+
if (!xmlobj.ListMultipartUploadsResult) {
83+
throw new errors.InvalidXMLError('Missing tag: "ListMultipartUploadsResult"')
84+
}
85+
xmlobj = xmlobj.ListMultipartUploadsResult
86+
if (xmlobj.IsTruncated && xmlobj.IsTruncated === 'true') result.isTruncated = true
87+
if (xmlobj.NextKeyMarker) result.nextKeyMarker = xmlobj.NextKeyMarker
9188
if (xmlobj.NextUploadIdMarker) result.nextUploadIdMarker = xmlobj.NextUploadIdMarker[0]
9289
if (xmlobj.CommonPrefixes) xmlobj.CommonPrefixes.forEach(prefix => {
9390
result.prefixes.push({prefix: prefix[0]})
9491
})
95-
if (xmlobj.Upload) xmlobj.Upload.forEach(upload => {
96-
result.uploads.push({
97-
key: upload.Key[0],
98-
uploadId: upload.UploadId[0],
99-
initiated: new Date(upload.Initiated[0])
92+
if (xmlobj.Upload) {
93+
if (!Array.isArray(xmlobj.Upload)) {
94+
xmlobj.Upload = Array(xmlobj.Upload)
95+
}
96+
xmlobj.Upload.forEach(upload => {
97+
var key = upload.Key
98+
var uploadId = upload.UploadId
99+
var initiator = {id: upload.Initiator.ID, displayName: upload.Initiator.DisplayName}
100+
var owner = {id: upload.Owner.ID, displayName: upload.Owner.DisplayName}
101+
var storageClass = upload.StorageClass
102+
var initiated = new Date(upload.Initiated)
103+
result.uploads.push({key, uploadId, initiator, owner, storageClass, initiated})
100104
})
101-
})
105+
}
102106
return result
103107
}
104108

105109
// parse XML response to list all the owned buckets
106110
export function parseListBucket(xml) {
107111
var result = []
108112
var xmlobj = parseXml(xml)
113+
114+
if (!xmlobj.ListAllMyBucketsResult) {
115+
throw new errors.InvalidXMLError('Missing tag: "ListAllMyBucketsResult"')
116+
}
117+
xmlobj = xmlobj.ListAllMyBucketsResult
118+
109119
if (xmlobj.Buckets) {
110-
if (xmlobj.Buckets[0].Bucket) {
111-
xmlobj.Buckets[0].Bucket.forEach(bucket => {
112-
var name = bucket.Name[0]
113-
var creationDate = new Date(bucket.CreationDate[0])
120+
if (xmlobj.Buckets.Bucket) {
121+
if (!Array.isArray(xmlobj.Buckets.Bucket)) {
122+
xmlobj.Buckets.Bucket = Array(xmlobj.Buckets.Bucket)
123+
}
124+
xmlobj.Buckets.Bucket.forEach(bucket => {
125+
var name = bucket.Name
126+
var creationDate = new Date(bucket.CreationDate)
114127
result.push({name, creationDate})
115128
})
116129
}
@@ -149,7 +162,6 @@ export function parseBucketNotification(xml) {
149162
}
150163

151164
var xmlobj = parseXml(xml)
152-
153165
// Parse all topic configurations in the xml
154166
if (xmlobj.TopicConfiguration) {
155167
xmlobj.TopicConfiguration.forEach(config => {
@@ -186,7 +198,8 @@ export function parseBucketNotification(xml) {
186198

187199
// parse XML response for bucket region
188200
export function parseBucketRegion(xml) {
189-
return parseXml(xml)
201+
// return region information
202+
return parseXml(xml).LocationConstraint
190203
}
191204

192205
// parse XML response for list parts of an in progress multipart upload
@@ -197,15 +210,15 @@ export function parseListParts(xml) {
197210
parts: [],
198211
marker: undefined
199212
}
200-
if (xmlobj.IsTruncated && xmlobj.IsTruncated[0] === 'true') result.isTruncated = true
213+
if (xmlobj.IsTruncated && xmlobj.IsTruncated === 'true') result.isTruncated = true
201214
if (xmlobj.NextPartNumberMarker) result.marker = +xmlobj.NextPartNumberMarker[0]
202215
if (xmlobj.Part) {
203216
xmlobj.Part.forEach(p => {
204217
var part = +p.PartNumber[0]
205-
var lastModified = new Date(p.LastModified[0])
206-
var etag = p.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
218+
var lastModified = new Date(p.LastModified)
219+
var etag = p.ETag.replace(/^"/g, '').replace(/"$/g, '')
207220
.replace(/^"/g, '').replace(/"$/g, '')
208-
.replace(/^"/g, '').replace(/^"$/g, '')
221+
.replace(/^"/g, '').replace(/"$/g, '')
209222
result.parts.push({part, lastModified, etag})
210223
})
211224
}
@@ -214,21 +227,21 @@ export function parseListParts(xml) {
214227

215228
// parse XML response when a new multipart upload is initiated
216229
export function parseInitiateMultipart(xml) {
217-
var xmlobj = parseXml(xml)
218-
if (xmlobj.UploadId) return xmlobj.UploadId[0]
219-
throw new errors.InvalidXMLError('UploadId missing in XML')
230+
var xmlobj = parseXml(xml).InitiateMultipartUploadResult
231+
if (xmlobj.UploadId) return xmlobj.UploadId
232+
throw new errors.InvalidXMLError('Missing tag: "UploadId"')
220233
}
221234

222235
// parse XML response when a multipart upload is completed
223236
export function parseCompleteMultipart(xml) {
224-
var xmlobj = parseXml(xml)
237+
var xmlobj = parseXml(xml).CompleteMultipartUploadResult
225238
if (xmlobj.Location) {
226239
var location = xmlobj.Location[0]
227240
var bucket = xmlobj.Bucket[0]
228-
var key = xmlobj.Key[0]
229-
var etag = xmlobj.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
241+
var key = xmlobj.Key
242+
var etag = xmlobj.ETag.replace(/^"/g, '').replace(/"$/g, '')
230243
.replace(/^"/g, '').replace(/"$/g, '')
231-
.replace(/^"/g, '').replace(/^"$/g, '')
244+
.replace(/^"/g, '').replace(/"$/g, '')
232245

233246
return {location, bucket, key, etag}
234247
}
@@ -248,15 +261,23 @@ export function parseListObjects(xml) {
248261
}
249262
var nextMarker
250263
var xmlobj = parseXml(xml)
251-
if (xmlobj.IsTruncated && xmlobj.IsTruncated[0] === 'true') result.isTruncated = true
264+
265+
if (!xmlobj.ListBucketResult) {
266+
throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"')
267+
}
268+
xmlobj = xmlobj.ListBucketResult
269+
if (xmlobj.IsTruncated && xmlobj.IsTruncated === 'true') xmlobj.isTruncated = true
252270
if (xmlobj.Contents) {
271+
if (!Array.isArray(xmlobj.Contents)) {
272+
xmlobj.Contents = Array(xmlobj.Contents)
273+
}
253274
xmlobj.Contents.forEach(content => {
254-
var name = content.Key[0]
255-
var lastModified = new Date(content.LastModified[0])
256-
var etag = content.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
275+
var name = content.Key
276+
var lastModified = new Date(content.LastModified)
277+
var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
257278
.replace(/^"/g, '').replace(/"$/g, '')
258-
.replace(/^"/g, '').replace(/^"$/g, '')
259-
var size = +content.Size[0]
279+
.replace(/^"/g, '').replace(/"$/g, '')
280+
var size = +content.Size
260281
result.objects.push({name, lastModified, etag, size})
261282
nextMarker = name
262283
})
@@ -281,17 +302,24 @@ export function parseListObjectsV2(xml) {
281302
isTruncated: false
282303
}
283304
var xmlobj = parseXml(xml)
284-
if (xmlobj.IsTruncated && xmlobj.IsTruncated[0] === 'true') result.isTruncated = true
305+
if (!xmlobj.ListBucketResult) {
306+
throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"')
307+
}
308+
xmlobj = xmlobj.ListBucketResult
309+
if (xmlobj.IsTruncated && xmlobj.IsTruncated === 'true') result.isTruncated = true
285310
if (xmlobj.NextContinuationToken) result.nextContinuationToken = xmlobj.NextContinuationToken[0]
286311

287312
if (xmlobj.Contents) {
313+
if (!Array.isArray(xmlobj.Contents)) {
314+
xmlobj.Contents = Array(xmlobj.Contents)
315+
}
288316
xmlobj.Contents.forEach(content => {
289-
var name = content.Key[0]
290-
var lastModified = new Date(content.LastModified[0])
291-
var etag = content.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
317+
var name = content.Key
318+
var lastModified = new Date(content.LastModified)
319+
var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
292320
.replace(/^"/g, '').replace(/"$/g, '')
293-
.replace(/^"/g, '').replace(/^"$/g, '')
294-
var size = +content.Size[0]
321+
.replace(/^"/g, '').replace(/"$/g, '')
322+
var size = +content.Size
295323
result.objects.push({name, lastModified, etag, size})
296324
})
297325
}
@@ -312,17 +340,24 @@ export function parseListObjectsV2WithMetadata(xml) {
312340
isTruncated: false
313341
}
314342
var xmlobj = parseXml(xml)
315-
if (xmlobj.IsTruncated && xmlobj.IsTruncated[0] === 'true') result.isTruncated = true
343+
if (!xmlobj.ListBucketResult) {
344+
throw new errors.InvalidXMLError('Missing tag: "ListBucketResult"')
345+
}
346+
xmlobj = xmlobj.ListBucketResult
347+
if (xmlobj.IsTruncated && xmlobj.IsTruncated === 'true') result.isTruncated = true
316348
if (xmlobj.NextContinuationToken) result.nextContinuationToken = xmlobj.NextContinuationToken[0]
317349

318350
if (xmlobj.Contents) {
351+
if (!Array.isArray(xmlobj.Contents)) {
352+
xmlobj.Contents = Array(xmlobj.Contents)
353+
}
319354
xmlobj.Contents.forEach(content => {
320-
var name = content.Key[0]
321-
var lastModified = new Date(content.LastModified[0])
322-
var etag = content.ETag[0].replace(/^"/g, '').replace(/"$/g, '')
355+
var name = content.Key
356+
var lastModified = new Date(content.LastModified)
357+
var etag = content.ETag.replace(/^"/g, '').replace(/"$/g, '')
323358
.replace(/^"/g, '').replace(/"$/g, '')
324-
.replace(/^"/g, '').replace(/^"$/g, '')
325-
var size = +content.Size[0]
359+
.replace(/^"/g, '').replace(/"$/g, '')
360+
var size = +content.Size
326361
var metadata
327362
if (content.UserMetadata != null) {
328363
metadata = content.UserMetadata[0]

‎src/test/functional/functional-tests.js

+1
Original file line numberDiff line numberDiff line change
@@ -1068,6 +1068,7 @@ describe('functional tests', function() {
10681068
})
10691069

10701070
step(`listObjects(bucketName, prefix, recursive)_bucketName:${bucketName}, recursive:false_`, done => {
1071+
listArray = []
10711072
client.listObjects(bucketName, '', false)
10721073
.on('error', done)
10731074
.on('end', () => {

0 commit comments

Comments
 (0)
Please sign in to comment.