Fix parts too small with S3 datastore (#284)

Author: mitjap

lib/models/StreamSplitter.js

@@ -96,7 +96,7 @@ class FileStreamSplitter extends stream.Writable {
                     return reject(err);
                 }
 
-                this.emit('chunkFinished', this.currentChunkPath);
+                this.emit('chunkFinished', { path: this.currentChunkPath, size: this.currentChunkSize });
 
                 this.currentChunkPath = null;
                 this.fileDescriptor = null;

lib/stores/S3Store.js

@@ -299,10 +299,11 @@ class S3Store extends DataStore {
    * @param {Object}         metadata upload metadata
    * @param {fs<ReadStream>} readStream incoming request
    * @param {Number}         currentPartNumber number of the current part/chunk
+   * @param {Number}         current_size current size of uploaded data
    * @return {Promise<Number>} which resolves with the current offset
    * @memberof S3Store
    */
-    _processUpload(metadata, readStream, currentPartNumber) {
+    _processUpload(metadata, readStream, currentPartNumber, current_size) {
         return new Promise((resolve, reject) => {
             const splitterStream = new FileStreamSplitter({
                 maxChunkSize: this.part_size,
@@ -330,20 +331,27 @@ class S3Store extends DataStore {
                 pendingChunkFilepath = filepath;
             });
 
-            splitterStream.on('chunkFinished', (filepath) => {
+            splitterStream.on('chunkFinished', ({ path, size }) => {
                 pendingChunkFilepath = null;
 
+                current_size += size;
                 const partNumber = currentPartNumber++;
 
                 const p = Promise.resolve()
                     .then(() => {
-                        return this._uploadPart(metadata, fs.createReadStream(filepath), partNumber);
-                    })
+                        // skip chunk if it is not last and is smaller than 5MB
+                        const is_last_chunk = parseInt(metadata.file.upload_length, 10) === current_size;
+                        if (!is_last_chunk && size < 5 * 1024 * 1024) {
+                            log(`[${metadata.file.id}] ignoring chuck smaller than 5MB`);
+                            return undefined;
+                        }
 
+                        return this._uploadPart(metadata, fs.createReadStream(path), partNumber);
+                    })
                     .finally(() => {
-                        fs.rm(filepath, (err) => {
+                        fs.rm(path, (err) => {
                             if (err) {
-                                log(`[${metadata.file.id}] failed to remove file ${filepath}`, err);
+                                log(`[${metadata.file.id}] failed to remove file ${path}`, err);
                             }
                         });
                     });
@@ -486,17 +494,17 @@ class S3Store extends DataStore {
 
     write(req, file_id) {
         return this._getMetadata(file_id)
-            .then((metadata) =>
-                this._countParts(file_id).then((part_number) => [part_number, metadata])
-            )
+            .then((metadata) => {
+                return Promise.all([metadata, this._countParts(file_id), this.getOffset(file_id)]);
+            })
             .then(async(results) => {
-                const [part_number, metadata] = results;
+                const [metadata, part_number, initial_offset] = results;
                 const next_part_number = part_number + 1;
 
                 return Promise.allSettled(
-                    await this._processUpload(metadata, req, next_part_number)
+                    await this._processUpload(metadata, req, next_part_number, initial_offset.size)
                 )
-                    .then(() => this.getOffset(metadata.file.id, true))
+                    .then(() => this.getOffset(file_id))
                     .then((current_offset) => {
                         if (parseInt(metadata.file.upload_length, 10) === current_offset.size) {
                             return this._finishMultipartUpload(metadata, current_offset.parts)
@@ -509,12 +517,12 @@ class S3Store extends DataStore {
                                         }),
                                     });
 
-                                    this._clearCache(metadata.file.id);
+                                    this._clearCache(file_id);
 
                                     return current_offset.size;
                                 })
                                 .catch((err) => {
-                                    log(`[${metadata.file.id}] failed to finish upload`, err);
+                                    log(`[${file_id}] failed to finish upload`, err);
                                     throw err;
                                 });
                         }