Node FTP does not execute `once ('close')` callback when loading a large file

Question

I have a way to download a file from an FTP server and it works fine on small files, but when I use it to upload a ~ 5 GB file of type zip, it downloads it, but after that it does nothing, when it reaches 100 % load, the script does not continue. Should I wait if it really does something in the background after the download is complete? Is there a file size limit?

const FTP = require('ftp') 

which can be found on npm

 downloadFile: params => { return new Promise((resolve, reject) => { let ftpClient = new FTP() let total = params.state.fileSize let progress = 0 ftpClient.on('ready', _ => { console.log(`Downloading ${params.targetedFile} ...`); ftpClient.get(params.targetedFile, (err, stream) => { if (err) reject(err) stream.on('data', buffer => { progress += buffer.length process.stdout.write(`Progress: ${(progress/total*100).toFixed(2)}% (${progress}/${total}) \r`) }) stream.once('close', _ => { ftpClient.end() console.log(`Saved downloaded file to ${params.localDir}`); resolve(params.localDir) }) stream.pipe(fs.createWriteStream(params.localDir)) }) }) ftpClient.connect(params.auth) }) } 

Basically, the callback for stream.once('close', ...) fails when a large file is uploaded. And it runs for a smaller file of the same type.

Answer 1

This code can give you an idea of ​​how to deal with this slightly hacked way.

Basically, this method allows you to download a file from an FTP server and save it to the local file system. It displays the current progress of complete_percentage% (current/total) on one line. Upon completion, it resolves the promise, returns the path to the local file, the same one that you passed as param.

 /** * @name downloadFile * @desc downloads file from FTP server * @param params, Object of params * @prop auth: object, or null, authorization params * @prop targetedFile: {String} filename eg data.txt * @prop localDir: {String} filename on local disk * @prop state: {Object} fileinfo object, {Int} .fileSize property is required * @return Promise, resolves given localDir */ downloadFile: params => { return new Promise((resolve, reject) => { // validate param types if(typeof params.auth !== 'object' || typeof params.targetedFile !== 'string' || typeof params.localDir !== 'string' || typeof params.state !== 'object' || typeof params.state.fileSize !== 'number' ) throw new Error('You are either missing properties or passed wrong types') // initialize let ftpClient = new FTP() let total = params.state.fileSize let progress = 0 // ftpClient.on('ready', _ => { console.log(`Downloading ${params.targetedFile} ...`) // get file ftpClient.get(params.targetedFile, (err, stream) => { if (err){ ftpClient.end() return reject(err) } // upon data receive stream.on('data', buffer => { progress += buffer.length // if progress is complete if(progress === total){ // start checking if local filesize matches server filesize let interval = setInterval(_ => { if(fs.statSync(params.localDir).size === total){ console.log(`Downloading file complete. Location: ${params.localDir}`); clearInterval(interval) ftpClient.end() resolve(params.localDir) } }) } // show current progress in percentages and bytes process.stdout.write(`Progress: ${(progress/total*100).toFixed(2)}% (${progress}/${total}) \r`) }) // pipe writestream to filesystem to write these bytes stream.pipe(fs.createWriteStream(params.localDir)) }) }) ftpClient.connect(params.auth) })//promise } 

Answer 2

I recommend that you handle the write stream closing event.

The reason is simple: we read from the read stream and the ftp stream to write the stream, everything is fine when the file is successfully closed.

So the code:

 downloadFile: params => { return new Promise((resolve, reject) => { let ftpClient = new FTP() let total = params.state.fileSize let progress = 0 ftpClient.on('ready', _ => { console.log(`Downloading ${params.targetedFile} ...`); ftpClient.get(params.targetedFile, (err, stream) => { if (err) { ftpClient.end(); return reject(err); } stream.on('data', buffer => { progress += buffer.length process.stdout.write(`Progress: ${(progress/total*100).toFixed(2)}% (${progress}/${total}) \r`) }); // opening writeStream to file let finished = false; const writeStream = fs.createWriteStream(params.localDir); writeStream.on('finish', (result) => { // handling finish finished = true; ftpClient.end(); console.log(`Finish triggered ${params.localDir}`); console.log(result); resolve(params.localDir); }); writeStream.on('close', (result) => { // handling close ftpClient.end(); console.log(`Close triggered ${params.localDir}`); console.log(result); resolve(params.localDir); }) // piping readStream to writeStream stream.pipe(writeStream); }) }) ftpClient.connect(params.auth) }) }