I have code that uses fast-csv to parse a 2GB csv file and paste into MongoDB.
When Pipe executes this code for records around 17K +, Nodejs consumes all the memory and stops with an error:
"CALL_AND_RETRY_LAST Allocation failed - process out of memory"
I tried setting null for variables to avoid possible leakage.
I am using the latest version of Mongoose and Node.js 0.12 ..
PS: If I commented on the p.save () method and ran this code, its execution without any memory leak. I checked up to 5 million records and its good ...
So, I suspect a memory leak in the mongoose save function is possible.
var csv = require( "fast-csv" );
var fs = require( "fs" );
var DBMong = require( "./db-mongoose.js" );
function processCSVnImportDB( CSVFilePath , callback )
{
var rowcounter = 0;
var dbObj = DB.getDBObject( );
var stream = fs.createReadStream( CSVFilePath );
var csvStream;
csvStream = csv( )
.on( "data",
function ( data )
{
csvStream.pause( );
try
{
rowcounter++;
var p = new DBMong.ProductModel;
var su = new models.supplierinfo();
su.cd = "FK";
p._id = "FK-" + data.productId;
p.supp = [];
p.supp.push( su );
p.save(
function ( err,res )
{
if ( err )
{
if ( err.code == "11000" )
{
console.log( CSVFilePath + "- duplicate ID - " );
} else
console.log( CSVFilePath + "- save error - " + err );
} else
{
console.log( CSVFilePath + "- save success " );
}
su = null;
p = null;
item = null;
data = null;
csvStream.resume( );
}
);
}
catch ( exc )
{
errcounter++;
console.log( exc.stack );
console.log( "ERROR: Filename:" + CSVFilePath + errcounter.toString( ) );
csvStream.resume( );
}
} )
.on( 'error', function ( error )
{
console.log( "ERROR: " + error.stack );
})
.on( "end", function ()
{
console.log( "Total ERROR: " + errcounter.toString( ) );
console.log( "done" );
});
stream.pipe( csvStream );
}
source
share