Memory leak in Mongoose Node.js

I have code that uses fast-csv to parse a 2GB csv file and paste into MongoDB.

When Pipe executes this code for records around 17K +, Nodejs consumes all the memory and stops with an error:

"CALL_AND_RETRY_LAST Allocation failed - process out of memory"

I tried setting null for variables to avoid possible leakage.

I am using the latest version of Mongoose and Node.js 0.12 ..

PS: If I commented on the p.save () method and ran this code, its execution without any memory leak. I checked up to 5 million records and its good ...

So, I suspect a memory leak in the mongoose save function is possible.

var csv = require( "fast-csv" );
var fs = require( "fs" );
var DBMong = require( "./db-mongoose.js" );

function processCSVnImportDB( CSVFilePath , callback )
{

    var rowcounter = 0;
    var dbObj = DB.getDBObject( ); //Mongoose connection object

    var stream = fs.createReadStream( CSVFilePath );
    var csvStream;
    csvStream = csv( )
        .on( "data",
         function ( data )
    {
        csvStream.pause( );
        try
        {
            rowcounter++;

            var p = new DBMong.ProductModel; //Mongoose Schema Object
            var su = new models.supplierinfo();
            su.cd = "FK";  
            p._id = "FK-" + data.productId;
            p.supp = [];
            p.supp.push( su );

            p.save(
                function ( err,res )
                {
                    if ( err )
                    {
                        if ( err.code == "11000" )
                        {
                            console.log( CSVFilePath + "- duplicate ID - "   );
                        } else
                            console.log( CSVFilePath + "- save error  - " + err );
                    } else
                    {
                                console.log( CSVFilePath + "- save success " );
                    }

                    su = null;
                    p = null;
                    item = null;
                    data = null;

                    csvStream.resume( );
                }
            ); 
        }
        catch ( exc )
        {
            errcounter++;
            console.log( exc.stack );
            console.log( "ERROR: Filename:" + CSVFilePath + errcounter.toString( ) );
            csvStream.resume( );
        }

    } )
       .on( 'error', function ( error )
    {
        console.log( "ERROR: " + error.stack );
    })
    .on( "end", function ()
    {
        console.log( "Total ERROR: " + errcounter.toString( ) );
        console.log( "done" );
    });
    stream.pipe( csvStream );
}
+4
source share

Source: https://habr.com/ru/post/1584505/


All Articles