Node js CSV for JSON (large file)

Question

I need to convert strings to csv file in json. I use the csvtojson package and works like a charm, but my csv file is very large, about 21,000,000 lines. :(

My node application can process all lines.

See my snippet below.

csv({noheader:true,
      delimiter:";",
    workerNum: 4})
  .fromFile(csvFilePath)
  .on('json',(jsonObj, rowIndex)=>{
      var TN = jsonObj.field1;
      var cod_op = jsonObj.field2;
      var msisdn = TN.toString();

     //UP TO FIREBASE HERE!!!

        noticia.child(msisdn).set(cod_op);
        console.log(TN + ' ' + rowIndex);



  })
  .on('done',(error)=>{
      console.log()
  })

Answer 1

I decided myself using csv-stream module

Here is a snippset

  var options = {
    delimiter : ';', // default is ,
    endLine : '\n', // default is \n,
    columns : ['VERSION_ID', 'TN','RN1','DATA','OPERACAO','CNL','EOT'], // by default read the first line and use values found as columns
    columnOffset : 7, // default is 0
    escapeChar : '"', // default is an empty string
    enclosedChar : '"' // default is an empty string
}

var csvStream = csv.createStream(options);
i = 0
request('http://yourhugefile.csv').pipe(csvStream)
    .on('error',function(err){
        console.error(err);
    })
    .on('data',function(data){
        // outputs an object containing a set of key/value pair representing a line found in the csv file.
        //console.log(data);
    })
    .on('column',function(key,value){
      if(key == 'TN' || key == 'RN1')
          firebaseUpload(value);
        // outputs the column name associated with the value found
        //console.log('#' + key + ' = ' + value);
    })