Removing the last character from a file stream in node.js (fs module)

Question

Using node.js, I am trying to build an array of objects and write them to a file. For this, I use the built-in fs library.

After calling var file = fs.createWriteStream('arrayOfObjects.json'); and file.write('[') I run several asynchronous functions in order to ultimately add such objects:

file.write(JSON.stringify(objectToAppend) + ',\n')

I can determine when all objects stopped adding, and this is where I run file.write(']') and file.end() . My problem is that adding the last comma to the end of the last object causes the JSON to be invalid.

It is very difficult to determine where and when the last object is created due to the asynchronous nature of the script, so I was wondering if there was a way to remove or remove characters from the file stream. If so, I could do this before adding the last character ']' .

I could do it manually, but I was hoping to transfer this to another application. The only solution I thought of is to use the fs.truncate() function, however this does not work for file streams, and neither file.length nor file.length() will give me the length of the content because it is not a string, therefore It’s difficult to determine how and where to trim the file.

Now I just added '{}]' to the end of the array to make it valid JSON, but this empty object may cause some problems later.

Also note: the array of objects that I write in this stream is VERY large, so I would prefer not to stop the stream and not open the file .

Answer 1

I would recommend adding a separator instead, so you can dynamically configure it after the first call:

 file.write('[\n') var sep = ""; forEach(function(objectToAppen) { file.write(sep + JSON.stringify(objectToAppend)) if (!sep) sep = ",\n"; }); 

Answer 2

An example of using JSONStream :

 var JSONStream = require('JSONStream'); var fs = require('fs'); var jsonwriter = JSONStream.stringify(); var file = fs.createWriteStream('arrayOfObjects.json'); // Pipe the JSON data to the file. jsonwriter.pipe(file); // Write your objects to the JSON stream. jsonwriter.write({ foo : 'bar#1' }); jsonwriter.write({ foo : 'bar#2' }); jsonwriter.write({ foo : 'bar#3' }); jsonwriter.write({ foo : 'bar#4' }); // When you're done, end it. jsonwriter.end(); 

Answer 3

Here is a snippet including robertklep . This will convert from a channelized file to json:

 var fs = require('fs'); var readline = require('readline'); var JSONStream = require('JSONStream'); // Make sure we got a filename on the command line. if (process.argv.length < 3) { console.log('Usage: node ' + process.argv[1] + ' FILENAME'); process.exit(1); } var filename = process.argv[2]; var outputFilename = filename + '.json'; console.log("Converting psv to json. Please wait."); var jsonwriter = JSONStream.stringify(); var outputFile = fs.createWriteStream(outputFilename); jsonwriter.pipe(outputFile); var rl = readline.createInterface({ input: fs.createReadStream(filename), terminal: false }).on('line', function(line) { console.log('Line: ' + line); if(!/ADDRESS_DETAIL_PID/.test(line)) { var split = line.split('|'); var line_as_json = { "address_detail_pid": split[0], "flat_type": split[1], "flat_number": split[2], "level_type": split[3], "level_number": split[4], "number_first": split[5], "street_name": split[6], "street_type_code": split[7], "locality_name": split[8], "state_abbreviation": split[9], "postcode": split[10], "longitude": split[11], "latitude": split[12] }; jsonwriter.write(line_as_json); } }).on('close', () => { jsonwriter.end(); });; console.log('psv2json complete.');