Read the contents of a Node.js stream into a string variable

Question

I am hacking a node.js program that captures SMTP mail and acts on mail data. The "smtp-protocol" node library provides mail data as a stream, and as a newbie to node.js, I'm not sure how to write this stream to a string variable. I currently have an entry in stdout using the line:

stream.pipe(process.stdout, { end : false }); 

As I said, I need to write this stream data record to a string variable, which I will use as soon as the stream ends.

Help rate!

Answer 1

The key is to use these two Stream Events :

• Event: 'data'
• Event: End

For stream.on('data', ...) you should collect data data in a buffer (if it is binary) or in a string.

For on('end', ...) you must call the callback with the buffer completed, or if you can embed it and use the return using the Promises library.

Answer 2

Hope this is more useful than the answer above (which now has a broken link).

Also note that string concatenation is not an efficient way to collect parts of a string, but it is used for simplicity (and maybe your code doesn't care about efficiency)

 var string = ''; stream.on('data',function(data){ string += data.toString(); console.log('stream data ' + part); }); stream.on('end',function(){ console.log('final output ' + string); }); 

Answer 3

None of this helped me. I needed to use a Buffer object:

  const chunks = []; readStream.on("data", function (chunk) { chunks.push(chunk); }); // Send the buffer or you can put it into a var readStream.on("end", function () { res.send(Buffer.concat(chunks)); }); 

Answer 4

Another way would be to convert the stream to a promise (see the Example below) and use then (or await ) to assign the allowed value to the variable.

 function streamToString (stream) { const chunks = [] return new Promise((resolve, reject) => { stream.on('data', chunk => chunks.push(chunk)) stream.on('error', reject) stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))) }) } const result = await streamToString(stream) 

Answer 5

I usually use this simple function to convert a stream to a string:

 function streamToString(stream, cb) { const chunks = []; stream.on('data', (chunk) => { chunks.push(chunk.toString()); }); stream.on('end', () => { cb(chunks.join('')); }); } 

Usage example:

 let stream = fs.createReadStream('./myFile.foo'); streamToString(stream, (data) => { console.log(data); // data is now my string variable }); 

Answer 6

In the nodejs documentation, you should do this - always remember the line without knowing that the encoding is just a bunch of bytes:

 var readable = getReadableStreamSomehow(); readable.setEncoding('utf8'); readable.on('data', function(chunk) { assert.equal(typeof chunk, 'string'); console.log('got %d characters of string data', chunk.length); }) 

Answer 7

There is no simple .toString() function in the streams (which I understand), but not something like the .toStringAsync(cb) function (which I do not understand).

So, I created my own helper function:

 var streamToString = function(stream, callback) { var str = ''; stream.on('data', function(chunk) { str += chunk; }); stream.on('end', function() { callback(str); }); } // how to use: streamToString(myStream, function(myStr) { console.log(myStr); }); 

Answer 8

I was luckier using it like this:

 let string = ''; readstream .on('data', (buf) => string += buf.toString()) .on('end', () => console.log(string)); 

I am using node v9.11.1 and readstream is the response from the http.get .

Answer 9

A simple way with the popular (over 5 million downloads per week) and easy get-stream library:

https://www.npmjs.com/package/get-stream

 const fs = require('fs'); const getStream = require('get-stream'); (async () => { const stream = fs.createReadStream('unicorn.txt'); console.log(await getStream(stream)); //output is string })(); 

Answer 10

How about something like a flow reducer?

Here is an example of using ES6 classes, how to use it.

 var stream = require('stream') class StreamReducer extends stream.Writable { constructor(chunkReducer, initialvalue, cb) { super(); this.reducer = chunkReducer; this.accumulator = initialvalue; this.cb = cb; } _write(chunk, enc, next) { this.accumulator = this.reducer(this.accumulator, chunk); next(); } end() { this.cb(null, this.accumulator) } } // just a test stream class EmitterStream extends stream.Readable { constructor(chunks) { super(); this.chunks = chunks; } _read() { this.chunks.forEach(function (chunk) { this.push(chunk); }.bind(this)); this.push(null); } } // just transform the strings into buffer as we would get from fs stream or http request stream (new EmitterStream( ["hello ", "world !"] .map(function(str) { return Buffer.from(str, 'utf8'); }) )).pipe(new StreamReducer( function (acc, v) { acc.push(v); return acc; }, [], function(err, chunks) { console.log(Buffer.concat(chunks).toString('utf8')); }) ); 

Answer 11

The cleanest solution would be to use the string-stream package, which converts the stream into a promise string.

 const streamString = require('stream-string') streamString(myStream).then(string_variable => { // myStream was converted to a string, and that string is stored in string_variable console.log(string_variable) }).catch(err => { // myStream emitted an error event (err), so the promise from stream-string was rejected throw err }) 

Answer 12

This worked for me and is based on Node v6.7.0 docs :

 let output = ''; stream.on('readable', function() { let read = stream.read(); if (read !== null) { // New stream data is available output += read.toString(); } else { // Stream is now finished when read is null. // You can callback here eg: callback(null, output); } }); stream.on('error', function(err) { callback(err, null); }) 

Answer 13

setEncoding ('utf8');

Well done Sebastian J. Above.

I had a “buffer problem” with several lines of test code that I had and added the encoding information and this solved it, see below.

Demonstrate problem

software

 // process.stdin.setEncoding('utf8'); process.stdin.on('data', (data) => { console.log(typeof(data), data); }); 

entrance

 hello world 

exit

 object <Buffer 68 65 6c 6c 6f 20 77 6f 72 6c 64 0d 0a> 

Demonstrate solution

software

 process.stdin.setEncoding('utf8'); // <- Activate! process.stdin.on('data', (data) => { console.log(typeof(data), data); }); 

entrance

 hello world 

exit

 string hello world 

Answer 14

Using the fairly popular stream-buffers package that you probably already have in your project dependencies is pretty simple:

 // imports const { WritableStreamBuffer } = require('stream-buffers'); const { promisify } = require('util'); const { createReadStream } = require('fs'); const pipeline = promisify(require('stream').pipeline); // sample stream let stream = createReadStream('/etc/hosts'); // pipeline the stream into a buffer, and print the contents when done let buf = new WritableStreamBuffer(); pipeline(stream, buf).then(() => console.log(buf.getContents().toString()));