Does consecutive HTTP requests perform a blocking operation on node?

Question

Please note that irrelevant information to my question will be “quoted”

so (feel free to skip them).

Problem

I use node to make HTTP requests in order on behalf of several clients. Thus, due to the fact that initially the client took several different page loads to get the desired result, now there is only one request to my server. I am currently using the async module for flow control and the request module for HTTP requests. There are about 5 callbacks that, using console.time, take about 2 seconds from start to finish (sketch code is included below).

Now I'm pretty inexperienced with node, but I know about the single-threaded nature of node. Although I read many times that nodeisnt is built for CPU related tasks, I really did not understand what it is so far. If I understand correctly what is happening, this means that what I currently have (in development) is in no way going to scale to more than 10 clients.

Question

Since I'm not an expert at node, I ask this question (in the header) to get confirmation that running multiple consecutive HTTP requests really blocks.

Epilogue

If this is the case, I expect that I will ask another SO question (after conducting the appropriate research), discussing various possible solutions, should I continue to search for this problem in node (which in itself may not be suitable for what I am trying to do).

Other closing thoughts

, , ( ).

, !

, :

var async = require('async');
var request = require('request');

...

async.waterfall([
    function(cb) {
        console.time('1');

        request(someUrl1, function(err, res, body) {
            // load and parse the given web page.

            // make a callback with data parsed from the web page
        });
    },
    function(someParameters, cb) {
        console.timeEnd('1');
        console.time('2');

        request({url: someUrl2, method: 'POST', form: {/* data */}}, function(err, res, body) {
            // more computation

            // make a callback with a session cookie given by the visited url
        });
    },
    function(jar, cb) {
        console.timeEnd('2');
        console.time('3');

        request({url: someUrl3, method: 'GET', jar: jar /* cookie from the previous callback */}, function(err, res, body) {
            // do more parsing + computation

            // make another callback with the results
        });
    },
    function(moreParameters, cb) {
        console.timeEnd('3');
        console.time('4');

        request({url: someUrl4, method: 'POST', jar: jar, form : {/*data*/}}, function(err, res, body) {
            // make final callback after some more computation.
            //This part takes about ~1s to complete
        });
    }
], function (err, result) {
    console.timeEnd('4'); //
    res.status(200).send();
});

Answer 1

I/O node.js . , . , 1 , 2 2 , .

sync-request request. , .

, -:

var req = require('request');
var sync = require('sync-request');

// Load example.com N times (yes, it a real website):
var N = 10;

console.log('BLOCKING test ==========');
var start = new Date().valueOf();
for (var i=0;i<N;i++) {
    var res = sync('GET','http://www.example.com')
    console.log('Downloaded ' + res.getBody().length + ' bytes');
}
var end = new Date().valueOf();
console.log('Total time: ' + (end-start) + 'ms');

console.log('NON-BLOCKING test ======');
var loaded = 0;
var start = new Date().valueOf();
for (var i=0;i<N;i++) {
    req('http://www.example.com',function( err, response, body ) {
        loaded++;
        console.log('Downloaded ' + body.length + ' bytes');
        if (loaded == N) {
            var end = new Date().valueOf();
            console.log('Total time: ' + (end-start) + 'ms');
        }
    })
}

, , , (, N = 10, 10 , ). , .

---

:

, , . CPU. (I/O, , , ), . , , :

async.waterfall([
    function(cb) {
        request(someUrl1, function(err, res, body) {
            console.time('1');
            // load and parse the given web page.
            console.timeEnd('1');
            // make a callback with data parsed from the web page
        });
    },
    function(someParameters, cb) {
        request({url: someUrl2, method: 'POST', form: {/* data */}}, function(err, res, body) {
            console.time('2');
            // more computation
            console.timeEnd('2');

            // make a callback with a session cookie given by the visited url
        });
    },
    function(jar, cb) {
        request({url: someUrl3, method: 'GET', jar: jar /* cookie from the previous callback */}, function(err, res, body) {
            console.time('3');
            // do more parsing + computation
            console.timeEnd('3');
            // make another callback with the results
        });
    },
    function(moreParameters, cb) {
        request({url: someUrl4, method: 'POST', jar: jar, form : {/*data*/}}, function(err, res, body) {
            console.time('4');
            // some more computation.
            console.timeEnd('4');

            // make final callback
        });
    }
], function (err, result) {
    res.status(200).send();
});

" ". , , . , , node . , ( , 1 ) node javascript- .

Answer 2

, - request(). , node.js , HTTP- .

async.waterfall() , . , async.waterfall() . , , , .

, , setTimeout(). , 5 , (, async.waterfall() n , ):

setTimeout(function() {
    setTimeout(function() {
        setTimeout(function() {
            setTimeout(function() {
                setTimeout(function() {
                    // it takes 5 seconds to get here
                }, 1000);
            }, 1000);
        }, 1000);
    }, 1000);
}, 1000);

, 5 . node.js , , 1 , setTimeout(), node.js , .

, node.js:

Node?

http-

Javascript/ Node, : , ?

JavaScript AJAX ? ( , )

, , , , ( ), 10 .

. node.js . - - (. ).

node . , , , , , - -, . , , . , /, , ? BS , .

- ( , , , ), compmuter , . , node.js -, , .

(, ), , . . ( node.js, ) nodejs. , - , - . . , node.js - , node.js .

Answer 3

http- nodeJs https://www.npmjs.com/package/concurrent-queue

    var cq = require('concurrent-queue');
    test_queue = cq();

    // request action method
    testQueue: function(req, res) {
        // queuing each request to process sequentially
        test_queue(req.user, function (err, user) {
            console.log(user.user_id+' done');
            res.json(200, user)
        });
    },


    // Queue will be processed one by one.
    test_queue.limit({ concurrency: 1 }).process(function (user, cb) {
        console.log(user.user_id + ' started')

        // async calls will go there
        setTimeout(function () {
            // on callback of async, call cb and return response.
            cb(null, user)
        }, 1000);

    });

, , -, .

- , .

:

, . , , ,

, . http , https .

testQueue: function(req, res) {

    // if queue not exist for current user.
    if(! (test_queue.hasOwnProperty(req.user.user_id)) ){
        // initialize queue for current user
        test_queue[req.user.user_id] = cq();
        // initialize queue processing for current user
        // Queue will be processed one by one.
        test_queue[req.user.user_id].limit({ concurrency: 1 }).process(function (task, cb) {
            console.log(task.user_id + ' started')
            // async functionality will go there
            setTimeout(function () {
                cb(null, task)
            }, 1000)
        });
    }

    // queuing each request in user specific queue to process sequentially
    test_queue[req.user.user_id](req.user, function (err, user) {
        if(err){
            res.serverError;
            return;
        }
        res.ok(user);
        console.log(user.user_id+' done');
    });
},

/ , .