How do I download a package on s3 and paste it into MongoDB from a nodeJS web server with the final callback?

Question

I have a web server that accepts images from a client, processes them, uploads them to S3, inserts inserts URLs into my mongoDB and finally sends the json result back to the client.

Working with a single image works as follows:

router.post("/upload", function(req, res){
    var form = new multiparty.Form();

    form.parse(req,function(err,fields,files){
        s3.upload({
                Key: filename,
                Bucket: bucketname,
                ACL: "public-read",
                Body: fs.createReadStream(filepath)
         }, function(err, data){
            if(err) //error handle

            Model.collection.insert({"name": "name", "url" : data.Location}, function(err, result){
                if(err) //error handle
                res.json({result: result})
            })
        })
    })
})

This works very well since I just upload the file data to s3 -> when it is done, insert s3 output (url) into the database -> when it is done, send the mongo result as jsonarray to the client.

, html type=file name=images, images[i]. images.length . , jsonarray , S3 upload- > mongo, , .

:

• , S3, URL- ([data.Location]). mongoDB jsonarray . , mongo S3.
• , S3 mongoDB . if (currentIndex = images.length), jsonarray. , , ( ).

s3, mongo, , s3 urls, filename .. jsonarray?

, !

Answer 1

Promises, . Bluebird.

S3 Promise.all(), , Mongo, , . , , : upload->insert to mongo, , . , . Promise.map() concurrency, , .

:

, getFiles, uploadFile uploadToMongo Promise.

var maxConcurrency =  10;

getFiles()
    .map(function(file){
        return uploadFile(file)
            .then(uploadToMongo) 
    },{concurrency: maxConcurrency})
    .then(function(){
            return finalCallback();
    }).catch(handleError);

, "promisify * S3:

function uploadMyFile(filename, filepath, bucketname) {
    return new Promise(function(resolve, reject){
        s3.upload({
            Key: filename,
            Bucket: bucketname,
            ACL: "public-read",
            Body: fs.createReadStream(filepath)
        }, function(err, data){
            //This err will get to the "catch" statement.
            if (err) return reject(err);
            // Handle success and eventually call:
            return resolve(data);
        });
    });
}

:

uploadMyFile
        .then(handleSuccess)
        .catch(handleFailure);

!

Answer 2

promises, . , .

, :

router.post("/upload", function(req, res){
    var form = new multiparty.Form();

    form.parse(req,function(err,fields,files){
        if (err){
            cb(err);
        } else {
            bulkUpload(files, fields, function(err, result){
                if (err){
                    cb(err);
                } else {
                    res.json({result:result});
                }
            })
        }
    })
})

function singleUpload(file, field, cb){
    s3.upload({
            Key: filename,
            Bucket: bucketname,
            ACL: "public-read",
            Body: fs.createReadStream(filepath)
     }, function(err, data){
        if(err)
            cb(err);
        } else {
            Model.collection.insert({"name": "name", "url" : data.Location}, function(err, result){
                if(err){ 
                    cb(err);
                } else {
                    cb(null, result);
                }
            })
        }
    })
}

function bulkUpload (files, fields, cb) {
    var count     = files.length;
    var successes = 0;
    var errors    = 0;

    for (i=0;i<files.length;i++) {
        singleUpload(files[i], fields[i], function (err, res) {
            if (err) {
                errors++
                //do something with the error?
            } else {
                successes++
                //do something with the result?
            }

            //when you have worked through all of the files, call the final callback
            if ((successes + errors) >= count) {
                cb(
                    null, 
                    {
                        successes:successes,
                        errors:errors
                    }
                )
            }
        });
    }
}

, promises. , .

!