Update using upsert, but only update if document date field in db is less than updated document

Question

I have a bit of trouble trying to come up with the logic for this. So, I want to do the following:

• Bulk update heap messages in my remote instance of MongoDB BUT
• If the update is updated only if the field lastModifiedin the remote collection is smaller than lastModifiedin the same document that I am going to update / insert

Basically, I want to update my list of documents if they have been changed since the last update. I can think of two ways of brute force to do this ...

Firstly, by requesting my entire collection, trying to manually delete and replace documents that meet the criteria, add new ones, and then bulk paste everything back into the remote collection after deleting everything on the remote computer.

Secondly, request each item, and then decide if it is on the remote control, if I want to update it or not. It seems that this would be very important when working with remote collections.

If necessary, I work on a NodeJS environment using the mondodbnpm package for database operations.

Answer 1

You can use the API to perform updates based on the logic that you specify how it handles this better. bulkWrite

, , , -, :

mongodb.connect(mongo_url, function(err, db) {
    if(err) console.log(err);
    else {
        var mongo_remote_collection = db.collection("remote_collection_name");

        /* data is from http call to an external service or ideally
           place this within the service callback
        */
        mongoUpsert(mongo_remote_collection, data, function() {
            db.close();
        })
    }
})

function mongoUpsert(collection, data_array, cb) {      
    var ops = data_array.map(function(data) {
        return {
            "updateOne": {
                "filter": { 
                    "_id": data._id, // or any other filtering mechanism to identify a doc
                    "lastModified": { "$lt": data.lastModified }
                },
                "update": { "$set": data },
                "upsert": true
            }
        };
    });

    collection.bulkWrite(ops, function(err, r) {
        // do something with result
    });

    return cb(false);
}

---

, , 500, , 500 .

MongoDB 1000 , 500 , , MongoDB, , > 1000 . , , , 500 .

var ops = [],
    counter = 0;

data_array.forEach(function(data) {
    ops.push({
        "updateOne": {
            "filter": { 
                "_id": data._id,
                "lastModified": { "$lt": data.lastModified }
            },
            "update": { "$set": data },
            "upsert": true
        }
    });
    counter++;

    if (counter % 500 === 0) {
        collection.bulkWrite(ops, function(err, r) {
            // do something with result
        });
        ops = [];
    }
})

if (counter % 500 != 0) {
    collection.bulkWrite(ops, function(err, r) {
        // do something with result
    }
}