All geek questions in one place

How to handle async Node.js in a loop

I have a loop like this:

var i,j,temparray,chunk = 200;
for (i=0,j=document.mainarray.length; i<j; i+=chunk) {
  temparray = document.mainarray.slice(i,i+chunk);

  var docs =  collection.find({ id: { "$in": temparray}}).toArray();

  docs.then(function(singleDoc)
  {
    if(singleDoc)
    {
      console.log("single doc length : " + singleDoc.length);
      var t;
      for(t = 0, len = singleDoc.length; t < len;t++)
      {
        fs.appendFile("C:/Users/x/Desktop/names.txt", singleDoc[t].name + "\n", function(err) {
          if(err) {
            return console.log(err);
          }
        });
      }
    }
  });
}

The cycle is repeated twice. At the first iteration, he gets 200 elements, and secondly, he gets 130 elements. And when I open the .txt file, I see only 130 names. I think due to the asynchronous nature of Node.js, only the second part of the array is processed. What to do to process all parts of the array? Thanks in advance.

EDIT: I finally included the code:

var generalArr = [];
var i,j,temparray,chunk = 200;
for (i=0,j=document.mainarray.length; i<j; i+=chunk) {
    temparray = document.mainarray.slice(i,i+chunk);

generalArr.push(temparray);


} 

async.each(generalArr, function(item, callback)
{

  var docs =  collection.find({ id: { "$in": item}}).toArray();

   docs.then(function(singleDoc)
  {
    if(singleDoc)
    {
      console.log("single doc length : " + singleDoc.length);
              var t;
        for(t = 0, len = singleDoc.length; t < len;t++)
        {    
           fs.appendFile("C:/Users/x/Desktop/names.txt", singleDoc[t].name + "\n", function(err) {
          if(err) {
          return console.log(err);
          }
        });
        }
    }


  });

  callback(null);
})

When I change this line:

var docs =  collection.find({ id: { "$in": item}}).toArray();

To this line:

var docs =  collection.find({ id: { "$in": item}}).project({ name: 1 }).toArray();

It works, I can print all the names. I think there is a memory problem when I try without .project(). How can I do this work without using a project? Should I change some memory limits? Thanks in advance.

+4
6

, - . node 8 async await. Promise - async/await, .

API node Promise, . promisify promises.

, , ( , ).

, collection.find , , Promise. , API, , , ( promisify).

var findPromise =  collection.find({ id: { "$in": item}});

findPromise find. , , ( ) ( ). , , then:

// The result of collection.find is the collection of matches
findPromise.then(function(docs) {
    // Any code we run here happens asynchronously
});

// Code here will run first

promises ( - , , ) Promise.all, :

var p = new Promise(function(resolve, reject) {
    var findPromise =  collection.find({ id: { "$in": item}});
    findPromise.then(function(docs) {
        var singleDocNames = [];
        for(var i = 0; i < docs.length; i++) {
            var singleDoc = docs[i];
            if(!singleDoc)
                 continue;

            for(var t = 0; t < singleDoc.length; t++)
                singleDocNames.push(singleDoc[t].name);
        }

        // Resolve the outer promise with the final result
        resolve(singleDocNames);
    });
});

// When the promise finishes log it to the console
p.then(console.log);

// Code inline here will fire before the promise

node 8 async/await:

async function p() {
    // Await puts the rest of this function in the .then() of the promise
    const docs = await collection.find({ id: { "$in": item}});

    const singleDocNames = [];
    for(var i = 0; i < docs.length; i++) {
        // ... synchronous code unchanged ...
    }

    // Resolve the outer promise with the final result
    return singleDocNames;
});

// async functions can be treated like promises
p().then(console.log);

, : , , -, , .

+2

, , . .

var i, j, temparray, chunk = 200;
for (i = 0, j = document.mainarray.length; i < j; i += chunk) {
  temparray = document.mainarray.slice(i, i + chunk);
  generalArr.push(temparray);
}
const queryPromises = [];
generalArr.forEach((item, index) => {
  queryPromises.push(collection.find({ id: { "$in": item } }).toArray());
});
let stringToWrite = '';
Promise.all(queryPromises).then((result) => {
  result.forEach((item) => {
    item.forEach((element) => {
      //create a single string which you want to write
      stringToWrite = stringToWrite + "\n" + element.name;
    });
  });
  fs.appendFile("C:/Users/x/Desktop/names.txt", stringToWrite, function (err) {
    if (err) {
      return console.log(err);
    } else {
      // call your callback or return
    }
  });
});

.

  • db.
  • ,
+3

. , ES7 / .

, promises. :

let flowPromise = Promise.resolve();

const chunk = 200;
for (let i=0,j=document.mainarray.length; i<j; i+=chunk) {
    flowPromise = flowPromise.then(() => {
        const temparray = document.mainarray.slice(i,i+chunk);
        const docs =  collection.find({ id: { "$in": temparray}}).toArray();
        return docs.then((singleDoc) => {
            let innerFlowPromise = Promise.resolve();
            if(singleDoc) {
                console.log("single doc length : " + singleDoc.length);
                for(let t = 0, len = singleDoc.length; t < len;t++) {
                    innerFlowPromise = innerFlowPromise.then(() => new Promise((resolve, reject) =>
                        fs.appendFile(
                            "C:/Users/x/Desktop/names.txt", singleDoc[t].name + "\n",
                            err => (err ? reject(err) : resolve())
                        )
                    ));
                }
            }
            return innerFlowPromise;
        }
    });
}

flowPromise.then(() => {
    console.log('Done');
}).catch((err) => {
    console.log('Error: ', err);
})

async- Promises , async, then . async/wait.

+1

nodejs ? async/await, nodejs ( ). , fs.appendFile , , , promisify, appendFileSync - ( , .)

async function(){
    ...
    for(var item of generalArr) {
      var singleDoc = await collection.find({ id: { "$in": item}}).toArray();
      // if(singleDoc) { this won't do anything, since collection.find will always return something even if its just an empty array
      console.log("single doc length : " + singleDoc.length);
      var t;
      for(t = 0, len = singleDoc.length; t < len;t++){    
          fs.appendFileSync("C:/Users/x/Desktop/names.txt", singleDoc[t].name + "\n");
       }

    };
}
+1
var docs =  collection.find({ id: { "$in": document.mainarray}}), // returns a cursor
  doc,
  names = [],
  toInsert;

function saveToFile(cb) {
  toInsert = names.splice(0,100);
  if(!toInsert.length) return cb();
  fs.appendFile("C:/Users/x/Desktop/names.txt", toInsert.join("\n"), cb);
}

(function process() {
  if(docs.hasNext()) {
    doc = docs.next();

    doc.forEach(function(d) {
      names.push(d.name);
    });

    if(names.length === 100) {
      // save when we have 100 names in memory and clear the memory
      saveToFile(function(err) {
        process();
      });
    } else {
       process();
    }
  } else {
    saveToFile(function(){
      console.log('All done');
    });
  }
}()); // invoke the function
0

, , , ( FileSystem).

Here is how you can solve your problem without 3 party libraries, etc.

'use strict';

const
    fs = require('fs');

let chunk = 200;

// How many rounds of array chunking we expect 
let rounds = Math.ceil(mainArray.length/chunk);
// copy to temp (for the counter)
let tempRounds = rounds;
// set file name
let filePath = './names.txt'


// Open writable Stream
let myFileStream = fs.createWriteStream(filePath);


// from round: 0-${rounds}
for (let i = 0; i < rounds; i++) {
    // assume array has ${chunk} elements left in this round
    let tempChunk = chunk;
    // if ${chunk} is to big i.e. i=3 -> chunk = 600 , but mainArray.length = 512
    // This way we adjust the last round for "the leftovers"
    if (mainArray.length < i*chunk) tempChunk = Math.abs(mainArray.length - i*chunk);
    // slice it for this round
    let tempArray = mainArray.slice(i*chunk, i*chunk + tempChunk);
    // get stuff from DB
    let docs =  collection.find({ id: { "$in": tempArray}}).toArray();

    docs.then(function(singleDoc){
        // for each name in the doc
        for (let j = 0; j < singleDoc.length; j++) {
            // write to stream
            myFileStream.write(singleDoc[t].name + "\n");
        }
        // declare round done (reduce tempRounds) and check if it hits 0
        if (!--tempRounds) {
            // if all rounds are done, end the stream
            myFileStream.end();
            // BAM! you done
            console.log("Done")
        }
    });
}

The key is to use fs.WritableStreams :) link here to documents

0
source

Source: https://habr.com/ru/post/1681981/

More articles:


All Articles